1use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use ssp2::encode_message;
12use std::borrow::Cow;
13#[cfg(test)]
14use std::cell::Cell;
15use std::cell::RefCell;
16use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
17use std::sync::Arc;
18
19use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
20use rusqlite::{Connection, OpenFlags, OptionalExtension};
21use serde::{Deserialize, Serialize};
22use serde_json::{json, Map, Value};
23use sha2::{Digest, Sha256};
24use ssp2::decode::WIRE_VERSION;
25use ssp2::model::{
26 FencedRecord, Frame, LiveStatus, MediaType, Message, MsgKind, Op, OpResult, PushStatus,
27 SubStatus,
28};
29use ssp2::primitives::RawJson;
30use ssp2::registration::canonical_scope_json;
31use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
32use ssp2::{decode_message, encode_presence_publish, parse_control, ControlMessage, PresenceKind};
33
34use crate::api::{
35 AppliedServerCommit, AppliedServerCommitSource, ClientChangeBatch, ClientDiagnosticsHost,
36 ClientDiagnosticsLease, ClientDiagnosticsReplica, ClientDiagnosticsSchema,
37 ClientDiagnosticsSnapshot, ClientDiagnosticsStorage, ClientError, ClientLimits, CommandEffects,
38 CommitDelivery, CommitOperation, CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery,
39 CommitOutcomeResolution, CommitOutcomeStatus, ConflictRecord, CoverageSnapshot,
40 CurrentBaseProof, CurrentBaseProofReason, DiagnosticLastChange, DiagnosticLastRound,
41 DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticUnits, EchoRecord, FetchedBlob,
42 LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
43 LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
44 PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
45 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RetainedCommitRow,
46 RetainedUniqueConflict, RowState, SchemaFloor, SnapshotRead, SyncIntent, SyncOutcome,
47 SyncReport, SyncStatusSnapshot, TableChange, UnitCatchup, UnitRecord, UnitScopes, UnitStatus,
48 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
49 CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_QUERY_FAILURES,
50};
51use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
52#[cfg(feature = "bench-internals")]
53use crate::bench::{Phase, Recorder};
54use crate::previous_version::{
55 capture_previous_version_from_replica, first_incompatibility,
56 reconcile_previous_version_at_boot, set_local_schema_descriptor,
57 sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
58 PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
59};
60use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
61use crate::transport::{BlobDownload, SegmentRequest, Transport, TransportError};
62use crate::values::{
63 bytes_to_hex, column_value_to_json, decode_row_bytes, encode_sparse_row_json, full_row_values,
64 is_canonical_bytes_envelope, json_to_column_value, json_to_scope_map, normalize_values_casing,
65 render_row_id_json, scope_map_to_json, SealedRow,
66};
67
68const DEFAULT_ACCEPT: u8 = 0b0111;
72const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
73const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
74const ACCEPT_SQLITE: u8 = 1 << 2;
75const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
76const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
77
78fn recommended_action_of(details: &Option<Value>) -> Option<String> {
80 details
81 .as_ref()?
82 .get("recommendedAction")?
83 .as_str()
84 .map(str::to_owned)
85}
86
87pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
89const LOCAL_REVISION_KEY: &str = "localRevision";
90const REPLICA_CURSOR_KEY: &str = "replicaCursor";
92const ECHO_DIGEST_KEY: &str = "echoDigest";
94const REGISTERED_DIGEST_KEY: &str = "registrationDigest";
96const TABLE_GENERATIONS_KEY: &str = "localTableGenerations";
98const OUTBOX_ID_KEY: &str = "outboxId";
100const CLIENT_ID_KEY: &str = "clientId";
101const LEASE_STATE_KEY: &str = "leaseState";
102const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
103pub(crate) const LOG_EPOCH_KEY: &str = "logEpoch";
104const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
109const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
110const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
111const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
114const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
117const INVALID_REQUEST_CODE: &str = "sync.invalid_request:";
118const LEGACY_VALUE_REFUSAL_MESSAGE: &str =
120 "the persisted commit carries values the current codec refuses";
121const INVALID_STORED_VALUES_REASON: &str = "invalid_stored_values";
123
124#[derive(Debug, Serialize, Deserialize)]
125#[serde(rename_all = "camelCase", deny_unknown_fields)]
126struct PersistedLocalDataRebootstrapReceipt {
127 version: u8,
128 retained_commits: u64,
129 reset_subscriptions: u64,
130}
131
132fn invalid_local_rebootstrap_receipt() -> String {
133 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
134}
135
136fn encode_local_rebootstrap_receipt(
137 retained_commits: usize,
138 reset_subscriptions: usize,
139) -> Result<String, String> {
140 let retained_commits =
141 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
142 let reset_subscriptions =
143 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
144 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
145 return Err(invalid_local_rebootstrap_receipt());
146 }
147 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
148 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
149 retained_commits,
150 reset_subscriptions,
151 })
152 .map_err(|_| invalid_local_rebootstrap_receipt())
153}
154
155fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
156 if value == "v1" {
158 return Ok((0, 0));
159 }
160 let receipt: PersistedLocalDataRebootstrapReceipt =
161 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
162 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
163 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
164 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
165 {
166 return Err(invalid_local_rebootstrap_receipt());
167 }
168 Ok((
169 usize::try_from(receipt.retained_commits)
170 .map_err(|_| invalid_local_rebootstrap_receipt())?,
171 usize::try_from(receipt.reset_subscriptions)
172 .map_err(|_| invalid_local_rebootstrap_receipt())?,
173 ))
174}
175
176#[cfg(test)]
177mod observation_tests;
178
179#[doc(hidden)]
181pub struct CurrentBasePreparation {
182 client_commit_id: String,
183 sequence: i64,
184 units: Vec<UnitRecord>,
185 transport_epoch: u64,
186 registration_epoch: u64,
187 security_epoch: u64,
188}
189
190#[cfg(test)]
191mod units_tests;
192
193#[derive(Debug, Clone)]
194struct OutboxOp {
195 upsert: bool,
196 table: String,
197 row_id: String,
198 base_version: Option<i64>,
199 values: Option<Map<String, Value>>,
203 insert: bool,
207}
208
209impl OutboxOp {
210 fn presented_values(&self, table: &TableSchema) -> Option<Cow<'_, Map<String, Value>>> {
215 let values = self.values.as_ref()?;
216 if !self.insert || table.columns.iter().all(|c| values.contains_key(&c.name)) {
217 return Some(Cow::Borrowed(values));
218 }
219 full_row_values(table, values.clone()).ok().map(Cow::Owned)
220 }
221}
222
223impl From<CommitOperation> for OutboxOp {
224 fn from(operation: CommitOperation) -> Self {
225 Self {
226 upsert: operation.op == "upsert",
227 table: operation.table,
228 row_id: operation.row_id,
229 base_version: operation.base_version,
230 values: operation.values,
231 insert: operation.insert,
232 }
233 }
234}
235
236fn invalid_authoring(message: &str, details: Value) -> ClientError {
238 ClientError {
239 code: "sync.invalid_request".to_owned(),
240 message: message.to_owned(),
241 details: Some(details),
242 retryable: false,
243 }
244}
245
246fn chains_on(earlier: &OutboxOp, later: &OutboxOp) -> bool {
249 earlier.row_id == later.row_id
251 && earlier.table == later.table
252 && later.base_version.is_some_and(|base| base > 0)
253 && earlier.base_version == later.base_version
254 && later.values.as_ref().is_some_and(|values| {
255 values.keys().all(|column| {
256 earlier
257 .values
258 .as_ref()
259 .is_some_and(|earlier| earlier.contains_key(column))
260 })
261 })
262}
263
264impl From<&OutboxOp> for CommitOperation {
265 fn from(operation: &OutboxOp) -> Self {
266 Self {
267 table: operation.table.clone(),
268 row_id: operation.row_id.clone(),
269 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
270 base_version: operation.base_version,
271 values: operation.values.clone(),
272 insert: operation.insert,
273 }
274 }
275}
276
277#[derive(Debug, Clone)]
278struct OutboxCommit {
279 client_commit_id: String,
280 ops: Vec<OutboxOp>,
281}
282
283fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>, bool)> {
287 commit
288 .ops
289 .iter()
290 .filter(|operation| operation.upsert)
291 .map(|operation| {
292 (
293 operation.table.as_str(),
294 operation
295 .values
296 .as_ref()
297 .map(|values| values.keys().map(String::as_str).collect())
298 .unwrap_or_default(),
299 operation.insert,
300 )
301 })
302 .collect()
303}
304
305type FailedCommit = (OutboxCommit, Map<String, Value>);
306
307#[derive(Debug, Clone)]
308struct CompiledLocalDataPurgeTarget {
309 table: String,
310 selectors: Vec<(String, Vec<String>)>,
311}
312
313struct StoredCommitOutcomeRow {
314 sequence: i64,
315 client_commit_id: String,
316 status: String,
317 recorded_at_ms: i64,
318 results_json: String,
319 operations_json: Option<String>,
320 resolution: String,
321 resolved_at_ms: Option<i64>,
322 replacement_client_commit_id: Option<String>,
323}
324
325impl StoredCommitOutcomeRow {
326 fn from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<Self> {
327 Ok(Self {
328 sequence: row.get(0)?,
329 client_commit_id: row.get(1)?,
330 status: row.get(2)?,
331 recorded_at_ms: row.get(3)?,
332 results_json: row.get(4)?,
333 operations_json: row.get(5)?,
334 resolution: row.get(6)?,
335 resolved_at_ms: row.get(7)?,
336 replacement_client_commit_id: row.get(8)?,
337 })
338 }
339}
340
341struct SectionStart {
345 id: String,
346 status: SubStatus,
347 reason_code: String,
348 bootstrap: bool,
349 fenced: Vec<FencedRecord>,
350}
351
352enum SectionError {
353 FailClosed,
354 Refused,
356 Deferred,
359 Abort(String, String),
360 Transport(TransportError),
361}
362
363#[derive(Debug)]
364pub(crate) struct RequestMeta {
365 pushed_ids: Vec<String>,
366 registration_digest: String,
369 pub(crate) accept: u8,
370 deferred_commits: usize,
374 pub(crate) applied: BTreeMap<String, BTreeMap<String, i64>>,
377 pub(crate) deferred: BTreeSet<(String, String)>,
381}
382
383#[derive(Default)]
384struct ChangeAccumulator {
385 tables: BTreeMap<String, Option<BTreeSet<String>>>,
387 windows: BTreeMap<String, BTreeSet<String>>,
388 status: bool,
389 conflicts: bool,
390 rejections: bool,
391 outcomes: bool,
392}
393
394impl ChangeAccumulator {
395 fn table(&mut self, table: &str) {
396 self.tables.insert(table.to_owned(), None);
397 }
398
399 fn scope(&mut self, table: &str, key: String) {
400 match self.tables.get_mut(table) {
401 Some(None) => {}
402 Some(Some(keys)) => {
403 keys.insert(key);
404 }
405 None => {
406 self.tables
407 .insert(table.to_owned(), Some(BTreeSet::from([key])));
408 }
409 }
410 }
411
412 fn window(&mut self, base_key: &str, value: &str) {
413 self.windows
414 .entry(base_key.to_owned())
415 .or_default()
416 .insert(value.to_owned());
417 }
418
419 fn touched(&self) -> bool {
420 !self.tables.is_empty()
421 || !self.windows.is_empty()
422 || self.status
423 || self.conflicts
424 || self.rejections
425 || self.outcomes
426 }
427}
428
429const PUSH_OPS_PER_REQUEST: usize = 500;
435const MAX_LOCAL_PURGE_TARGETS: usize = 64;
436const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
437const MAX_LOCAL_PURGE_VALUES: usize = 128;
438const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
439pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
440
441struct OverlayDirty(RefCell<Option<BTreeSet<String>>>);
443
444impl OverlayDirty {
445 fn clean() -> Self {
446 Self(RefCell::new(Some(BTreeSet::new())))
447 }
448 fn get(&self) -> bool {
449 self.0
450 .borrow()
451 .as_ref()
452 .is_none_or(|tables| !tables.is_empty())
453 }
454 fn set(&self, dirty: bool) {
455 *self.0.borrow_mut() = if dirty { None } else { Some(BTreeSet::new()) };
456 }
457 fn table(&self, table: &str) {
458 if let Some(tables) = self.0.borrow_mut().as_mut() {
459 tables.insert(table.to_owned());
460 }
461 }
462 fn snapshot(&self) -> Option<BTreeSet<String>> {
463 self.0.borrow().clone()
464 }
465 fn restore(&self, tables: Option<BTreeSet<String>>) {
466 *self.0.borrow_mut() = tables;
467 }
468}
469
470#[derive(Debug, Clone, Default)]
473struct Shared<T>(Arc<T>);
474
475impl<T> std::ops::Deref for Shared<T> {
476 type Target = T;
477 fn deref(&self) -> &T {
478 &self.0
479 }
480}
481
482impl<T: Clone> std::ops::DerefMut for Shared<T> {
483 fn deref_mut(&mut self) -> &mut T {
484 Arc::make_mut(&mut self.0)
485 }
486}
487
488impl<T> From<T> for Shared<T> {
489 fn from(value: T) -> Self {
490 Self(Arc::new(value))
491 }
492}
493
494#[derive(Debug, Clone)]
498struct Rollback {
499 outbox: Shared<Vec<Arc<OutboxCommit>>>,
500 failed_commits: Shared<Vec<FailedCommit>>,
501 conflicts: Shared<Vec<ConflictRecord>>,
502 rejections: Shared<Vec<RejectionRecord>>,
503 schema_floor: Option<SchemaFloor>,
504 lease_state: Option<LeaseState>,
505 stopped: bool,
506 upgrading: bool,
507 sync_needed: bool,
508 active_round: Option<uuid::Uuid>,
509 overlay_dirty: Option<BTreeSet<String>>,
510 change_queue: usize,
511 sync_intent_queue: usize,
512 last_change: Option<DiagnosticLastChange>,
513}
514
515pub struct SyncClient {
516 #[cfg(feature = "bench-internals")]
517 benchmark_phases: Recorder,
518 conn: Connection,
519 schema: ClientSchema,
520 client_id: String,
521 limits: ClientLimits,
522 outbox: Shared<Vec<Arc<OutboxCommit>>>,
523 failed_commits: Shared<Vec<FailedCommit>>,
524 retain_failed_commits: bool,
525 conflicts: Shared<Vec<ConflictRecord>>,
526 rejections: Shared<Vec<RejectionRecord>>,
527 schema_floor: Option<SchemaFloor>,
528 lease_state: Option<LeaseState>,
530 stopped: bool,
532 upgrading: bool,
534 sync_needed: bool,
536 realtime_policy: RealtimePolicy,
539 realtime_state: RealtimeState,
540 realtime_reason_code: Option<String>,
541 realtime_retry_delay_ms: Option<u64>,
542 presence: HashMap<String, HashMap<String, PresencePeer>>,
544 now_ms: Option<std::sync::Arc<std::sync::atomic::AtomicI64>>,
548 encryption: crate::values::EncryptionConfig,
552 security_preflight: bool,
555 security_epoch: u64,
556 transport_enabled: bool,
557 transport_epoch: u64,
558 registration_epoch: u64,
561 previous_version: Option<PreviousVersionContextConfig>,
565 insert_sql: RefCell<HashMap<String, String>>,
571 overlay_dirty: OverlayDirty,
574 storage_failure: RefCell<Option<QueryReadFailure>>,
576 #[cfg(test)]
579 overlay_rebuild_count: Cell<usize>,
580 #[cfg(test)]
581 acknowledged_replay_count: Cell<usize>,
582 #[cfg(test)]
585 outcome_prune_count: Cell<usize>,
586 #[cfg(test)]
588 image_chunk_ms: Cell<Option<f64>>,
589 progress: ProgressObserver,
591 change_queue: VecDeque<ClientChangeBatch>,
592 sync_intent_queue: VecDeque<SyncIntent>,
593 retry_delay_ms: u64,
595 round_retry_delay_ms: Option<u64>,
597 image_chunk_rows: HashMap<String, u32>,
600 round_failure: Option<String>,
605 active_round: Option<uuid::Uuid>,
606 last_round: Option<DiagnosticLastRound>,
607 last_change: Option<DiagnosticLastChange>,
608 query_failures: Vec<DiagnosticQueryFailure>,
610 coalesced: Option<CoalescedApply>,
612 coalesced_failure: Option<String>,
615}
616
617struct CoalescedApply {
621 rows: usize,
622 rollback: Rollback,
623}
624
625const COALESCED_APPLY_ROWS: usize = 1024;
626
627pub(crate) fn quote_ident(name: &str) -> String {
628 format!("\"{}\"", name.replace('"', "\"\""))
629}
630pub(crate) fn meta_get(conn: &Connection, key: &str) -> rusqlite::Result<Option<String>> {
634 conn.query_row(
635 "SELECT value FROM _syncular_meta WHERE key = ?1",
636 rusqlite::params![key],
637 |row| row.get::<_, String>(0),
638 )
639 .optional()
640}
641
642pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<()> {
643 conn.execute(
644 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
645 rusqlite::params![key, value],
646 )
647 .map(|_| ())
648}
649
650pub(crate) fn meta_delete(conn: &Connection, key: &str) -> rusqlite::Result<()> {
651 conn.execute(
652 "DELETE FROM _syncular_meta WHERE key = ?1",
653 rusqlite::params![key],
654 )
655 .map(|_| ())
656}
657
658fn row_id_predicate(table: &TableSchema) -> String {
660 let key = quote_ident(&table.primary_key);
661 let text_match = format!("CAST({key} AS TEXT) = ?1");
662 match table.columns[table.pk_index].ty {
663 ColumnType::String | ColumnType::Json => {
664 format!("{key} = ?1 AND {text_match}")
665 }
666 ColumnType::Integer | ColumnType::Boolean => {
670 format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
671 }
672 _ => text_match,
675 }
676}
677
678fn local_scope_predicate(
685 table: &TableSchema,
686 effective: &[(String, Vec<String>)],
687 owners: &[Vec<(String, Vec<String>)>],
688) -> Result<Option<(String, Vec<SqlValue>)>, ()> {
689 if effective.is_empty() || effective.iter().any(|(_, values)| values.is_empty()) {
690 return Ok(None);
691 }
692 let column = |variable: &str| {
693 let name = table.scope_column(variable).ok_or(())?;
694 let column = table
695 .columns
696 .iter()
697 .find(|column| column.name == name)
698 .ok_or(())?;
699 Ok((quote_ident(name), column.ty == ColumnType::Integer))
700 };
701 let mut clauses = Vec::new();
702 let mut params = Vec::new();
703 for (variable, values) in effective {
704 let (key, integer) = column(variable)?;
705 let holes = vec!["?"; values.len()].join(", ");
706 clauses.push(if integer {
707 params.extend(values.iter().cloned().map(SqlValue::Text));
708 let seek = vec!["+CAST(? AS INTEGER)"; values.len()].join(", ");
709 format!("{key} IN ({seek}) AND CAST({key} AS TEXT) IN ({holes})")
710 } else {
711 format!("{key} IN ({holes})")
712 });
713 params.extend(values.iter().cloned().map(SqlValue::Text));
714 }
715 let mut groups: BTreeMap<Vec<&str>, Vec<Vec<&[String]>>> = BTreeMap::new();
721 for owner in owners {
722 if owner.is_empty()
723 || owner.iter().any(|(variable, values)| {
724 values.is_empty()
725 || effective.iter().any(|(name, wanted)| {
726 name == variable && wanted.iter().all(|value| !values.contains(value))
727 })
728 })
729 {
730 continue;
731 }
732 let mut owner: Vec<_> = owner.iter().collect();
733 owner.sort_by(|a, b| a.0.cmp(&b.0));
734 groups
735 .entry(
736 owner
737 .iter()
738 .map(|(variable, _)| variable.as_str())
739 .collect(),
740 )
741 .or_default()
742 .push(owner.iter().map(|(_, values)| values.as_slice()).collect());
743 }
744 let within = |variable: &str, list: &str| {
745 let (key, integer) = column(variable)?;
746 let key = if integer {
747 format!("CAST({key} AS TEXT)")
748 } else {
749 key
750 };
751 Ok(format!("{key} IN (SELECT value FROM json_each({list}))"))
752 };
753 for (variables, tuples) in groups {
754 if let [variable] = variables[..] {
755 let values: BTreeSet<&String> = tuples.iter().flatten().copied().flatten().collect();
756 params.push(SqlValue::Text(
757 Value::from_iter(values.into_iter().cloned()).to_string(),
758 ));
759 clauses.push(format!("NOT {}", within(variable, "?")?));
760 } else {
761 params.push(SqlValue::Text(
762 serde_json::to_string(&tuples).expect("owner tuples serialize"),
763 ));
764 let matches = variables
765 .iter()
766 .enumerate()
767 .map(|(index, variable)| within(variable, &format!("o.value,'$[{index}]'")))
768 .collect::<Result<Vec<_>, ()>>()?;
769 clauses.push(format!(
770 "NOT EXISTS (SELECT 1 FROM json_each(?) o WHERE {})",
771 matches.join(" AND ")
772 ));
773 }
774 }
775 Ok(Some((clauses.join(" AND "), params)))
776}
777
778fn owns_local_row(
780 table: &TableSchema,
781 effective: &[(String, Vec<String>)],
782 owners: &[Vec<(String, Vec<String>)>],
783 values: &Map<String, Value>,
784) -> bool {
785 let matches = |scopes: &[(String, Vec<String>)]| {
786 !scopes.is_empty()
787 && scopes.iter().all(|(variable, allowed)| {
788 table
789 .scope_column(variable)
790 .and_then(|column| values.get(column))
791 .filter(|value| !value.is_null())
792 .is_some_and(|value| {
793 allowed.contains(
794 &value
795 .as_str()
796 .map_or_else(|| value.to_string(), str::to_owned),
797 )
798 })
799 })
800 };
801 matches(effective) && !owners.iter().any(|owner| matches(owner))
802}
803
804fn is_local_operation_code_like(value: &str) -> bool {
805 let bytes = value.as_bytes();
806 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
807 && bytes
808 .iter()
809 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
810}
811
812fn base_table(name: &str) -> String {
813 quote_ident(&format!("_syncular_base_{name}"))
814}
815
816type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
818
819fn window_base_key(base: &WindowBase) -> String {
821 format!(
822 "{}\0{}",
823 base.variable,
824 ssp2::registration::canonical_unit_json(&scope_pairs(&base.fixed_scopes))
825 )
826}
827
828fn window_unit_scopes(base: &WindowBase, value: &str) -> UnitScopes {
830 let mut scopes = base.fixed_scopes.clone();
831 scopes.insert(base.variable.clone(), value.to_owned());
832 scopes
833}
834
835fn scope_pairs(scopes: &UnitScopes) -> Vec<(String, String)> {
837 let mut pairs: Vec<(String, String)> = scopes
838 .iter()
839 .map(|(variable, value)| (variable.clone(), value.clone()))
840 .collect();
841 pairs.sort_by(|a, b| a.0.encode_utf16().cmp(b.0.encode_utf16()));
842 pairs
843}
844
845fn covers(table: &TableSchema, scopes: &UnitScopes) -> bool {
847 scopes
848 .keys()
849 .all(|variable| table.scope_column(variable).is_some())
850}
851
852fn pair_scopes(
856 scopes: &UnitScopes,
857 table: &str,
858 echo: Option<&[EchoRecord]>,
859) -> Option<Vec<(String, Vec<String>)>> {
860 let fenced = echo.and_then(|echo| echo.iter().find(|record| record.table == table));
861 if fenced.is_some_and(|record| record.status == "revoked") {
862 return None;
863 }
864 let mut effective: BTreeMap<String, Vec<String>> = fenced
865 .map(|record| record.scopes.clone())
866 .unwrap_or_default();
867 for (variable, value) in scopes {
868 effective.insert(variable.clone(), vec![value.clone()]);
869 }
870 Some(effective.into_iter().collect())
871}
872
873fn unit_owners(
878 units: &[UnitRecord],
879 table: &TableSchema,
880 except_id: &str,
881) -> Vec<Vec<(String, Vec<String>)>> {
882 units
883 .iter()
884 .filter(|unit| {
885 unit.id != except_id
886 && unit.status == UnitStatus::Active
887 && unit.echo.is_some()
888 && (!unit.bootstrapping() || unit.seeded.contains(&table.name))
889 && covers(table, &unit.scopes)
890 })
891 .filter_map(|unit| pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()))
892 .collect()
893}
894
895fn unit_scope_lists(scopes: &UnitScopes) -> Vec<(String, Vec<String>)> {
897 scopes
898 .iter()
899 .map(|(variable, value)| (variable.clone(), vec![value.clone()]))
900 .collect()
901}
902
903const ACKED_ROWS_BY_ROW: &str = "SELECT a.op_json,a.intent_json FROM json_each(?1) k JOIN _syncular_acked_rows a ON a.tbl=json_extract(k.value,'$[0]') AND a.id=json_extract(k.value,'$[1]') WHERE ?2 IS NULL OR a.tbl IN(SELECT value FROM json_each(?2)) ORDER BY a.commit_seq,a.idx";
907
908const UNIT_COLUMNS: &str =
909 "id, scopes, explicit, status, reason_code, cursor, bootstrap_state, seeded, echo";
910
911fn unit_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result<UnitRecord> {
914 fn json<T: serde::de::DeserializeOwned>(index: usize, raw: &str) -> rusqlite::Result<T> {
915 serde_json::from_str(raw).map_err(|error| {
916 rusqlite::Error::FromSqlConversionFailure(
917 index,
918 rusqlite::types::Type::Text,
919 Box::new(error),
920 )
921 })
922 }
923 Ok(UnitRecord {
924 id: row.get(0)?,
925 scopes: json(1, &row.get::<_, String>(1)?)?,
926 explicit: row.get::<_, i64>(2)? == 1,
927 status: json(
928 3,
929 &serde_json::to_string(&row.get::<_, String>(3)?).expect("text"),
930 )?,
931 reason_code: row.get(4)?,
932 cursor: row.get(5)?,
933 bootstrap_state: row.get(6)?,
934 seeded: json(7, &row.get::<_, String>(7)?)?,
935 echo: row
936 .get::<_, Option<String>>(8)?
937 .map(|echo| json::<Vec<EchoRecord>>(8, &echo))
938 .transpose()?
939 .map(|echo| {
940 if echo
941 .iter()
942 .all(|record| record.status == "active" || record.status == "revoked")
943 {
944 Ok(echo)
945 } else {
946 Err(rusqlite::Error::FromSqlConversionFailure(
947 8,
948 rusqlite::types::Type::Text,
949 "echo status is neither active nor revoked".into(),
950 ))
951 }
952 })
953 .transpose()?,
954 })
955}
956
957fn load_units(conn: &Connection) -> rusqlite::Result<Vec<UnitRecord>> {
959 conn.prepare_cached(&format!(
960 "SELECT {UNIT_COLUMNS} FROM _syncular_units ORDER BY rowid"
961 ))?
962 .query_map([], unit_from_row)?
963 .collect()
964}
965
966fn get_unit(conn: &Connection, id: &str) -> rusqlite::Result<Option<UnitRecord>> {
967 conn.query_row(
968 &format!("SELECT {UNIT_COLUMNS} FROM _syncular_units WHERE id = ?1"),
969 [id],
970 unit_from_row,
971 )
972 .optional()
973}
974
975fn save_unit(conn: &Connection, unit: &UnitRecord) -> rusqlite::Result<()> {
978 conn.prepare_cached(&format!(
979 "INSERT INTO _syncular_units({UNIT_COLUMNS}) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)
980 ON CONFLICT(id) DO UPDATE SET explicit=excluded.explicit, status=excluded.status,
981 reason_code=excluded.reason_code, cursor=excluded.cursor,
982 bootstrap_state=excluded.bootstrap_state, seeded=excluded.seeded, echo=excluded.echo,
983 applied=CASE WHEN cursor IS excluded.cursor
984 AND bootstrap_state IS excluded.bootstrap_state THEN applied END"
985 ))?
986 .execute(rusqlite::params![
987 unit.id,
988 serde_json::to_string(&unit.scopes).expect("scope map serializes"),
989 i64::from(unit.explicit),
990 unit.status.name(),
991 unit.reason_code,
992 unit.cursor,
993 unit.bootstrap_state,
994 serde_json::to_string(&unit.seeded).expect("table list serializes"),
995 unit.echo
996 .as_ref()
997 .map(|echo| serde_json::to_string(echo).expect("echo serializes")),
998 ])
999 .map(|_| ())
1000}
1001
1002fn applied_rows(conn: &Connection, id: &str) -> rusqlite::Result<BTreeMap<String, i64>> {
1005 conn.query_row(
1006 "SELECT applied FROM _syncular_units WHERE id = ?1",
1007 [id],
1008 |row| row.get::<_, Option<String>>(0),
1009 )
1010 .optional()?
1011 .flatten()
1012 .map_or(Ok(BTreeMap::new()), |raw| {
1013 serde_json::from_str(&raw).map_err(|error| {
1014 rusqlite::Error::FromSqlConversionFailure(
1015 0,
1016 rusqlite::types::Type::Text,
1017 Box::new(error),
1018 )
1019 })
1020 })
1021}
1022
1023fn record_applied_rows(
1026 conn: &Connection,
1027 id: &str,
1028 segment_id: &str,
1029 rows: i64,
1030) -> rusqlite::Result<()> {
1031 conn.prepare_cached(
1032 "UPDATE _syncular_units SET applied=json_patch(coalesce(applied,'{}'),?1) WHERE id=?2",
1033 )?
1034 .execute(rusqlite::params![
1035 serde_json::json!({ segment_id: rows }).to_string(),
1036 id
1037 ])
1038 .map(|_| ())
1039}
1040
1041fn window_refs(conn: &Connection, unit_id: &str) -> rusqlite::Result<Vec<(String, String)>> {
1043 conn.prepare_cached("SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1")?
1044 .query_map([unit_id], |row| Ok((row.get(0)?, row.get(1)?)))?
1045 .collect()
1046}
1047
1048fn reset_units(conn: &Connection, keep: &BTreeSet<String>) -> rusqlite::Result<()> {
1054 conn.execute(
1055 "UPDATE _syncular_units SET status='active', reason_code=NULL, cursor=-1,
1056 bootstrap_state=NULL, seeded='[]', echo=NULL, applied=NULL
1057 WHERE id NOT IN (SELECT value FROM json_each(?1))",
1058 [serde_json::to_string(keep).expect("unit ids serialize")],
1059 )?;
1060 conn.execute_batch(
1061 "DELETE FROM _syncular_meta WHERE key IN ('echoDigest','registrationDigest');
1062 DELETE FROM _syncular_meta WHERE key='replicaCursor'
1063 AND NOT EXISTS (SELECT 1 FROM _syncular_units WHERE cursor IS NULL);",
1064 )
1065}
1066
1067fn replica_cursor(conn: &Connection) -> rusqlite::Result<i64> {
1069 meta_get(conn, REPLICA_CURSOR_KEY)?.map_or(Ok(-1), |raw| {
1070 raw.parse().map_err(|error| {
1071 rusqlite::Error::FromSqlConversionFailure(
1072 0,
1073 rusqlite::types::Type::Text,
1074 Box::new(error),
1075 )
1076 })
1077 })
1078}
1079
1080fn visible_table(name: &str) -> String {
1081 quote_ident(name)
1082}
1083
1084const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1085
1086fn is_synced_table_name(name: &str) -> bool {
1089 if name.starts_with("sqlite_") {
1090 return false;
1091 }
1092 if name.starts_with("_syncular_base_") {
1093 return true; }
1095 !name.starts_with("_syncular_")
1097}
1098
1099fn blob_id_for(bytes: &[u8]) -> String {
1101 let digest = Sha256::digest(bytes);
1102 format!("sha256:{}", bytes_to_hex(&digest))
1103}
1104
1105enum RowParam<'a> {
1109 Cell(&'a Option<ColumnValue>),
1110 Version(i64),
1111 Text(Option<&'a str>),
1112}
1113
1114impl rusqlite::ToSql for RowParam<'_> {
1115 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1116 Ok(match self {
1117 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
1118 RowParam::Text(text) => match text {
1119 Some(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())),
1120 None => ToSqlOutput::Owned(SqlValue::Null),
1121 },
1122 RowParam::Cell(cell) => match cell {
1123 None => ToSqlOutput::Owned(SqlValue::Null),
1124 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
1125 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
1126 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
1127 Some(ColumnValue::Boolean(b)) => {
1128 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
1129 }
1130 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
1131 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
1132 }
1133 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
1135 ToSqlOutput::Borrowed(ValueRef::Blob(b))
1136 }
1137 },
1138 })
1139 }
1140}
1141
1142fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
1144 match param.to_sql().map_err(|error| error.to_string())? {
1145 ToSqlOutput::Owned(value) => Ok(value),
1146 ToSqlOutput::Borrowed(value) => {
1147 SqlValue::try_from(value).map_err(|error| error.to_string())
1148 }
1149 _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
1150 }
1151}
1152
1153fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
1161 use ssp2::segment::ColumnType;
1162 let mismatch = || {
1163 Err(format!(
1164 "image column {:?} holds a value of the wrong type",
1165 column.name
1166 ))
1167 };
1168 match value {
1169 ValueRef::Null => {
1170 if !column.nullable {
1171 return Err(format!(
1172 "image column {:?} is NULL but not nullable",
1173 column.name
1174 ));
1175 }
1176 Ok(ToSqlOutput::Owned(SqlValue::Null))
1177 }
1178 ValueRef::Integer(i) => match column.ty {
1179 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
1180 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
1181 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
1182 _ => mismatch(),
1183 },
1184 ValueRef::Real(_) => match column.ty {
1185 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
1186 _ => mismatch(),
1187 },
1188 ValueRef::Text(t) => {
1189 std::str::from_utf8(t)
1190 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
1191 match column.ty {
1192 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
1193 Ok(ToSqlOutput::Borrowed(value))
1194 }
1195 _ => mismatch(),
1196 }
1197 }
1198 ValueRef::Blob(_) => match column.ty {
1199 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
1201 _ => mismatch(),
1202 },
1203 }
1204}
1205
1206pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
1207 use rusqlite::types::ValueRef;
1208 match value {
1209 ValueRef::Null => Value::Null,
1210 ValueRef::Integer(i) => match column.ty {
1211 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
1212 ssp2::segment::ColumnType::Float => {
1213 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
1214 }
1215 _ => Value::from(i),
1216 },
1217 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1218 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1219 ValueRef::Blob(b) => {
1220 let mut map = Map::new();
1221 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1222 Value::Object(map)
1223 }
1224 }
1225}
1226
1227fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
1231 Ok(match value {
1232 Value::Null => SqlValue::Null,
1233 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
1234 Value::Number(n) => {
1235 if let Some(i) = n.as_i64() {
1236 SqlValue::Integer(i)
1237 } else if let Some(f) = n.as_f64() {
1238 SqlValue::Real(f)
1239 } else {
1240 return Err(format!("query param number {n} is out of range"));
1241 }
1242 }
1243 Value::String(s) => SqlValue::Text(s.clone()),
1244 Value::Object(_) => {
1245 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
1246 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
1247 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
1248 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
1249 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
1250 })?)
1251 } else {
1252 return Err(
1253 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
1254 .to_owned(),
1255 );
1256 }
1257 }
1258 Value::Array(_) => return Err("query array params are not supported".to_owned()),
1259 })
1260}
1261
1262fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
1267 use rusqlite::types::ValueRef;
1268 match value {
1269 ValueRef::Null => Value::Null,
1270 ValueRef::Integer(i) => {
1271 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
1275 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
1276 Value::from(i)
1277 } else {
1278 let mut map = Map::new();
1279 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
1280 Value::Object(map)
1281 }
1282 }
1283 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
1284 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
1285 ValueRef::Blob(b) => {
1286 let mut map = Map::new();
1287 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
1288 Value::Object(map)
1289 }
1290 }
1291}
1292
1293fn query_connection(
1295 conn: &Connection,
1296 sql: &str,
1297 params: &[QueryValue],
1298) -> Result<Vec<QueryRow>, QueryReadFailure> {
1299 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
1300 let mut stmt = crate::query_guard::prepare_read_only(conn, &lowered_sql)?;
1301 let bound: Vec<SqlValue> = params
1302 .iter()
1303 .map(json_param_to_sql)
1304 .collect::<Result<_, _>>()?;
1305 let columns: Vec<(usize, String)> = stmt
1310 .column_names()
1311 .enumerate()
1312 .filter(|(_, name)| {
1313 !matches!(
1314 *name,
1315 "_syncular_version" | "_sync_sealed_key" | "_sync_sealed"
1316 )
1317 })
1318 .map(|(i, name)| (i, name.to_owned()))
1319 .collect();
1320 let bound_refs: Vec<&dyn rusqlite::ToSql> =
1321 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
1322 let mut sql_rows = stmt.query(bound_refs.as_slice())?;
1323 let mut out = Vec::new();
1324 while let Some(row) = sql_rows.next()? {
1325 let mut record = Map::new();
1326 for (i, name) in &columns {
1327 let value = row.get_ref(*i)?;
1328 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
1329 }
1330 out.push(record);
1331 }
1332 Ok(out)
1333}
1334
1335fn persisted_window_state(
1336 conn: &Connection,
1337 base: &WindowBase,
1338) -> Result<WindowState, QueryReadFailure> {
1339 let mut stmt = conn.prepare(
1340 "SELECT windows.unit, units.status, units.cursor, units.bootstrap_state
1341 FROM _syncular_windows AS windows
1342 LEFT JOIN _syncular_units AS units ON units.id = windows.sub_id
1343 WHERE windows.base = ?1
1344 ORDER BY windows.unit ASC",
1345 )?;
1346 let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
1347 Ok((
1348 row.get::<_, String>(0)?,
1349 row.get::<_, Option<String>>(1)?,
1350 row.get::<_, Option<i64>>(2)?,
1351 row.get::<_, Option<String>>(3)?,
1352 ))
1353 })?;
1354 let mut units = Vec::new();
1355 let mut pending = Vec::new();
1356 for row in rows {
1357 let (unit, status, cursor, bootstrap_state) = row?;
1358 if status.as_deref() != Some("active")
1360 || cursor.is_some_and(|cursor| cursor < 0 || bootstrap_state.is_some())
1361 {
1362 pending.push(unit.clone());
1363 }
1364 units.push(unit);
1365 }
1366 Ok(WindowState { units, pending })
1367}
1368
1369#[derive(Debug, Clone, Copy)]
1371pub struct SnapshotStatement<'a> {
1372 pub sql: &'a str,
1373 pub params: &'a [Value],
1374}
1375
1376#[derive(Debug, Clone, Default)]
1380pub struct SnapshotReadRequest<'a> {
1381 pub statements: Vec<SnapshotStatement<'a>>,
1382 pub coverage: &'a [WindowCoverage],
1383 pub units: Vec<String>,
1384 pub commit_ids: Vec<String>,
1385}
1386
1387#[cfg(test)]
1388thread_local! {
1389 static SNAPSHOT_READ_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
1393 std::cell::RefCell::new(None);
1394}
1395
1396#[cfg(test)]
1397fn run_snapshot_read_hook() {
1398 SNAPSHOT_READ_HOOK.with(|slot| {
1399 if let Some(hook) = slot.borrow_mut().take() {
1400 hook();
1401 }
1402 });
1403}
1404
1405fn snapshot_read_connection(
1410 conn: &Connection,
1411 request: &SnapshotReadRequest<'_>,
1412) -> Result<SnapshotRead, QueryReadFailure> {
1413 conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
1414 let corrupt = |message: &str| QueryReadFailure {
1415 code: Some("sync.local_corrupt"),
1416 sqlite_code: None,
1417 sqlite_message: None,
1418 rollback_failure: None,
1419 message: message.to_owned(),
1420 };
1421 let stored_failure = |error: rusqlite::Error, message: &str| match error {
1422 rusqlite::Error::Utf8Error(..)
1423 | rusqlite::Error::InvalidColumnType(..)
1424 | rusqlite::Error::FromSqlConversionFailure(..)
1425 | rusqlite::Error::IntegralValueOutOfRange(..) => corrupt(message),
1426 _ => QueryReadFailure::from(error),
1427 };
1428 let result: Result<SnapshotRead, QueryReadFailure> = (|| {
1429 let revision: Option<String> = conn
1432 .query_row(
1433 "SELECT value FROM _syncular_meta WHERE key = ?1",
1434 rusqlite::params![LOCAL_REVISION_KEY],
1435 |row| row.get(0),
1436 )
1437 .optional()
1438 .map_err(|error| stored_failure(error, "persisted local revision is invalid"))?;
1439 let revision = match revision {
1440 None => "0".to_owned(),
1441 Some(value)
1442 if value == "0"
1443 || (matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1444 && value.bytes().all(|b| b.is_ascii_digit())
1445 && value.parse::<u64>().is_ok()) =>
1446 {
1447 value
1448 }
1449 Some(_) => return Err(corrupt("persisted local revision is invalid")),
1450 };
1451 let mut queries = Vec::with_capacity(request.statements.len());
1452 for statement in &request.statements {
1453 queries.push(query_connection(conn, statement.sql, statement.params)?);
1454 #[cfg(test)]
1455 run_snapshot_read_hook();
1456 }
1457 let mut pending = Vec::new();
1458 let mut missing = Vec::new();
1459 for requested in request.coverage {
1460 let base_key = window_base_key(&requested.base);
1461 let state = persisted_window_state(conn, &requested.base)?;
1462 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
1463 let reference = WindowUnitRef {
1464 base_key: base_key.clone(),
1465 unit: unit.clone(),
1466 };
1467 if !state.units.iter().any(|held| held == &unit) {
1468 missing.push(reference);
1469 } else if state.pending.iter().any(|held| held == &unit) {
1470 pending.push(reference);
1471 }
1472 }
1473 }
1474 let mut units = Vec::with_capacity(request.units.len());
1475 for id in &request.units {
1476 let exists: bool = conn.query_row(
1477 "SELECT EXISTS(SELECT 1 FROM _syncular_units WHERE id = ?1)",
1478 [id],
1479 |row| row.get(0),
1480 )?;
1481 if !exists {
1482 units.push(UnitCatchup::Unknown { id: id.clone() });
1483 continue;
1484 }
1485 let unit = get_unit(conn, id)
1488 .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?
1489 .ok_or_else(|| corrupt("persisted unit state is invalid"))?;
1490 let cursor = match unit.cursor {
1491 Some(cursor) => cursor,
1492 None => replica_cursor(conn)
1493 .map_err(|error| stored_failure(error, "persisted unit state is invalid"))?,
1494 };
1495 if cursor.unsigned_abs() > MAX_JS_SAFE_INTEGER {
1496 return Err(corrupt("persisted unit state is invalid"));
1497 }
1498 let active = unit.status == UnitStatus::Active;
1499 let pending = unit.bootstrapping();
1500 units.push(UnitCatchup::Known {
1501 id: id.clone(),
1502 status: unit.status,
1503 live: unit.cursor.is_none(),
1504 cursor,
1505 has_resume_token: unit.bootstrap_state.is_some(),
1506 bootstrap_complete: active && !pending,
1507 known_pending_pages: active && pending,
1508 reason_code: unit.reason_code,
1509 });
1510 }
1511 let mut deliveries = Vec::with_capacity(request.commit_ids.len());
1512 for id in &request.commit_ids {
1513 let pending: bool = conn.query_row(
1514 "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id = ?1)",
1515 rusqlite::params![id],
1516 |row| row.get(0),
1517 )?;
1518 if pending {
1519 deliveries.push(CommitDelivery::Pending {
1520 client_commit_id: id.clone(),
1521 });
1522 continue;
1523 }
1524 let row = conn
1525 .query_row(
1526 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
1527 resolution, resolved_at_ms, replacement_client_commit_id
1528 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
1529 rusqlite::params![id],
1530 StoredCommitOutcomeRow::from_row,
1531 )
1532 .optional()
1533 .map_err(|error| stored_failure(error, "persisted commit outcome is invalid"))?;
1534 deliveries.push(match row {
1535 Some(row) => CommitDelivery::Known {
1536 client_commit_id: id.clone(),
1537 outcome: SyncClient::outcome_from_row(row)
1538 .map_err(|_| corrupt("persisted commit outcome is invalid"))?,
1539 },
1540 None => CommitDelivery::Unknown {
1541 client_commit_id: id.clone(),
1542 },
1543 });
1544 }
1545 Ok(SnapshotRead {
1546 revision,
1547 queries,
1548 coverage: CoverageSnapshot {
1549 complete: pending.is_empty() && missing.is_empty(),
1550 pending,
1551 missing,
1552 },
1553 units,
1554 deliveries,
1555 })
1556 })();
1557 match result {
1558 Ok(read) => {
1559 conn.execute_batch("RELEASE syncular_snapshot_read")?;
1560 Ok(read)
1561 }
1562 Err(mut error) => {
1563 if let Err(rollback) = conn
1564 .execute_batch("ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read")
1565 {
1566 error.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
1567 }
1568 Err(error)
1569 }
1570 }
1571}
1572
1573fn snapshot_connection(
1574 conn: &Connection,
1575 sql: &str,
1576 params: &[Value],
1577 coverage: &[WindowCoverage],
1578) -> Result<QuerySnapshot, QueryReadFailure> {
1579 let read = snapshot_read_connection(
1580 conn,
1581 &SnapshotReadRequest {
1582 statements: vec![SnapshotStatement { sql, params }],
1583 coverage,
1584 units: Vec::new(),
1585 commit_ids: Vec::new(),
1586 },
1587 )?;
1588 Ok(QuerySnapshot {
1589 revision: read.revision,
1590 rows: read
1591 .queries
1592 .into_iter()
1593 .next()
1594 .expect("single-query snapshot returns one result"),
1595 coverage: read.coverage,
1596 })
1597}
1598
1599pub struct FileQuerySnapshotReader {
1604 path: String,
1605 conn: Option<Connection>,
1606}
1607
1608impl FileQuerySnapshotReader {
1609 #[must_use]
1610 pub fn new(path: impl Into<String>) -> Self {
1611 Self {
1612 path: path.into(),
1613 conn: None,
1614 }
1615 }
1616
1617 fn connection(&mut self) -> Result<&Connection, QueryReadFailure> {
1618 if self.conn.is_none() {
1619 let conn = Connection::open_with_flags(
1620 &self.path,
1621 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
1622 )
1623 .map_err(QueryReadFailure::from)?;
1624 conn.busy_timeout(std::time::Duration::from_millis(250))
1625 .map_err(QueryReadFailure::from)?;
1626 self.conn = Some(conn);
1627 }
1628 self.conn
1629 .as_ref()
1630 .ok_or_else(|| QueryReadFailure::from("read sidecar connection missing".to_owned()))
1631 }
1632
1633 pub fn snapshot_read(
1636 &mut self,
1637 request: &SnapshotReadRequest<'_>,
1638 ) -> Result<SnapshotRead, QueryReadFailure> {
1639 snapshot_read_connection(self.connection()?, request)
1640 }
1641
1642 pub fn local_revision(&mut self) -> Result<String, QueryReadFailure> {
1645 Ok(self
1646 .snapshot_read(&SnapshotReadRequest::default())?
1647 .revision)
1648 }
1649
1650 pub fn unit_catchup(&mut self, id: &str) -> Result<UnitCatchup, QueryReadFailure> {
1652 let read = self.snapshot_read(&SnapshotReadRequest {
1653 units: vec![id.to_owned()],
1654 ..Default::default()
1655 })?;
1656 Ok(read
1657 .units
1658 .into_iter()
1659 .next()
1660 .expect("one requested catch-up"))
1661 }
1662
1663 pub fn commit_delivery(&mut self, id: &str) -> Result<CommitDelivery, QueryReadFailure> {
1666 let read = self.snapshot_read(&SnapshotReadRequest {
1667 commit_ids: vec![id.to_owned()],
1668 ..Default::default()
1669 })?;
1670 Ok(read
1671 .deliveries
1672 .into_iter()
1673 .next()
1674 .expect("one requested delivery"))
1675 }
1676
1677 pub fn query_snapshot(
1680 &mut self,
1681 sql: &str,
1682 params: &[Value],
1683 coverage: &[WindowCoverage],
1684 ) -> Result<QuerySnapshot, QueryReadFailure> {
1685 snapshot_connection(self.connection()?, sql, params, coverage)
1686 }
1687}
1688
1689fn validate_authority_reads(
1690 schema: &ClientSchema,
1691 declarations: &[crate::api::AuthorityReadDeclaration],
1692) -> Result<(), String> {
1693 let mut seen = BTreeSet::new();
1694 for read in declarations {
1695 let table = schema
1696 .table(&read.table)
1697 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
1698 let plain = |name: &str| {
1699 table.columns.iter().enumerate().any(|(i, c)| {
1700 c.name == name
1701 && name != "authority_version"
1702 && !name.starts_with("_sync")
1703 && !table.encrypted_columns.iter().any(|e| e.index == i)
1704 && matches!(
1705 c.ty,
1706 ssp2::segment::ColumnType::String
1707 | ssp2::segment::ColumnType::Integer
1708 | ssp2::segment::ColumnType::Float
1709 | ssp2::segment::ColumnType::Boolean
1710 | ssp2::segment::ColumnType::Json
1711 )
1712 })
1713 };
1714 if !seen.insert(&read.table)
1715 || read.columns.is_empty()
1716 || read.columns.iter().collect::<BTreeSet<_>>().len() != read.columns.len()
1717 || !read.columns.contains(&table.primary_key)
1718 || !read.columns.iter().all(|c| plain(c))
1719 || read.scopes.is_empty()
1720 || !read.scopes.iter().all(|(v, values)| {
1721 table
1722 .scope_column(v)
1723 .is_some_and(|c| read.columns.iter().any(|column| column == c) && plain(c))
1724 && !values.is_empty()
1725 && values.iter().all(|v| !v.is_empty() && v != "*")
1726 })
1727 {
1728 return Err("client.authority_read_forbidden: authority reads require plain columns, a primary key and nonempty plain scope selectors".into());
1729 }
1730 }
1731 Ok(())
1732}
1733
1734fn schema_marker_read_failure(error: rusqlite::Error) -> String {
1739 let failure = QueryReadFailure::from(error);
1740 if failure.code.is_some() {
1741 failure.to_string()
1742 } else {
1743 "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned()
1744 }
1745}
1746
1747fn validate_client_limits(limits: &crate::api::ClientLimits) -> Result<(), String> {
1751 if limits.outcome_retention_max_entries == Some(0) {
1752 return Err("sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned());
1753 }
1754 if [
1755 limits.max_push_commits_per_request,
1756 limits.max_push_operations_per_request,
1757 limits.max_push_request_bytes,
1758 ]
1759 .into_iter()
1760 .flatten()
1761 .any(|value| value == 0 || value > u32::MAX as usize)
1762 {
1763 return Err(
1764 "sync.invalid_request: push limit must be an integer in 1..=4294967295".to_owned(),
1765 );
1766 }
1767 Ok(())
1768}
1769
1770fn read_local_schema_version(conn: &Connection, requested: i32) -> Result<Option<i32>, String> {
1771 let corrupt =
1772 || "sync.local_corrupt: persisted local schema marker is unreadable or invalid".to_owned();
1773 let table_type: Option<String> = conn
1774 .query_row(
1775 "SELECT type FROM sqlite_master WHERE name = '_syncular_meta'",
1776 [],
1777 |row| row.get(0),
1778 )
1779 .optional()
1780 .map_err(schema_marker_read_failure)?;
1781 let Some(table_type) = table_type else {
1782 return Ok(None);
1783 };
1784 if table_type != "table" {
1785 return Err(corrupt());
1786 }
1787 let mut statement = conn
1791 .prepare("SELECT value FROM _syncular_meta WHERE key = ?1")
1792 .map_err(schema_marker_read_failure)?;
1793 let mut rows = statement
1794 .query([LOCAL_SCHEMA_VERSION_KEY])
1795 .map_err(schema_marker_read_failure)?;
1796 let value: Option<String> = rows
1797 .next()
1798 .map_err(schema_marker_read_failure)?
1799 .map(|row| row.get(0))
1800 .transpose()
1801 .map_err(schema_marker_read_failure)?;
1802 if rows.next().map_err(schema_marker_read_failure)?.is_some() {
1803 return Err(corrupt());
1804 }
1805 let Some(value) = value else {
1806 let has_descriptor = conn
1807 .query_row(
1808 "SELECT 1 FROM _syncular_meta WHERE key = ?1 LIMIT 1",
1809 [crate::previous_version::LOCAL_SCHEMA_DESCRIPTOR_KEY],
1810 |_| Ok(()),
1811 )
1812 .optional()
1813 .map_err(schema_marker_read_failure)?
1814 .is_some();
1815 if has_descriptor {
1816 return Err(corrupt());
1817 }
1818 return Ok(None);
1819 };
1820 if !matches!(value.as_bytes().first(), Some(b'1'..=b'9'))
1821 || !value.bytes().all(|b| b.is_ascii_digit())
1822 {
1823 return Err(corrupt());
1824 }
1825 let version = value.parse::<i32>().map_err(|_| corrupt())?;
1826 if version > requested {
1827 return Err(
1828 "client.schema_downgrade: persisted local schema is newer than the requested schema"
1829 .to_owned(),
1830 );
1831 }
1832 Ok(Some(version))
1833}
1834
1835#[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
1839fn prepare_wasm_replica(path: &str) -> Result<(), String> {
1840 if path.is_empty() || path == ":memory:" {
1841 return Ok(());
1842 }
1843 crate::wasm_vfs::disable_foreign_lock_check()
1844}
1845
1846#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
1847fn prepare_wasm_replica(_path: &str) -> Result<(), String> {
1848 Ok(())
1849}
1850
1851impl SyncClient {
1852 pub fn new_with_identity(
1853 client_id: Option<String>,
1854 schema_json: &Value,
1855 limits: ClientLimits,
1856 ) -> Result<Self, String> {
1857 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
1858 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1859 Self::with_connection(resolved, schema_json, limits, conn)
1860 }
1861
1862 pub fn new(
1863 client_id: String,
1864 schema_json: &Value,
1865 limits: ClientLimits,
1866 ) -> Result<Self, String> {
1867 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
1868 Self::with_connection(client_id, schema_json, limits, conn)
1869 }
1870
1871 pub fn open_path(
1877 client_id: String,
1878 schema_json: &Value,
1879 limits: ClientLimits,
1880 path: &str,
1881 ) -> Result<Self, String> {
1882 validate_client_limits(&limits)?;
1883 parse_schema_json(schema_json)?;
1884 prepare_wasm_replica(path)?;
1885 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1886 Self::with_connection(client_id, schema_json, limits, conn)
1887 }
1888
1889 pub fn open_path_with_identity(
1893 client_id: Option<String>,
1894 schema_json: &Value,
1895 limits: ClientLimits,
1896 path: &str,
1897 ) -> Result<Self, String> {
1898 Self::open_path_with_identity_journal(
1899 client_id,
1900 schema_json,
1901 limits,
1902 path,
1903 !cfg!(all(target_arch = "wasm32", target_os = "unknown")),
1904 )
1905 }
1906
1907 fn open_path_with_identity_journal(
1908 client_id: Option<String>,
1909 schema_json: &Value,
1910 limits: ClientLimits,
1911 path: &str,
1912 require_wal: bool,
1913 ) -> Result<Self, String> {
1914 validate_client_limits(&limits)?;
1915 parse_schema_json(schema_json)?;
1918 prepare_wasm_replica(path)?;
1919 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
1920 conn.busy_timeout(std::time::Duration::from_millis(250))
1928 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
1929 conn.pragma_update(None, "synchronous", "FULL")
1934 .map_err(|error| format!("configure db {path:?} synchronous: {error}"))?;
1935 let client = Self::with_connection_identity(client_id, schema_json, limits, conn)?;
1936 if require_wal {
1940 let journal_mode: String = client
1941 .conn
1942 .pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))
1943 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
1944 if journal_mode != "wal" {
1945 return Err(
1946 "sync.invalid_request: file-backed client requires WAL journal mode".to_owned(),
1947 );
1948 }
1949 }
1950 Ok(client)
1951 }
1952
1953 pub fn reset_replica(path: &str) -> Result<(), String> {
1967 if path.is_empty() || path == ":memory:" {
1968 return Err("sync.invalid_request: a memory replica has nothing to reset".to_owned());
1969 }
1970 let vfs = unsafe { rusqlite::ffi::sqlite3_vfs_find(std::ptr::null()) };
1974 if vfs.is_null() {
1975 return Err("client.storage_io: no SQLite VFS is registered".to_owned());
1976 }
1977 let delete = unsafe { (*vfs).xDelete }
1978 .ok_or_else(|| "client.storage_io: the SQLite VFS cannot delete files".to_owned())?;
1979 for suffix in ["-wal", "-shm", "-journal", ""] {
1980 let file_name = format!("{path}{suffix}");
1981 let c_name = std::ffi::CString::new(file_name.as_str()).map_err(|_| {
1982 format!("sync.invalid_request: replica path {file_name:?} contains a NUL byte")
1983 })?;
1984 let code = unsafe { delete(vfs, c_name.as_ptr(), 1) };
1987 if code != rusqlite::ffi::SQLITE_OK && code != rusqlite::ffi::SQLITE_IOERR_DELETE_NOENT
1990 {
1991 return Err(format!(
1992 "client.storage_io: deleting replica file {file_name:?} failed with SQLite code {code}"
1993 ));
1994 }
1995 }
1996 Ok(())
1997 }
1998
1999 #[must_use]
2000 pub fn client_id(&self) -> &str {
2001 &self.client_id
2002 }
2003
2004 pub fn reset_client_id(&mut self, transport: &mut dyn Transport) -> Result<String, String> {
2011 self.disconnect_realtime(transport);
2012 let client_id = uuid::Uuid::new_v4().to_string();
2013 self.set_meta(CLIENT_ID_KEY, &client_id)?;
2014 self.client_id.clone_from(&client_id);
2015 self.round_failure = None;
2016 Ok(client_id)
2017 }
2018
2019 pub fn with_connection(
2026 client_id: String,
2027 schema_json: &Value,
2028 limits: ClientLimits,
2029 conn: Connection,
2030 ) -> Result<Self, String> {
2031 Self::with_connection_identity(Some(client_id), schema_json, limits, conn)
2032 }
2033
2034 fn with_connection_identity(
2037 client_id: Option<String>,
2038 schema_json: &Value,
2039 limits: ClientLimits,
2040 conn: Connection,
2041 ) -> Result<Self, String> {
2042 validate_client_limits(&limits)?;
2043 let schema = parse_schema_json(schema_json)?;
2044 validate_authority_reads(&schema, &limits.authority_reads)?;
2045 let previous_version = match limits.previous_version_context {
2048 Some(config) => {
2049 config.validate()?;
2050 Some(config)
2051 }
2052 None => None,
2053 };
2054 let mut client = SyncClient {
2055 #[cfg(feature = "bench-internals")]
2056 benchmark_phases: Recorder::default(),
2057 conn,
2058 schema,
2059 client_id: client_id
2060 .clone()
2061 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
2062 limits,
2063 outbox: Shared::default(),
2064 failed_commits: Shared::default(),
2065 retain_failed_commits: false,
2066 conflicts: Shared::default(),
2067 rejections: Shared::default(),
2068 schema_floor: None,
2069 lease_state: None,
2070 stopped: false,
2071 upgrading: false,
2072 sync_needed: false,
2073 realtime_policy: RealtimePolicy::Optional,
2074 realtime_state: RealtimeState::Disconnected,
2075 realtime_reason_code: None,
2076 realtime_retry_delay_ms: None,
2077 presence: HashMap::new(),
2078 now_ms: None,
2079 encryption: crate::values::EncryptionConfig::default(),
2080 security_preflight: false,
2081 security_epoch: 0,
2082 transport_enabled: true,
2083 transport_epoch: 0,
2084 registration_epoch: 0,
2085 previous_version,
2086 insert_sql: RefCell::new(HashMap::new()),
2087 overlay_dirty: OverlayDirty::clean(),
2088 storage_failure: RefCell::new(None),
2089 #[cfg(test)]
2090 overlay_rebuild_count: Cell::new(0),
2091 #[cfg(test)]
2092 acknowledged_replay_count: Cell::new(0),
2093 #[cfg(test)]
2094 outcome_prune_count: Cell::new(0),
2095 #[cfg(test)]
2096 image_chunk_ms: Cell::new(None),
2097 progress: ProgressObserver::default(),
2098 change_queue: VecDeque::new(),
2099 sync_intent_queue: VecDeque::new(),
2100 retry_delay_ms: 250,
2101 round_retry_delay_ms: None,
2102 image_chunk_rows: HashMap::new(),
2103 round_failure: None,
2104 active_round: None,
2105 last_round: None,
2106 last_change: None,
2107 query_failures: Vec::new(),
2108 coalesced: None,
2109 coalesced_failure: None,
2110 };
2111 client
2115 .conn
2116 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2117 client
2123 .conn
2124 .execute_batch("BEGIN IMMEDIATE")
2125 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2126 let startup = (|| -> Result<(), String> {
2127 let marker = read_local_schema_version(&client.conn, client.schema.version)?;
2130 client.create_bookkeeping_tables()?;
2133 match client.get_meta(CLIENT_ID_KEY)? {
2134 Some(existing) => match client_id {
2135 Some(requested) if existing != requested => {
2136 return Err(format!(
2137 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2138 ))
2139 }
2140 _ => client.client_id = existing,
2141 },
2142 None => client.set_meta(CLIENT_ID_KEY, &client.client_id)?,
2143 }
2144 client.restore_persisted_state()?;
2145 client.failed_commits = client.load_failed_commits()?.into();
2146 match marker {
2147 None => {
2148 client.create_synced_tables()?;
2149 client.persist_schema_version()?;
2150 client.save_unit_scope_schema()?;
2151 }
2152 Some(version) if version == client.schema.version => {
2153 client.create_synced_tables()?;
2154 set_local_schema_descriptor(&client.conn, &client.schema)
2158 .map_err(|error| Self::sqlite_failure(&client.storage_failure, error))?;
2159 client.save_unit_scope_schema()?;
2160 }
2161 Some(_) => client.run_schema_reset()?,
2162 }
2163 client.reconcile_previous_version_at_boot()?;
2169 client.clear_satisfied_persisted_schema_floor()?;
2170 client.prune_unknown_units(false)?;
2171 client.drop_unencodable_outbox(false, None)?;
2180 if marker == Some(client.schema.version)
2181 && (!client.outbox.is_empty()
2182 || !client.failed_commits.is_empty()
2183 || client.has_acknowledged_rows()?)
2184 {
2185 client.overlay_dirty.set(true);
2189 client.rebuild_overlay()?;
2190 }
2191 client.enqueue_startup_sync_if_needed()?;
2198 Ok(())
2199 })();
2200 match startup {
2201 Ok(()) => match client.conn.execute_batch("COMMIT") {
2202 Ok(()) => {
2206 if !client.security_preflight {
2207 client.drain_pending_evictions()?;
2208 }
2209 Ok(client)
2210 }
2211 Err(error) => {
2212 let failure = Self::sqlite_failure(&client.storage_failure, error);
2213 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2214 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2216 }
2217 Err(failure)
2218 }
2219 },
2220 Err(error) => {
2221 if let Err(rollback) = client.conn.execute_batch("ROLLBACK") {
2222 let _ = Self::sqlite_failure(&client.storage_failure, rollback);
2224 }
2225 Err(error)
2226 }
2227 }
2228 }
2229
2230 pub fn set_now_ms(&mut self, now_ms: i64) {
2233 self.now_ms = Some(std::sync::Arc::new(now_ms.into()));
2234 }
2235
2236 pub fn pinned_clock(&self) -> Option<std::sync::Arc<std::sync::atomic::AtomicI64>> {
2239 self.now_ms.clone()
2240 }
2241
2242 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2246 self.encryption = encryption;
2247 }
2248
2249 #[must_use]
2250 pub fn security_lifecycle(&self) -> &'static str {
2251 if self.security_preflight {
2252 "preflight"
2253 } else {
2254 "active"
2255 }
2256 }
2257
2258 #[must_use]
2259 pub fn security_preflight(&self) -> bool {
2260 self.security_preflight
2261 }
2262
2263 pub fn set_transport_enabled(&mut self, transport: &mut dyn Transport, enabled: bool) {
2265 if self.transport_enabled != enabled {
2266 self.transport_enabled = enabled;
2267 self.transport_epoch = self.transport_epoch.wrapping_add(1);
2268 self.sync_intent_queue.clear();
2269 self.sync_intent_queue
2270 .push_back(if enabled && !self.security_preflight {
2271 SyncIntent::Interactive
2272 } else {
2273 SyncIntent::None
2274 });
2275 }
2276 if !enabled && self.active_round.is_none() {
2278 self.disconnect_realtime(transport);
2279 }
2280 }
2281
2282 pub fn transport_enabled(&self) -> bool {
2283 self.transport_enabled
2284 }
2285
2286 pub fn begin_security_preflight(&mut self) -> Result<(), String> {
2293 let entering = !self.security_preflight;
2294 self.seal_security_on_teardown();
2295 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1")?;
2298 if entering {
2299 self.emit_status_batch()?;
2300 }
2301 Ok(())
2302 }
2303
2304 fn emit_status_batch(&mut self) -> Result<(), String> {
2306 self.begin_observation("syncular_status")?;
2307 let batch = ChangeAccumulator {
2308 status: true,
2309 ..ChangeAccumulator::default()
2310 };
2311 self.finish_observation("syncular_status", batch)
2312 .inspect_err(|_| self.rollback_observation("syncular_status"))
2313 }
2314
2315 pub fn seal_security_on_teardown(&mut self) {
2325 self.cancel_sync_round();
2326 self.security_preflight = true;
2327 self.security_epoch = self.security_epoch.wrapping_add(1);
2328 self.encryption = crate::values::EncryptionConfig::default();
2329 self.sync_intent_queue.clear();
2330 }
2331
2332 pub fn activate_security(
2334 &mut self,
2335 encryption: crate::values::EncryptionConfig,
2336 ) -> Result<(), String> {
2337 if !self.security_preflight {
2338 return Err(
2339 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2340 );
2341 }
2342 self.drain_pending_evictions()?;
2344 self.enqueue_startup_sync_if_needed()?;
2345 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY)?;
2346 self.encryption = encryption;
2347 self.security_preflight = false;
2348 self.reveal_sealed_rows()
2350 .map_err(|error| format!("{}: {}", error.code, error.message))?;
2351 self.emit_status_batch()
2352 }
2353
2354 fn clock_now_ms(&self) -> i64 {
2355 self.now_ms
2356 .as_ref()
2357 .map_or_else(crate::wall_clock_ms, |clock| {
2358 clock.load(std::sync::atomic::Ordering::Relaxed)
2359 })
2360 }
2361
2362 fn create_bookkeeping_tables(&self) -> Result<(), String> {
2363 let exists = |name: &str| {
2364 self.conn
2365 .query_row(
2366 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1)",
2367 [name],
2368 |row| row.get::<_, bool>(0),
2369 )
2370 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2371 };
2372 let outbox_existed = exists("_syncular_outbox")?;
2373 self.conn
2375 .execute_batch(
2376 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2377 seq INTEGER PRIMARY KEY AUTOINCREMENT,
2378 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2379 CREATE TABLE IF NOT EXISTS _syncular_row_deliveries (
2380 tbl TEXT NOT NULL, id TEXT NOT NULL, commit_seq INTEGER NOT NULL, PRIMARY KEY(tbl,id));
2381 CREATE TABLE IF NOT EXISTS _syncular_acked_rows (
2382 commit_id TEXT NOT NULL, idx INTEGER NOT NULL, tbl TEXT NOT NULL, id TEXT NOT NULL,
2383 commit_seq INTEGER NOT NULL, op_json TEXT NOT NULL, intent_json TEXT,
2384 PRIMARY KEY(commit_id, idx));
2385 CREATE INDEX IF NOT EXISTS _syncular_acked_rows_row
2386 ON _syncular_acked_rows(tbl, id);
2387 CREATE TABLE IF NOT EXISTS _syncular_failed_commits (
2388 seq INTEGER PRIMARY KEY AUTOINCREMENT, client_commit_id TEXT NOT NULL UNIQUE,
2389 operations_json TEXT NOT NULL, initial_json TEXT NOT NULL);
2390 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2391 seq INTEGER PRIMARY KEY AUTOINCREMENT,
2392 client_commit_id TEXT NOT NULL UNIQUE,
2393 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2394 recorded_at_ms INTEGER NOT NULL,
2395 results_json TEXT NOT NULL,
2396 operations_json TEXT,
2397 resolution TEXT NOT NULL DEFAULT 'active'
2398 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2399 resolved_at_ms INTEGER,
2400 replacement_client_commit_id TEXT);
2401 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2402 ON _syncular_commit_outcomes(resolution, seq);
2403 CREATE TABLE IF NOT EXISTS _syncular_units (
2404 id TEXT PRIMARY KEY, scopes TEXT NOT NULL,
2405 explicit INTEGER NOT NULL DEFAULT 0,
2406 status TEXT NOT NULL DEFAULT 'active', reason_code TEXT,
2407 cursor INTEGER DEFAULT -1, bootstrap_state TEXT,
2408 seeded TEXT NOT NULL DEFAULT '[]', echo TEXT, applied TEXT);
2409 CREATE TABLE IF NOT EXISTS _syncular_meta (
2410 key TEXT PRIMARY KEY, value TEXT NOT NULL);
2411 CREATE TABLE IF NOT EXISTS _syncular_windows (
2412 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2413 PRIMARY KEY (base, unit));
2414 CREATE INDEX IF NOT EXISTS _syncular_windows_unit ON _syncular_windows(sub_id);
2415 CREATE TABLE IF NOT EXISTS _syncular_pending_evictions (
2416 unit_id TEXT NOT NULL, tbl TEXT NOT NULL,
2417 effective_scopes TEXT NOT NULL, PRIMARY KEY (unit_id, tbl));
2418 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
2419 commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
2420 PRIMARY KEY(commit_id, blob_id));
2421 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
2422 ON _syncular_blob_commit_refs(blob_id);",
2423 )
2424 .map_err(|e| e.to_string())?;
2425 if !self
2427 .conn
2428 .prepare("SELECT 1 FROM pragma_table_info('_syncular_units') WHERE name = 'applied'")
2429 .and_then(|mut stmt| stmt.exists([]))
2430 .map_err(|e| e.to_string())?
2431 {
2432 self.conn
2433 .execute_batch("ALTER TABLE _syncular_units ADD COLUMN applied TEXT")
2434 .map_err(|e| e.to_string())?;
2435 }
2436 let _ = self
2439 .conn
2440 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2441 if self.get_meta(LOCAL_REVISION_KEY)?.is_none() {
2442 self.set_meta(LOCAL_REVISION_KEY, "0")?;
2443 }
2444 if !outbox_existed || self.get_meta(OUTBOX_ID_KEY)?.is_none() {
2447 self.set_meta(OUTBOX_ID_KEY, &uuid::Uuid::new_v4().to_string())?;
2448 }
2449 if exists("_syncular_window_pending_evict")? {
2454 let invalid: bool = self
2455 .conn
2456 .query_row(
2457 "SELECT EXISTS(SELECT 1 FROM _syncular_window_pending_evict
2458 WHERE NOT json_valid(effective_scopes)
2459 OR json_type(effective_scopes) <> 'object'
2460 OR EXISTS(SELECT 1 FROM json_each(effective_scopes)
2461 WHERE type <> 'array'
2462 OR EXISTS(SELECT 1 FROM json_each(value) AS v
2463 WHERE v.type <> 'text')))",
2464 [],
2465 |row| row.get(0),
2466 )
2467 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2468 if invalid {
2469 return Err(
2470 "sync.local_corrupt: a version 3 pending eviction cannot be upgraded"
2471 .to_owned(),
2472 );
2473 }
2474 self.conn
2475 .execute_batch(
2476 "INSERT OR REPLACE INTO _syncular_pending_evictions
2477 SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict;
2478 DROP TABLE _syncular_window_pending_evict;
2479 DROP TABLE IF EXISTS _syncular_subscriptions;
2480 DELETE FROM _syncular_windows;",
2481 )
2482 .and_then(|()| reset_units(&self.conn, &BTreeSet::new()))
2483 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2484 }
2485 if self.schema_has_blobs() {
2488 if exists("_syncular_blobs")? {
2489 let mut statement = self
2490 .conn
2491 .prepare("PRAGMA table_info(_syncular_blobs)")
2492 .map_err(|e| e.to_string())?;
2493 let mut columns = statement
2494 .query_map([], |row| row.get::<_, String>(1))
2495 .map_err(|e| e.to_string())?
2496 .collect::<Result<Vec<_>, _>>()
2497 .map_err(|e| e.to_string())?;
2498 columns.sort();
2499 let mut expected = vec![
2500 "blob_id",
2501 "byte_length",
2502 "bytes",
2503 "created_at_ms",
2504 "media_type",
2505 ];
2506 expected.sort_unstable();
2507 if columns != expected {
2508 return Err(
2509 "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
2510 );
2511 }
2512 } else {
2513 self.conn
2514 .execute_batch(
2515 "CREATE TABLE _syncular_blobs(
2516 blob_id TEXT PRIMARY KEY,
2517 bytes BLOB NOT NULL,
2518 byte_length INTEGER NOT NULL,
2519 media_type TEXT,
2520 created_at_ms INTEGER NOT NULL);",
2521 )
2522 .map_err(|e| e.to_string())?;
2523 }
2524 self.conn
2525 .execute_batch(
2526 "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
2527 blob_id TEXT PRIMARY KEY,
2528 media_type TEXT,
2529 created_at_ms INTEGER NOT NULL);",
2530 )
2531 .map_err(|e| e.to_string())?;
2532 }
2533 Ok(())
2534 }
2535
2536 fn schema_has_blobs(&self) -> bool {
2538 self.schema
2539 .tables
2540 .iter()
2541 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2542 }
2543
2544 fn get_meta(&self, key: &str) -> Result<Option<String>, String> {
2547 meta_get(&self.conn, key)
2548 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2549 }
2550
2551 fn persist_schema_version(&self) -> Result<(), String> {
2554 self.conn
2555 .execute(
2556 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
2557 rusqlite::params![LOCAL_SCHEMA_VERSION_KEY, self.schema.version.to_string()],
2558 )
2559 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2560 set_local_schema_descriptor(&self.conn, &self.schema)
2561 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2562 }
2563
2564 pub(crate) fn set_meta(&self, key: &str, value: &str) -> Result<(), String> {
2565 meta_set(&self.conn, key, value)
2566 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2567 }
2568
2569 fn delete_meta(&self, key: &str) -> Result<(), String> {
2570 meta_delete(&self.conn, key)
2571 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2572 }
2573
2574 fn load_outbox(&self) -> Result<Vec<Arc<OutboxCommit>>, String> {
2576 let mut stmt = self
2577 .conn
2578 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2579 .map_err(|error| error.to_string())?;
2580 let rows = stmt
2581 .query_map([], |row| {
2582 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2583 })
2584 .map_err(|error| error.to_string())?;
2585 let mut commits = Vec::new();
2586 for row in rows {
2587 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2588 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2589 format!("invalid persisted outbox {client_commit_id:?}: {error}")
2590 })?;
2591 let mut ops = Vec::with_capacity(entries.len());
2592 for entry in entries {
2593 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2594 ops.push(OutboxOp {
2595 upsert: op == "upsert",
2596 table: entry
2597 .get("table")
2598 .and_then(Value::as_str)
2599 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2600 .to_owned(),
2601 row_id: entry
2602 .get("rowId")
2603 .and_then(Value::as_str)
2604 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2605 .to_owned(),
2606 base_version: entry.get("baseVersion").and_then(Value::as_i64),
2607 values: entry.get("values").and_then(Value::as_object).cloned(),
2610 insert: entry.get("insert") == Some(&Value::Bool(true)),
2611 });
2612 }
2613 commits.push(Arc::new(OutboxCommit {
2614 client_commit_id,
2615 ops,
2616 }));
2617 }
2618 Ok(commits)
2619 }
2620
2621 fn restore_persisted_state(&mut self) -> Result<(), String> {
2622 self.outbox = self.load_outbox()?.into();
2623
2624 self.prune_commit_outcomes()?;
2625 let active = self.commit_outcomes(CommitOutcomeQuery {
2626 active_only: true,
2627 ..CommitOutcomeQuery::default()
2628 })?;
2629 self.conflicts = active
2630 .iter()
2631 .flat_map(|outcome| outcome.results.iter())
2632 .filter_map(|result| match result {
2633 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2634 _ => None,
2635 })
2636 .collect::<Vec<_>>()
2637 .into();
2638 self.rejections = active
2639 .iter()
2640 .flat_map(|outcome| outcome.results.iter())
2641 .filter_map(|result| match result {
2642 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2643 _ => None,
2644 })
2645 .collect::<Vec<_>>()
2646 .into();
2647
2648 self.lease_state = self
2649 .get_meta(LEASE_STATE_KEY)?
2650 .map(|raw| serde_json::from_str(&raw))
2651 .transpose()
2652 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2653 self.schema_floor = self
2654 .get_meta(SCHEMA_FLOOR_KEY)?
2655 .map(|raw| serde_json::from_str(&raw))
2656 .transpose()
2657 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2658 self.stopped = self.schema_floor.is_some();
2659 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY)?.as_deref() == Some("1") {
2662 self.security_preflight = true;
2663 }
2664 Ok(())
2665 }
2666
2667 fn save_unit_scope_schema(&self) -> Result<(), String> {
2668 let declarations: BTreeMap<_, BTreeMap<_, _>> = self
2669 .schema
2670 .tables
2671 .iter()
2672 .map(|table| {
2673 (
2674 table.name.clone(),
2675 table
2676 .scope_variables
2677 .iter()
2678 .map(|scope| {
2679 (
2680 scope.variable.clone(),
2681 [scope.prefix.clone(), scope.column.clone()],
2682 )
2683 })
2684 .collect(),
2685 )
2686 })
2687 .collect();
2688 self.set_meta(
2689 "subscriptionScopeSchema",
2690 &serde_json::to_string(&declarations).expect("scope schema serialization"),
2691 )?;
2692 let generations: BTreeMap<_, _> = self
2693 .schema
2694 .tables
2695 .iter()
2696 .filter_map(|table| Some((&table.name, table.generation.as_ref()?)))
2697 .collect();
2698 self.set_meta(
2699 TABLE_GENERATIONS_KEY,
2700 &serde_json::to_string(&generations).expect("generations serialize"),
2701 )
2702 }
2703
2704 fn prune_unknown_units(&mut self, bump: bool) -> Result<(), String> {
2709 let previous: Option<BTreeMap<String, BTreeMap<String, [String; 2]>>> = if bump {
2710 self.get_meta("subscriptionScopeSchema")?
2711 .map(|raw| serde_json::from_str(&raw))
2712 .transpose()
2713 .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?
2714 } else {
2715 None
2716 };
2717 let unchanged = |name: &str, variables: &[&String]| {
2722 let (Some(old), Some(table)) = (
2723 previous.as_ref().and_then(|previous| previous.get(name)),
2724 self.schema.table(name),
2725 ) else {
2726 return false;
2727 };
2728 variables.iter().all(|variable| {
2729 table
2730 .scope_variables
2731 .iter()
2732 .find(|scope| &scope.variable == *variable)
2733 .is_some_and(|scope| {
2734 old.get(*variable) == Some(&[scope.prefix.clone(), scope.column.clone()])
2735 })
2736 })
2737 };
2738 let compatible = |scopes: &UnitScopes| {
2739 if !self.schema.tables.iter().any(|table| covers(table, scopes)) {
2740 return false;
2741 }
2742 if !bump {
2743 return true;
2744 }
2745 let Some(previous) = &previous else {
2746 return false;
2747 };
2748 previous.iter().all(|(name, old)| {
2749 !scopes.keys().all(|variable| old.contains_key(variable))
2750 || unchanged(name, &scopes.keys().collect::<Vec<_>>())
2751 })
2752 };
2753 let mut stale: BTreeSet<String> = self
2754 .stored(load_units(&self.conn))?
2755 .into_iter()
2756 .filter(|unit| !compatible(&unit.scopes))
2757 .map(|unit| unit.id)
2758 .collect();
2759 for (id, table, effective) in self.load_pending_evictions()? {
2764 if self.schema.table(&table).is_none()
2765 || (bump
2766 && !unchanged(
2767 &table,
2768 &effective
2769 .iter()
2770 .map(|(variable, _)| variable)
2771 .collect::<Vec<_>>(),
2772 ))
2773 {
2774 stale.insert(id);
2775 }
2776 }
2777 for id in &stale {
2778 for sql in [
2779 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2780 "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
2781 "DELETE FROM _syncular_units WHERE id = ?1",
2782 ] {
2783 self.conn
2784 .execute(sql, rusqlite::params![id])
2785 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
2786 }
2787 }
2788 Ok(())
2789 }
2790
2791 pub fn local_revision(&self) -> Result<u64, String> {
2792 Ok(self
2793 .get_meta(LOCAL_REVISION_KEY)?
2794 .and_then(|value| value.parse().ok())
2795 .unwrap_or(0))
2796 }
2797
2798 pub fn snapshot_read(
2804 &mut self,
2805 request: &SnapshotReadRequest<'_>,
2806 owner: Option<QueryOwner<'_>>,
2807 ) -> Result<SnapshotRead, QueryReadFailure> {
2808 let result = snapshot_read_connection(&self.conn, request);
2809 if let Some(owner) = owner {
2810 self.record_query_read(owner, result.as_ref().err());
2811 }
2812 result
2813 }
2814
2815 #[must_use]
2816 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2817 SyncStatusSnapshot {
2818 current_schema_version: self.schema.version,
2819 outbox: self.outbox.len(),
2820 upgrading: self.upgrading,
2821 lease_state: self.lease_state.clone(),
2822 schema_floor: self.schema_floor.clone(),
2823 sync_needed: self.sync_needed,
2824 previous_version_context: self.previous_version_status(),
2825 security_lifecycle: self.security_lifecycle(),
2826 }
2827 }
2828
2829 fn previous_version_status(&self) -> PreviousVersionStatus {
2832 let Some(replica_path) = self.previous_version_replica_path() else {
2833 return PreviousVersionStatus {
2834 present: false,
2835 created_at_ms: None,
2836 };
2837 };
2838 match crate::previous_version::read_previous_version_record(&replica_path) {
2839 Ok(Some(record)) => PreviousVersionStatus {
2840 present: true,
2841 created_at_ms: Some(record.created_at_ms),
2842 },
2843 _ => PreviousVersionStatus {
2844 present: std::path::Path::new(
2845 &crate::previous_version::previous_version_container_path(&replica_path),
2846 )
2847 .exists(),
2848 created_at_ms: None,
2849 },
2850 }
2851 }
2852
2853 pub fn previous_version_snapshot(
2859 &self,
2860 spec: &PreviousVersionReadSpec,
2861 ) -> Result<PreviousVersionSnapshot, String> {
2862 let now_ms = self.clock_now_ms();
2863 crate::previous_version::previous_version_snapshot(
2864 &self.conn,
2865 self.previous_version_replica_path().as_deref(),
2866 self.previous_version.as_ref(),
2867 PreviousVersionLifecycle {
2868 lease_inactive: self.previous_version_lease_inactive(now_ms),
2869 scope_revoked: self
2870 .stored(load_units(&self.conn))?
2871 .iter()
2872 .any(|unit| unit.status == UnitStatus::Revoked),
2873 coverage_complete: self.previous_version_coverage_complete()?,
2874 },
2875 now_ms,
2876 self.schema.version,
2877 spec,
2878 )
2879 }
2880
2881 pub fn previous_version_audit(
2883 &self,
2884 ) -> Result<Option<crate::previous_version::PreviousVersionAudit>, String> {
2885 crate::previous_version::stored_previous_version_audit(&self.conn)
2886 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
2887 }
2888
2889 pub fn previous_version_discard(
2893 &self,
2894 ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2895 self.drop_previous_version()
2896 }
2897
2898 fn drop_previous_version(
2899 &self,
2900 ) -> Result<crate::previous_version::PreviousVersionDiscardOutcome, String> {
2901 let Some(replica_path) = self.previous_version_replica_path() else {
2902 return Ok(crate::previous_version::PreviousVersionDiscardOutcome {
2903 present: false,
2904 discarded: false,
2905 });
2906 };
2907 crate::previous_version::discard_previous_version(&self.conn, &replica_path)
2908 }
2909
2910 fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
2912 let Some(lease) = &self.lease_state else {
2913 return false;
2914 };
2915 if lease.error_code.is_some() {
2916 return true;
2917 }
2918 lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
2919 }
2920
2921 fn previous_version_coverage_complete(&self) -> Result<bool, String> {
2925 let units = self.stored(load_units(&self.conn))?;
2926 let mut active = units
2927 .iter()
2928 .filter(|unit| unit.status == UnitStatus::Active)
2929 .peekable();
2930 Ok(active.peek().is_some() && !active.any(UnitRecord::bootstrapping))
2931 }
2932
2933 fn previous_version_lifetime_check(&self) {
2937 let Some(config) = self.previous_version.filter(|config| config.enabled) else {
2938 return;
2939 };
2940 let now_ms = self.clock_now_ms();
2941 if self.previous_version_lease_inactive(now_ms) {
2942 let _ = self.drop_previous_version();
2943 return;
2944 }
2945 let Some(replica_path) = self.previous_version_replica_path() else {
2946 return;
2947 };
2948 let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
2949 else {
2950 return;
2951 };
2952 let Ok(units) = load_units(&self.conn) else {
2953 return;
2954 };
2955 if units.iter().any(|unit| unit.status == UnitStatus::Revoked) {
2956 let _ = self.drop_previous_version();
2957 return;
2958 }
2959 if now_ms - record.created_at_ms > config.max_age_ms {
2960 let _ = self.drop_previous_version();
2961 return;
2962 }
2963 if self.previous_version_coverage_complete() == Ok(true) {
2964 let _ = self.drop_previous_version();
2965 }
2966 }
2967
2968 pub fn diagnostics_snapshot(&self) -> Result<ClientDiagnosticsSnapshot, String> {
2969 let registered = self.stored(load_units(&self.conn))?;
2970 let (local_revision, replica_cursor): (Option<String>, Option<String>) = self.stored(
2973 self.conn
2974 .prepare_cached(
2975 "SELECT (SELECT value FROM _syncular_meta WHERE key = ?1),
2976 (SELECT value FROM _syncular_meta WHERE key = ?2)",
2977 )
2978 .and_then(|mut statement| {
2979 statement.query_row([LOCAL_REVISION_KEY, REPLICA_CURSOR_KEY], |row| {
2980 Ok((row.get(0)?, row.get(1)?))
2981 })
2982 }),
2983 )?;
2984 let mut units = DiagnosticUnits::default();
2985 for unit in ®istered {
2986 *match unit.status {
2987 UnitStatus::Revoked => &mut units.revoked,
2988 UnitStatus::Failed => &mut units.failed,
2989 UnitStatus::Active if !unit.bootstrapping() => &mut units.complete,
2990 UnitStatus::Active
2991 if unit.reason_code.as_deref() == Some("sync.cursor_expired") =>
2992 {
2993 &mut units.reset
2994 }
2995 UnitStatus::Active => &mut units.bootstrapping,
2996 } += 1;
2997 }
2998 units.reason_codes = registered
2999 .iter()
3000 .filter_map(|unit| unit.reason_code.as_deref().map(Self::diagnostic_code))
3001 .collect::<BTreeSet<_>>()
3002 .into_iter()
3003 .collect();
3004 let captured_at_ms = self.clock_now_ms();
3005 let lease = if let Some(error_code) = self
3006 .lease_state
3007 .as_ref()
3008 .and_then(|state| state.error_code.clone())
3009 {
3010 ClientDiagnosticsLease {
3011 state: "stopped".to_owned(),
3012 expires_at_ms: self
3013 .lease_state
3014 .as_ref()
3015 .and_then(|state| state.expires_at_ms),
3016 error_code: Some(Self::diagnostic_code(&error_code)),
3017 }
3018 } else if let Some(expires_at_ms) = self
3019 .lease_state
3020 .as_ref()
3021 .and_then(|state| state.expires_at_ms)
3022 {
3023 ClientDiagnosticsLease {
3024 state: if expires_at_ms <= captured_at_ms {
3025 "expired".to_owned()
3026 } else {
3027 "active".to_owned()
3028 },
3029 expires_at_ms: Some(expires_at_ms),
3030 error_code: None,
3031 }
3032 } else {
3033 ClientDiagnosticsLease {
3034 state: "none".to_owned(),
3035 expires_at_ms: None,
3036 error_code: None,
3037 }
3038 };
3039 let connectivity = match self.last_round.as_ref() {
3040 _ if !self.transport_enabled && self.active_round.is_none() => "offline",
3041 Some(round) if round.status == "succeeded" => "online",
3042 Some(round)
3043 if round.status == "failed"
3044 && matches!(
3045 round.cause_kind,
3046 Some(
3047 crate::TransportCauseKind::Network | crate::TransportCauseKind::Connect
3048 )
3049 ) =>
3050 {
3051 "offline"
3052 }
3053 Some(round) if round.status == "failed" && round.http_status.is_some() => "online",
3054 _ => "unknown",
3055 };
3056 Ok(ClientDiagnosticsSnapshot {
3057 version: CLIENT_DIAGNOSTICS_VERSION,
3058 captured_at_ms,
3059 host: ClientDiagnosticsHost {
3060 kind: "direct".to_owned(),
3061 role: "single".to_owned(),
3062 connectivity: connectivity.to_owned(),
3063 realtime: self.realtime_state,
3064 realtime_policy: self.realtime_policy,
3065 realtime_reason_code: self.realtime_reason_code.clone(),
3066 realtime_retry_delay_ms: self.realtime_retry_delay_ms,
3067 },
3068 security_lifecycle: self.security_lifecycle().to_owned(),
3069 schema: ClientDiagnosticsSchema {
3070 current_version: self.schema.version,
3071 upgrading: self.upgrading,
3072 required_version: self
3073 .schema_floor
3074 .as_ref()
3075 .and_then(|floor| floor.required_schema_version),
3076 latest_version: self
3077 .schema_floor
3078 .as_ref()
3079 .and_then(|floor| floor.latest_schema_version),
3080 },
3081 replica: ClientDiagnosticsReplica {
3082 local_revision: local_revision
3083 .and_then(|value| value.parse::<u64>().ok())
3084 .unwrap_or(0)
3085 .to_string(),
3086 sync_needed: self.sync_needed,
3087 pending_outbox: self.outbox.len(),
3088 },
3089 lease,
3090 active_units: registered
3091 .iter()
3092 .filter(|unit| unit.status == UnitStatus::Active)
3093 .count(),
3094 units,
3095 replica_cursor: replica_cursor.map_or(Ok(-1), |raw| {
3096 raw.parse()
3097 .map_err(|_| "sync.local_corrupt: persisted unit state is invalid".to_owned())
3098 })?,
3099 last_round: self.last_round.clone(),
3100 last_change: self.last_change.clone(),
3101 storage: self.diagnostics_storage(),
3102 query_failures: self.query_failures.clone(),
3103 })
3104 }
3105
3106 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3107 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3108 let page_count: i64 = self
3109 .conn
3110 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3111 let page_size: i64 = self
3112 .conn
3113 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3114 let (outbox_bytes, outcome_entries, outcome_bytes): (i64, i64, i64) = self
3117 .conn
3118 .prepare_cached(
3119 "SELECT (SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox),
3120 COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0)
3121 FROM _syncular_commit_outcomes",
3122 )?
3123 .query_row([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?;
3124 let blob_bytes = if self.schema_has_blobs() {
3125 self.conn
3126 .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
3127 .query_row([], |row| row.get(0))?
3128 } else {
3129 0
3130 };
3131 let pressure = self
3132 .limits
3133 .blob_cache_max_bytes
3134 .is_some_and(|limit| blob_bytes > limit);
3135 Ok(ClientDiagnosticsStorage {
3136 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3137 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3138 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3139 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3140 retained_outcome_entries: Some(outcome_entries.max(0)),
3141 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3142 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3143 })
3144 };
3145 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3146 status: "unreadable".to_owned(),
3147 database_bytes_approx: None,
3148 pending_outbox_bytes_approx: None,
3149 retained_outcome_bytes_approx: None,
3150 retained_outcome_entries: None,
3151 blob_cache_bytes_approx: None,
3152 pressure_reason_code: None,
3153 })
3154 }
3155
3156 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3157 self.change_queue.drain(..).collect()
3158 }
3159
3160 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3161 let intents = self.sync_intent_queue.drain(..).collect();
3162 if self.transport_enabled {
3163 intents
3164 } else {
3165 vec![SyncIntent::None]
3166 }
3167 }
3168
3169 fn schedule_background_retry(&mut self) -> u64 {
3170 let delay_ms = self.retry_delay_ms;
3171 self.sync_intent_queue
3172 .push_back(SyncIntent::Background { delay_ms });
3173 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3174 self.round_retry_delay_ms = Some(delay_ms);
3175 delay_ms
3176 }
3177
3178 fn reset_background_retry(&mut self) {
3179 self.retry_delay_ms = 250;
3180 }
3181
3182 fn retryable_transport_code(code: &str) -> bool {
3183 code == "transport.failed"
3184 || code == "transport.timeout"
3185 || code == "transport.unavailable"
3186 || code == "sync.transport_failed"
3187 }
3188
3189 fn unreachable_transport(details: &Option<serde_json::Value>) -> bool {
3194 matches!(
3195 details
3196 .as_ref()
3197 .and_then(|details| details.get("causeKind"))
3198 .and_then(serde_json::Value::as_str),
3199 Some("connect" | "network")
3200 )
3201 }
3202
3203 fn segment_fetch_failed(&mut self, error: TransportError) -> SectionError {
3208 if ssp2::model::valid_round_failure_code(&error.code) {
3209 self.round_failure = Some(error.code.clone());
3210 }
3211 SectionError::Transport(error)
3212 }
3213
3214 fn retryable_failure_code(code: &str) -> bool {
3219 Self::retryable_transport_code(code)
3220 || matches!(
3221 code,
3222 "sync.auth_required"
3223 | "sync.auth_lease_required"
3224 | "sync.auth_lease_revoked"
3225 | "sync.internal_error"
3226 | "sync.idempotency_cache_miss"
3227 | "sync.schema_not_ready"
3228 | "sync.segment_expired"
3229 | "sync.rate_limited"
3230 | "sync.websocket_connection_limit"
3231 | "sync.idle_timeout"
3232 )
3233 }
3234
3235 fn diagnostic_code(code: &str) -> String {
3236 let valid = !code.is_empty()
3237 && code.len() <= 96
3238 && code.contains('.')
3239 && code
3240 .bytes()
3241 .next()
3242 .is_some_and(|byte| byte.is_ascii_lowercase())
3243 && code.bytes().all(|byte| {
3244 byte.is_ascii_lowercase()
3245 || byte.is_ascii_digit()
3246 || matches!(byte, b'.' | b'_' | b'-')
3247 });
3248 if valid {
3249 code.to_owned()
3250 } else {
3251 "client.unknown_failure".to_owned()
3252 }
3253 }
3254
3255 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3256 if self.sync_needed != value {
3257 if self.begin_observation("syncular_status").is_ok() {
3258 self.sync_needed = value;
3259 let batch = ChangeAccumulator {
3260 status: true,
3261 ..ChangeAccumulator::default()
3262 };
3263 if self.finish_observation("syncular_status", batch).is_err() {
3264 self.rollback_observation("syncular_status");
3265 }
3266 } else {
3267 self.sync_needed = value;
3268 }
3269 }
3270 if value && interactive {
3271 self.sync_intent_queue.push_back(SyncIntent::Interactive);
3272 }
3273 }
3274
3275 fn stored<T>(&self, result: rusqlite::Result<T>) -> Result<T, String> {
3278 result.map_err(|error| match error {
3279 rusqlite::Error::FromSqlConversionFailure(..) => {
3280 "sync.local_corrupt: persisted unit state is invalid".to_owned()
3281 }
3282 error => Self::sqlite_failure(&self.storage_failure, error),
3283 })
3284 }
3285
3286 fn sqlite_failure(
3287 storage_failure: &RefCell<Option<QueryReadFailure>>,
3288 error: rusqlite::Error,
3289 ) -> String {
3290 let message = error.to_string();
3291 let failure = QueryReadFailure::from(error);
3292 let code = failure.code;
3293 if code.is_some() && storage_failure.borrow().is_none() {
3294 *storage_failure.borrow_mut() = Some(failure);
3295 }
3296 match code {
3297 Some(code) => format!("{code}: {message}"),
3298 None => message,
3299 }
3300 }
3301
3302 fn begin_observation(&self, name: &str) -> Result<(), String> {
3303 if let Some(failure) = &self.coalesced_failure {
3304 return Err(failure.clone());
3305 }
3306 if self.coalesced.is_some() && self.conn.is_autocommit() {
3309 return Err("the coalesced apply transaction was rolled back".to_owned());
3310 }
3311 self.conn
3312 .execute_batch(&format!("SAVEPOINT {name}"))
3313 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3314 }
3315
3316 fn begin_coalesced_apply(&mut self) -> Result<(), String> {
3319 if self.coalesced.is_some() || !self.conn.is_autocommit() {
3320 return Ok(());
3321 }
3322 self.conn
3323 .execute_batch("BEGIN")
3324 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3325 self.coalesced = Some(CoalescedApply {
3326 rows: 0,
3327 rollback: self.capture(),
3328 });
3329 Ok(())
3330 }
3331
3332 fn capture(&self) -> Rollback {
3333 Rollback {
3334 outbox: self.outbox.clone(),
3335 failed_commits: self.failed_commits.clone(),
3336 conflicts: self.conflicts.clone(),
3337 rejections: self.rejections.clone(),
3338 schema_floor: self.schema_floor.clone(),
3339 lease_state: self.lease_state.clone(),
3340 stopped: self.stopped,
3341 upgrading: self.upgrading,
3342 sync_needed: self.sync_needed,
3343 active_round: self.active_round,
3344 overlay_dirty: self.overlay_dirty.snapshot(),
3345 change_queue: self.change_queue.len(),
3346 sync_intent_queue: self.sync_intent_queue.len(),
3347 last_change: self.last_change.clone(),
3348 }
3349 }
3350
3351 fn restore(&mut self, rollback: Rollback) {
3352 self.outbox = rollback.outbox;
3353 self.failed_commits = rollback.failed_commits;
3354 self.conflicts = rollback.conflicts;
3355 self.rejections = rollback.rejections;
3356 self.schema_floor = rollback.schema_floor;
3357 self.lease_state = rollback.lease_state;
3358 self.stopped = rollback.stopped;
3359 self.upgrading = rollback.upgrading;
3360 self.sync_needed = rollback.sync_needed;
3361 self.active_round = rollback.active_round;
3362 self.overlay_dirty.restore(rollback.overlay_dirty);
3363 self.change_queue.truncate(rollback.change_queue);
3364 self.sync_intent_queue.truncate(rollback.sync_intent_queue);
3365 self.last_change = rollback.last_change;
3366 }
3367
3368 fn coalesce_unit(&mut self, rows: usize) -> Result<(), String> {
3371 let Some(open) = &self.coalesced else {
3372 return Ok(());
3373 };
3374 if open.rows > 0 && open.rows + rows > COALESCED_APPLY_ROWS {
3375 self.commit_coalesced_apply()?;
3376 self.begin_coalesced_apply()?;
3377 }
3378 if let Some(open) = &mut self.coalesced {
3379 open.rows += rows;
3380 }
3381 Ok(())
3382 }
3383
3384 fn commit_coalesced_apply(&mut self) -> Result<(), String> {
3389 let Some(open) = self.coalesced.take() else {
3390 return Ok(());
3391 };
3392 let committed = if self.conn.is_autocommit() {
3393 Err("the coalesced apply transaction was rolled back".to_owned())
3394 } else {
3395 self.conn
3396 .execute_batch("COMMIT")
3397 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
3398 };
3399 let Err(error) = committed else {
3400 return Ok(());
3401 };
3402 if !self.conn.is_autocommit() {
3403 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3404 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3405 }
3406 }
3407 self.restore(open.rollback);
3408 self.coalesced_failure = Some(error.clone());
3409 Err(error)
3410 }
3411
3412 fn finish_coalesced_apply(&mut self) -> Result<(), String> {
3414 let committed = self.commit_coalesced_apply();
3415 match self.coalesced_failure.take() {
3416 Some(failure) => Err(failure),
3417 None => committed,
3418 }
3419 }
3420
3421 fn rollback_observation(&self, name: &str) {
3422 if let Err(error) = self
3423 .conn
3424 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"))
3425 {
3426 if let Some(failure) = self.storage_failure.borrow_mut().as_mut() {
3427 if failure.rollback_failure.is_none() {
3428 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(error)));
3429 }
3430 }
3431 }
3432 }
3433
3434 fn finish_observation(
3435 &mut self,
3436 name: &str,
3437 mut batch: ChangeAccumulator,
3438 ) -> Result<(), String> {
3439 #[cfg(feature = "bench-internals")]
3440 let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
3441 if self.failed_commits.iter().any(|(commit, _)| {
3442 commit
3443 .ops
3444 .iter()
3445 .any(|operation| batch.tables.contains_key(&operation.table))
3446 }) {
3447 batch.outcomes = true;
3448 }
3449 if !batch.touched() {
3450 self.conn
3451 .execute_batch(&format!("RELEASE {name}"))
3452 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3453 return Ok(());
3454 }
3455 let revision = self
3456 .local_revision()?
3457 .checked_add(1)
3458 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3459 self.conn
3460 .execute(
3461 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3462 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3463 )
3464 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3465 let status = batch.status.then(|| self.status_snapshot());
3466 let event = ClientChangeBatch {
3467 revision: revision.to_string(),
3468 tables: batch
3469 .tables
3470 .into_iter()
3471 .map(|(table, scope_keys)| TableChange {
3472 table,
3473 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3474 })
3475 .collect(),
3476 windows: batch
3477 .windows
3478 .into_iter()
3479 .map(|(base_key, units)| WindowChange {
3480 base_key,
3481 units: units.into_iter().collect(),
3482 })
3483 .collect(),
3484 status,
3485 conflicts_changed: batch.conflicts,
3486 rejections_changed: batch.rejections,
3487 outcomes_changed: batch.outcomes,
3488 applied_commits: Vec::new(),
3489 };
3490 self.conn
3491 .execute_batch(&format!("RELEASE {name}"))
3492 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3493 let mut diagnostic_tables = event
3494 .tables
3495 .iter()
3496 .map(|entry| entry.table.clone())
3497 .collect::<BTreeSet<_>>()
3498 .into_iter()
3499 .collect::<Vec<_>>();
3500 let mut diagnostic_windows = event
3502 .windows
3503 .iter()
3504 .map(|entry| {
3505 entry
3506 .base_key
3507 .split('\0')
3508 .next()
3509 .unwrap_or_default()
3510 .to_owned()
3511 })
3512 .collect::<BTreeSet<_>>()
3513 .into_iter()
3514 .collect::<Vec<_>>();
3515 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3516 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3517 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3518 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3519 self.last_change = Some(DiagnosticLastChange {
3520 revision: event.revision.clone(),
3521 recorded_at_ms: self.clock_now_ms(),
3522 tables: diagnostic_tables,
3523 windows: diagnostic_windows,
3524 domains_truncated,
3525 status_changed: event.status.is_some(),
3526 conflicts_changed: event.conflicts_changed,
3527 rejections_changed: event.rejections_changed,
3528 outcomes_changed: event.outcomes_changed,
3529 });
3530 self.change_queue.push_back(event);
3531 Ok(())
3532 }
3533
3534 fn record_scope_map(
3535 &self,
3536 batch: &mut ChangeAccumulator,
3537 table_name: &str,
3538 scopes: &[(String, Vec<String>)],
3539 ) {
3540 let Some(table) = self.schema.table(table_name) else {
3541 return;
3542 };
3543 if table.scope_variables.iter().any(|scope| {
3546 !scopes
3547 .iter()
3548 .any(|(variable, _)| *variable == scope.variable)
3549 }) {
3550 batch.table(table_name);
3551 return;
3552 }
3553 for (variable, values) in scopes {
3554 let Some(scope) = table
3555 .scope_variables
3556 .iter()
3557 .find(|scope| &scope.variable == variable)
3558 else {
3559 continue;
3560 };
3561 for value in values {
3562 batch.scope(table_name, format!("{}:{value}", scope.prefix));
3563 }
3564 }
3565 }
3566
3567 fn record_row_scopes(
3569 &self,
3570 batch: &mut ChangeAccumulator,
3571 table_name: &str,
3572 row_id: &str,
3573 base: bool,
3574 ) -> bool {
3575 let Some(table) = self.schema.table(table_name) else {
3576 return false;
3577 };
3578 if table.scope_variables.is_empty() {
3579 return false;
3580 }
3581 let columns = table
3582 .scope_variables
3583 .iter()
3584 .map(|scope| quote_ident(&scope.column))
3585 .collect::<Vec<_>>()
3586 .join(", ");
3587 let full_table = if base {
3588 base_table(table_name)
3589 } else {
3590 visible_table(table_name)
3591 };
3592 let sql = format!(
3593 "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
3594 row_id_predicate(table)
3595 );
3596 let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
3597 return false;
3598 };
3599 let values = stmt.query_row(rusqlite::params![row_id], |row| {
3600 let mut values = Vec::with_capacity(table.scope_variables.len());
3601 for index in 0..table.scope_variables.len() {
3602 values.push(row.get::<_, Option<String>>(index)?);
3603 }
3604 Ok(values)
3605 });
3606 let Ok(values) = values else {
3607 return false;
3608 };
3609 let mut recorded = false;
3610 for (scope, value) in table.scope_variables.iter().zip(values) {
3611 if let Some(value) = value {
3612 batch.scope(table_name, format!("{}:{value}", scope.prefix));
3613 recorded = true;
3614 }
3615 }
3616 recorded
3617 }
3618
3619 fn record_commit_changes(
3620 &self,
3621 batch: &mut ChangeAccumulator,
3622 tables: &[String],
3623 changes: &[ssp2::model::Change],
3624 ) {
3625 #[cfg(feature = "bench-internals")]
3626 let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
3627 for change in changes {
3628 let Some(table_name) = tables.get(change.table_index as usize) else {
3629 continue;
3630 };
3631 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3632 if let Some(table) = self.schema.table(table_name) {
3633 for (variable, value) in &change.scopes {
3634 if let Some(scope) = table
3635 .scope_variables
3636 .iter()
3637 .find(|scope| &scope.variable == variable)
3638 {
3639 batch.scope(table_name, format!("{}:{value}", scope.prefix));
3640 precise = true;
3641 }
3642 }
3643 }
3644 if !precise {
3645 batch.table(table_name);
3646 }
3647 }
3648 }
3649
3650 fn scoped_rows_exist(
3653 &self,
3654 table_name: &str,
3655 effective: &[(String, Vec<String>)],
3656 owners: &[Vec<(String, Vec<String>)>],
3657 ) -> Result<bool, String> {
3658 let Some(table) = self.schema.table(table_name) else {
3659 return Ok(false);
3660 };
3661 let Ok(Some((predicate, params))) = local_scope_predicate(table, effective, owners) else {
3662 return Ok(false);
3663 };
3664 for source in [base_table(table_name), visible_table(table_name)] {
3665 if self
3666 .conn
3667 .query_row(
3668 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
3669 rusqlite::params_from_iter(¶ms),
3670 |row| row.get::<_, bool>(0),
3671 )
3672 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
3673 {
3674 return Ok(true);
3675 }
3676 }
3677 Ok(false)
3678 }
3679
3680 fn clear_scope(
3685 &mut self,
3686 table: &TableSchema,
3687 effective: &[(String, Vec<String>)],
3688 unit_id: &str,
3689 ) -> Result<bool, SectionError> {
3690 let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
3691 let matched = self
3692 .scoped_rows_exist(&table.name, effective, &[])
3693 .map_err(storage)?;
3694 let owners = if matched {
3695 unit_owners(
3696 &self.stored(load_units(&self.conn)).map_err(storage)?,
3697 table,
3698 unit_id,
3699 )
3700 } else {
3701 Vec::new()
3702 };
3703 let changed = matched
3704 && (owners.is_empty()
3705 || self
3706 .scoped_rows_exist(&table.name, effective, &owners)
3707 .map_err(storage)?);
3708 if !self
3709 .purge_scope_rows(&table.name, effective, &owners)
3710 .map_err(storage)?
3711 {
3712 return Err(SectionError::FailClosed);
3713 }
3714 Ok(changed)
3715 }
3716
3717 pub fn upgrading(&self) -> bool {
3719 self.upgrading
3720 }
3721
3722 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3730 let new_schema = parse_schema_json(schema_json)?;
3731 validate_authority_reads(&new_schema, &self.limits.authority_reads)?;
3732 self.conn
3733 .execute_batch("BEGIN IMMEDIATE")
3734 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3735 let prior_schema = std::mem::replace(&mut self.schema, new_schema);
3736 let rollback = self.capture();
3742 let guarded = (|| -> Result<(), String> {
3743 let marker = read_local_schema_version(&self.conn, self.schema.version)?;
3748 self.create_bookkeeping_tables()?;
3749 match marker {
3750 Some(version) if version < self.schema.version => self.run_schema_reset()?,
3751 None => {
3752 self.create_synced_tables()?;
3753 self.persist_schema_version()?;
3754 }
3755 _ => {}
3756 }
3757 self.outbox = self.load_outbox()?.into();
3762 self.drop_unencodable_outbox(false, None)?;
3767 if marker == Some(self.schema.version)
3768 && (!self.outbox.is_empty()
3769 || !self.failed_commits.is_empty()
3770 || self.has_acknowledged_rows()?)
3771 {
3772 self.overlay_dirty.set(true);
3773 self.rebuild_overlay()?;
3774 }
3775 self.prune_unknown_units(false)?;
3776 self.save_unit_scope_schema()?;
3777 self.enqueue_startup_sync_if_needed()?;
3781 Ok(())
3782 })();
3783 let outcome = match guarded {
3784 Ok(()) => self
3785 .conn
3786 .execute_batch("COMMIT")
3787 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error)),
3788 Err(error) => Err(error),
3789 };
3790 if let Err(error) = outcome {
3791 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
3792 let _ = Self::sqlite_failure(&self.storage_failure, rollback);
3794 }
3795 self.schema = prior_schema;
3796 self.restore(rollback);
3797 self.insert_sql.borrow_mut().clear();
3799 return Err(error);
3800 }
3801 if !self.security_preflight {
3802 self.drain_pending_evictions()?;
3803 }
3804 Ok(())
3805 }
3806
3807 fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
3808 let startup_work = !self.stopped
3809 && (!self.outbox.is_empty()
3810 || self.has_acknowledged_rows()?
3811 || !self.load_pending_evictions()?.is_empty()
3812 || self
3813 .stored(load_units(&self.conn))?
3814 .iter()
3815 .any(|unit| unit.status == UnitStatus::Active));
3816 if startup_work {
3817 self.sync_needed = true;
3818 self.sync_intent_queue.push_back(SyncIntent::Interactive);
3819 }
3820 Ok(())
3821 }
3822
3823 fn clear_satisfied_persisted_schema_floor(&mut self) -> Result<(), String> {
3830 let satisfied = self
3831 .schema_floor
3832 .as_ref()
3833 .and_then(|floor| floor.required_schema_version)
3834 .is_some_and(|required| self.schema.version >= required);
3835 if !satisfied {
3836 return Ok(());
3837 }
3838 self.delete_meta(SCHEMA_FLOOR_KEY)?;
3839 self.schema_floor = None;
3840 self.stopped = false;
3841 Ok(())
3842 }
3843
3844 fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
3849 let Some(replica_path) = self.previous_version_replica_path() else {
3850 return Ok(());
3851 };
3852 let max_age_ms = self
3853 .previous_version
3854 .filter(|config| config.enabled)
3855 .map(|config| config.max_age_ms);
3856 reconcile_previous_version_at_boot(
3857 &self.conn,
3858 &replica_path,
3859 self.schema.version,
3860 max_age_ms,
3861 self.clock_now_ms(),
3862 )
3863 }
3864
3865 fn previous_version_replica_path(&self) -> Option<String> {
3869 let path = self.conn.path()?;
3870 if path.is_empty() || path.starts_with("file:") {
3871 return None;
3872 }
3873 Some(path.to_owned())
3874 }
3875
3876 fn run_schema_reset(&mut self) -> Result<(), String> {
3882 self.begin_observation("syncular_schema_reset")?;
3883 let rollback = self.capture();
3884 let mut batch = ChangeAccumulator::default();
3885 let result = self
3886 .run_schema_reset_observed(&mut batch, true)
3887 .and_then(|()| self.finish_observation("syncular_schema_reset", batch));
3888 if let Err(error) = result {
3889 self.rollback_observation("syncular_schema_reset");
3890 self.restore(rollback);
3891 return Err(error);
3892 }
3893 Ok(())
3894 }
3895
3896 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3897 if self.get_meta(LOG_EPOCH_KEY)?.is_none() {
3898 self.begin_observation("syncular_epoch_acquisition")?;
3899 let rollback = self.capture();
3900 if let Err(error) = self.conn.execute(
3901 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3902 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3903 ) {
3904 self.rollback_observation("syncular_epoch_acquisition");
3905 return Err(Self::sqlite_failure(&self.storage_failure, error));
3906 }
3907 self.sync_needed = true;
3908 let batch = ChangeAccumulator {
3909 status: true,
3910 ..ChangeAccumulator::default()
3911 };
3912 if let Err(error) = self.finish_observation("syncular_epoch_acquisition", batch) {
3913 self.rollback_observation("syncular_epoch_acquisition");
3914 self.restore(rollback);
3915 return Err(error);
3916 }
3917 self.sync_intent_queue.push_back(SyncIntent::Interactive);
3918 return Ok(Vec::new());
3919 }
3920 let resets = self
3921 .stored(load_units(&self.conn))?
3922 .into_iter()
3923 .map(|unit| unit.id)
3924 .collect();
3925 self.begin_observation("syncular_log_epoch_reset")?;
3926 let rollback = self.capture();
3927 let mut batch = ChangeAccumulator::default();
3928 let result = self
3929 .run_schema_reset_observed(&mut batch, false)
3930 .and_then(|()| {
3931 self.conn
3932 .execute(
3933 "INSERT OR REPLACE INTO _syncular_meta(key,value) VALUES (?1,?2)",
3934 rusqlite::params![LOG_EPOCH_KEY, log_epoch],
3935 )
3936 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
3937 self.sync_needed = true;
3938 batch.status = true;
3939 self.finish_observation("syncular_log_epoch_reset", batch)
3940 });
3941 if let Err(error) = result {
3942 self.rollback_observation("syncular_log_epoch_reset");
3943 self.restore(rollback);
3944 return Err(error);
3945 }
3946 self.sync_intent_queue.push_back(SyncIntent::Interactive);
3947 Ok(resets)
3948 }
3949
3950 fn run_schema_reset_observed(
3951 &mut self,
3952 batch: &mut ChangeAccumulator,
3953 capture: bool,
3954 ) -> Result<(), String> {
3955 read_local_schema_version(&self.conn, self.schema.version)?;
3961 let mut kept = BTreeSet::new();
3965 let mut previous_declarations: Option<BTreeMap<String, BTreeMap<String, Value>>> = None;
3966 if capture {
3967 let persisted: BTreeMap<String, String> = self
3968 .get_meta(TABLE_GENERATIONS_KEY)?
3969 .map(|raw| serde_json::from_str(&raw))
3970 .transpose()
3971 .map_err(|_| "sync.local_corrupt: invalid persisted table generations".to_owned())?
3972 .unwrap_or_default();
3973 kept = self
3974 .schema
3975 .tables
3976 .iter()
3977 .filter(|table| {
3978 table.generation.is_some()
3979 && table.generation.as_ref() == persisted.get(&table.name)
3980 })
3981 .map(|table| table.name.clone())
3982 .collect();
3983 previous_declarations = self
3984 .get_meta("subscriptionScopeSchema")?
3985 .map(|raw| serde_json::from_str(&raw))
3986 .transpose()
3987 .map_err(|_| "sync.local_corrupt: invalid persisted scope schema".to_owned())?;
3988 }
3989 let mut kept_objects = BTreeSet::new();
3992 for table in self
3993 .schema
3994 .tables
3995 .iter()
3996 .filter(|table| kept.contains(&table.name))
3997 {
3998 kept_objects.insert(table.name.clone());
3999 kept_objects.insert(format!("_syncular_base_{}", table.name));
4000 for index in &table.fts_indexes {
4001 kept_objects.insert(index.name.clone());
4002 for shadow in ["data", "idx", "content", "docsize", "config"] {
4003 kept_objects.insert(format!("{}_{shadow}", index.name));
4004 }
4005 }
4006 }
4007 let kept_json = serde_json::to_string(&kept).expect("table names serialize");
4008 for sql in [
4009 "DELETE FROM _syncular_acked_rows WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4010 "DELETE FROM _syncular_row_deliveries WHERE tbl NOT IN (SELECT value FROM json_each(?1))",
4011 ] {
4012 self.conn
4013 .execute(sql, [&kept_json])
4014 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4015 }
4016 self.cancel_sync_round();
4017 self.upgrading = true;
4018 batch.status = true;
4019 if self.previous_version.is_some_and(|config| config.enabled) {
4027 let previous_version = self
4028 .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4029 .and_then(|value| value.parse::<i32>().ok())
4030 .unwrap_or(self.schema.version);
4031 let pending = self
4032 .outbox
4033 .iter()
4034 .map(|commit| PendingCommitAudit {
4035 commit_id: &commit.client_commit_id,
4036 operations: commit_audit_operations(commit),
4037 })
4038 .collect::<Vec<_>>();
4039 crate::previous_version::write_previous_version_audit(
4040 &self.conn,
4041 &crate::previous_version::build_previous_version_audit(
4042 &self.schema,
4043 &pending,
4044 previous_version,
4045 self.schema.version,
4046 self.clock_now_ms(),
4047 ),
4048 )?;
4049 }
4050 if let Some(replica_path) = self.previous_version_replica_path() {
4055 sweep_previous_version_container(&self.conn, &replica_path)?;
4056 if capture {
4057 if let Some(config) = self.previous_version.filter(|config| config.enabled) {
4058 let previous_version = self
4059 .get_meta(LOCAL_SCHEMA_VERSION_KEY)?
4060 .and_then(|value| value.parse::<i32>().ok())
4061 .unwrap_or(self.schema.version);
4062 capture_previous_version_from_replica(
4063 &self.conn,
4064 &replica_path,
4065 previous_version,
4066 self.schema.version,
4067 &config,
4068 self.clock_now_ms(),
4069 )?;
4070 }
4071 }
4072 }
4073 for table in self
4074 .schema
4075 .tables
4076 .iter()
4077 .filter(|table| !kept.contains(&table.name))
4078 {
4079 batch.table(&table.name);
4080 }
4081 for (base_key, value) in self.load_window_registry()? {
4082 batch.window(&base_key, &value);
4083 }
4084 if capture {
4085 self.prune_unknown_units(true)?;
4086 }
4087 self.insert_sql.borrow_mut().clear();
4089 self.overlay_dirty.set(true);
4090 let virtual_tables: Vec<String> = {
4098 let mut stmt = self
4099 .conn
4100 .prepare(
4101 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
4102 )
4103 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4104 let rows = stmt
4105 .query_map([], |row| row.get::<_, String>(0))
4106 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4107 rows.filter_map(Result::ok)
4108 .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4109 .collect()
4110 };
4111 for name in virtual_tables {
4112 self.conn
4113 .execute_batch(&format!(
4114 "DROP TABLE IF EXISTS {}",
4115 quote_ident(&format!("_syncular_fts_{name}"))
4116 ))
4117 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4118 self.conn
4119 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4120 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4121 }
4122 let existing: Vec<String> = {
4123 let mut stmt = self
4124 .conn
4125 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
4126 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4127 let rows = stmt
4128 .query_map([], |row| row.get::<_, String>(0))
4129 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4130 rows.filter_map(Result::ok)
4131 .filter(|name| is_synced_table_name(name) && !kept_objects.contains(name))
4132 .collect()
4133 };
4134 for name in &existing {
4135 if let Some(table) = name.strip_prefix("_syncular_base_") {
4136 batch.table(table);
4137 } else if !name.starts_with("_syncular_") {
4138 batch.table(name);
4139 }
4140 }
4141 for name in existing {
4142 self.conn
4143 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
4144 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4145 }
4146 self.create_synced_tables()?;
4148 let keep_units = match &previous_declarations {
4152 Some(previous) if !kept.is_empty() => self
4153 .stored(load_units(&self.conn))?
4154 .into_iter()
4155 .filter(|unit| {
4156 previous.iter().all(|(name, old)| {
4157 kept.contains(name) || !unit.scopes.keys().all(|v| old.contains_key(v))
4158 }) && self
4159 .schema
4160 .tables
4161 .iter()
4162 .all(|table| kept.contains(&table.name) || !covers(table, &unit.scopes))
4163 })
4164 .map(|unit| unit.id)
4165 .collect(),
4166 _ => BTreeSet::new(),
4167 };
4168 reset_units(&self.conn, &keep_units)
4169 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4170 self.stopped = false;
4172 self.schema_floor = None;
4173 self.delete_meta(SCHEMA_FLOOR_KEY)?;
4174 self.persist_schema_version()?;
4176 self.save_unit_scope_schema()?;
4177 self.drop_unencodable_outbox(false, Some(batch))?;
4188 self.rebuild_overlay()?;
4190 Ok(())
4191 }
4192
4193 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
4202 let incompatible = self
4203 .outbox
4204 .iter()
4205 .filter(|commit| {
4206 first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
4207 })
4208 .cloned()
4209 .collect::<Vec<_>>();
4210 if incompatible.is_empty() {
4211 return Ok(false);
4212 }
4213 self.drop_outbox_commits(
4214 &incompatible,
4215 OUTBOX_INCOMPATIBLE_CODE,
4216 "the persisted commit cannot encode under the current schema",
4217 )?;
4218 Ok(true)
4219 }
4220
4221 fn drop_outbox_commits(
4225 &mut self,
4226 incompatible: &[Arc<OutboxCommit>],
4227 code: &str,
4228 message: &str,
4229 ) -> Result<(), String> {
4230 let rollback = self.capture();
4236 let owns_transaction = self.conn.is_autocommit();
4237 self.begin_observation("syncular_outbox_drop")?;
4238 let mut batch = ChangeAccumulator::default();
4239 let dropped = (|| -> Result<(), String> {
4240 let mut rejections = Vec::new();
4241 for commit in incompatible {
4242 let results = commit
4243 .ops
4244 .iter()
4245 .enumerate()
4246 .map(|(op_index, operation)| {
4247 let rejection = RejectionRecord {
4248 client_commit_id: commit.client_commit_id.clone(),
4249 op_index: op_index as i32,
4250 code: code.to_owned(),
4251 message: message.to_owned(),
4252 retryable: false,
4253 details: None,
4254 operation: Some(CommitOperation::from(operation)),
4255 };
4256 rejections.push(rejection.clone());
4257 CommitOperationOutcome::Error { rejection }
4258 })
4259 .collect::<Vec<_>>();
4260 self.persist_commit_outcome(
4261 &commit.client_commit_id,
4262 CommitOutcomeStatus::Rejected,
4263 &results,
4264 Some(&commit.ops),
4265 )?;
4266 self.delete_outbox_persisted(&commit.client_commit_id)?;
4267 for operation in &commit.ops {
4270 if self.schema.table(&operation.table).is_some() {
4271 batch.table(&operation.table);
4272 }
4273 }
4274 }
4275 self.prune_commit_outcomes()?;
4276 let incompatible_ids = incompatible
4277 .iter()
4278 .map(|commit| commit.client_commit_id.as_str())
4279 .collect::<BTreeSet<_>>();
4280 self.outbox
4281 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
4282 self.rejections.extend(rejections);
4283 self.overlay_dirty.set(true);
4286 self.rebuild_overlay()?;
4287 batch.status = true;
4288 batch.rejections = true;
4289 batch.outcomes = true;
4290 Ok(())
4291 })();
4292 let result = dropped.and_then(|()| self.finish_observation("syncular_outbox_drop", batch));
4293 if let Err(error) = result {
4294 if owns_transaction {
4298 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
4299 let mut failure = self.storage_failure.borrow_mut();
4300 let failure =
4301 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
4302 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
4303 }
4304 } else {
4305 self.rollback_observation("syncular_outbox_drop");
4306 }
4307 self.restore(rollback);
4308 return Err(error);
4309 }
4310 Ok(())
4311 }
4312
4313 fn create_synced_tables(&self) -> Result<(), String> {
4316 for table in &self.schema.tables {
4317 for (full, index_prefix) in [
4321 (base_table(&table.name), "_syncular_base_"),
4322 (visible_table(&table.name), ""),
4323 ] {
4324 let mut cols: Vec<String> =
4325 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
4326 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
4327 cols.extend(
4328 table
4329 .sealed_columns()
4330 .iter()
4331 .map(|name| format!("{} TEXT", quote_ident(name))),
4332 );
4333 let sql = format!(
4334 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
4335 cols.join(", "),
4336 quote_ident(&table.primary_key)
4337 );
4338 self.conn
4339 .execute(&sql, [])
4340 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4341 for name in table.sealed_columns() {
4345 let present: bool = self
4346 .conn
4347 .query_row(
4348 "SELECT EXISTS(SELECT 1 FROM pragma_table_info(?1) WHERE name = ?2)",
4349 rusqlite::params![format!("{index_prefix}{}", table.name), name],
4350 |row| row.get(0),
4351 )
4352 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4353 if !present {
4354 self.conn
4355 .execute(
4356 &format!(
4357 "ALTER TABLE {full} ADD COLUMN {} TEXT",
4358 quote_ident(name)
4359 ),
4360 [],
4361 )
4362 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4363 }
4364 }
4365 if table.has_encrypted_columns() && !index_prefix.is_empty() {
4368 self.conn
4369 .execute(
4370 &format!(
4371 "CREATE INDEX IF NOT EXISTS {} ON {full} (\"_sync_sealed_key\") WHERE \"_sync_sealed_key\" IS NOT NULL",
4372 quote_ident(&format!("_syncular_sealed_{}", table.name))
4373 ),
4374 [],
4375 )
4376 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4377 }
4378 for index in &table.indexes {
4382 let unique = if index.unique { "UNIQUE " } else { "" };
4383 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
4384 let cols_sql = index
4385 .columns
4386 .iter()
4387 .map(|c| quote_ident(c))
4388 .collect::<Vec<_>>()
4389 .join(", ");
4390 let index_sql = format!(
4391 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
4392 );
4393 self.conn
4394 .execute(&index_sql, [])
4395 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4396 }
4397 }
4398 }
4399 for table in &self.schema.tables {
4402 for index in &table.fts_indexes {
4403 self.create_fts_projection(table, index)?;
4404 }
4405 }
4406 Ok(())
4407 }
4408
4409 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
4410 let count: i64 = self
4411 .conn
4412 .query_row(
4413 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
4414 rusqlite::params![index.name],
4415 |row| row.get(0),
4416 )
4417 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4418 Ok(count > 0)
4419 }
4420
4421 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
4422 for suffix in ["ai", "ad", "au"] {
4423 self.conn
4424 .execute(
4425 &format!(
4426 "DROP TRIGGER IF EXISTS {}",
4427 quote_ident(&format!("{}_{suffix}", index.name))
4428 ),
4429 [],
4430 )
4431 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4432 }
4433 Ok(())
4434 }
4435
4436 fn create_fts_triggers(
4437 &self,
4438 table: &TableSchema,
4439 index: &FtsIndexSchema,
4440 ) -> Result<(), String> {
4441 let fts = quote_ident(&index.name);
4442 let source = visible_table(&table.name);
4443 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4444 let pk = quote_ident(&table.primary_key);
4445 let projection_columns = std::iter::once(source_id.clone())
4446 .chain(index.columns.iter().map(|column| quote_ident(column)))
4447 .collect::<Vec<_>>()
4448 .join(", ");
4449 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
4450 .chain(
4451 index
4452 .columns
4453 .iter()
4454 .map(|column| format!("new.{}", quote_ident(column))),
4455 )
4456 .collect::<Vec<_>>()
4457 .join(", ");
4458 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4459 let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
4460 let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
4461 let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
4462 let bi = quote_ident(&format!("{}_bi", index.name));
4463 let ai = quote_ident(&format!("{}_ai", index.name));
4464 let ad = quote_ident(&format!("{}_ad", index.name));
4465 let au = quote_ident(&format!("{}_au", index.name));
4466 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
4467 let sql = format!(
4468 "DROP TRIGGER IF EXISTS {bi};
4469 DROP TRIGGER IF EXISTS {ai};
4470 DROP TRIGGER IF EXISTS {ad};
4471 DROP TRIGGER IF EXISTS {au};
4472 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
4473 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
4474 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
4475 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
4476 );
4477 self.conn
4478 .execute_batch(&sql)
4479 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4480 }
4481
4482 fn rebuild_fts_projection(
4483 &self,
4484 table: &TableSchema,
4485 index: &FtsIndexSchema,
4486 ) -> Result<(), String> {
4487 let fts = quote_ident(&index.name);
4488 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
4489 let pk = quote_ident(&table.primary_key);
4490 let indexed_columns = index
4491 .columns
4492 .iter()
4493 .map(|column| quote_ident(column))
4494 .collect::<Vec<_>>();
4495 let projection_columns = std::iter::once(source_id.clone())
4496 .chain(indexed_columns.iter().cloned())
4497 .collect::<Vec<_>>()
4498 .join(", ");
4499 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4500 let sql = format!(
4501 "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
4502 columns = indexed_columns.join(", "),
4503 source = visible_table(&table.name),
4504 );
4505 self.conn
4506 .execute_batch(&sql)
4507 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4508 }
4509
4510 fn create_fts_projection(
4511 &self,
4512 table: &TableSchema,
4513 index: &FtsIndexSchema,
4514 ) -> Result<(), String> {
4515 self.begin_observation("syncular_fts_schema")?;
4516 let result = (|| {
4517 let existed = self.fts_projection_exists(index)?;
4518 let tokenizer = index.tokenize.replace('\'', "''");
4519 let columns = index
4520 .columns
4521 .iter()
4522 .map(|column| quote_ident(column))
4523 .collect::<Vec<_>>()
4524 .join(", ");
4525 let sql = format!(
4526 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
4527 fts = quote_ident(&index.name),
4528 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
4529 );
4530 self.conn.execute(&sql, []).map_err(|error| {
4531 format!(
4532 "cannot create local FTS5 projection {:?}: {error}",
4533 index.name
4534 )
4535 })?;
4536 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
4537 let mapped: bool = self
4538 .conn
4539 .query_row(
4540 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
4541 [format!("_syncular_fts_{}", index.name)],
4542 |row| row.get(0),
4543 )
4544 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4545 self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4546 if !mapped && existed {
4547 self.conn
4548 .execute_batch(&format!(
4549 "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
4550 quote_ident(FTS_SOURCE_ID_COLUMN),
4551 quote_ident(&index.name)
4552 ))
4553 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4554 }
4555 self.create_fts_triggers(table, index)?;
4556 let stale: bool = !existed
4560 || self
4561 .conn
4562 .query_row(
4563 &format!(
4564 "SELECT (SELECT count(*) FROM {mapping}) != (SELECT count(*) FROM {})",
4565 visible_table(&table.name)
4566 ),
4567 [],
4568 |row| row.get(0),
4569 )
4570 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4571 if stale {
4572 self.rebuild_fts_projection(table, index)?;
4573 }
4574 Ok(())
4575 })();
4576 if let Err(error) = result {
4577 self.rollback_observation("syncular_fts_schema");
4578 return Err(error);
4579 }
4580 if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
4581 self.rollback_observation("syncular_fts_schema");
4582 return Err(error.to_string());
4583 }
4584 Ok(())
4585 }
4586
4587 fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
4590 let ops: Vec<Value> = commit
4591 .ops
4592 .iter()
4593 .map(|op| {
4594 let mut entry = serde_json::json!({
4595 "op": if op.upsert { "upsert" } else { "delete" },
4596 "table": op.table,
4597 "rowId": op.row_id,
4598 "baseVersion": op.base_version,
4599 "values": op.values.clone().map(Value::Object),
4600 });
4601 if op.insert {
4602 entry["insert"] = Value::Bool(true);
4603 }
4604 entry
4605 })
4606 .collect();
4607 self.conn
4608 .execute(
4609 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
4610 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
4611 )
4612 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4613 for operation in &commit.ops {
4614 let Some(values) = &operation.values else {
4615 continue;
4616 };
4617 let Some(table) = self.schema.table(&operation.table) else {
4618 continue;
4619 };
4620 for column in table
4621 .columns
4622 .iter()
4623 .filter(|column| column.ty == ColumnType::BlobRef)
4624 {
4625 let Some(Value::String(raw)) = values.get(&column.name) else {
4626 continue;
4627 };
4628 let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
4629 continue;
4630 };
4631 let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
4632 continue;
4633 };
4634 self.conn
4635 .execute(
4636 "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
4637 SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
4638 rusqlite::params![commit.client_commit_id, blob_id],
4639 )
4640 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4641 }
4642 }
4643 Ok(())
4644 }
4645
4646 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
4647 self.conn
4648 .execute(
4649 "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
4650 rusqlite::params![client_commit_id],
4651 )
4652 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4653 self.conn
4654 .execute(
4655 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
4656 rusqlite::params![client_commit_id],
4657 )
4658 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4659 self.conn.execute("DELETE FROM _syncular_row_deliveries WHERE NOT EXISTS(SELECT 1 FROM _syncular_outbox,json_each(ops_json) WHERE json_extract(json_each.value,'$.table')=tbl AND json_extract(json_each.value,'$.rowId')=id)", [])
4660 .map(|_| ())
4661 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4662 }
4663
4664 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
4665 match status {
4666 CommitOutcomeStatus::Applied => "applied",
4667 CommitOutcomeStatus::Cached => "cached",
4668 CommitOutcomeStatus::Conflict => "conflict",
4669 CommitOutcomeStatus::Rejected => "rejected",
4670 }
4671 }
4672
4673 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
4674 match resolution {
4675 CommitOutcomeResolution::Active => "active",
4676 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
4677 CommitOutcomeResolution::Superseded => "superseded",
4678 CommitOutcomeResolution::Dismissed => "dismissed",
4679 }
4680 }
4681
4682 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
4683 match value {
4684 "applied" => Ok(CommitOutcomeStatus::Applied),
4685 "cached" => Ok(CommitOutcomeStatus::Cached),
4686 "conflict" => Ok(CommitOutcomeStatus::Conflict),
4687 "rejected" => Ok(CommitOutcomeStatus::Rejected),
4688 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
4689 }
4690 }
4691
4692 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
4693 match value {
4694 "active" => Ok(CommitOutcomeResolution::Active),
4695 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
4696 "superseded" => Ok(CommitOutcomeResolution::Superseded),
4697 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
4698 _ => Err(format!(
4699 "invalid persisted commit outcome resolution {value:?}"
4700 )),
4701 }
4702 }
4703
4704 fn persist_commit_outcome(
4705 &self,
4706 client_commit_id: &str,
4707 status: CommitOutcomeStatus,
4708 results: &[CommitOperationOutcome],
4709 operations: Option<&[OutboxOp]>,
4710 ) -> Result<(), String> {
4711 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
4712 let operations_json = operations
4713 .map(|items| {
4714 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
4715 })
4716 .transpose()
4717 .map_err(|error| error.to_string())?;
4718 self.conn
4719 .execute(
4720 "INSERT INTO _syncular_commit_outcomes (
4721 client_commit_id, status, recorded_at_ms, results_json,
4722 operations_json, resolution
4723 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
4724 rusqlite::params![
4725 client_commit_id,
4726 Self::outcome_status_name(status),
4727 self.clock_now_ms(),
4728 results_json,
4729 operations_json
4730 ],
4731 )
4732 .map(|_| ())
4733 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
4734 }
4735
4736 fn stored_cell_is_representable(cell: &Value) -> bool {
4739 cell.is_null()
4740 || cell.is_string()
4741 || cell.is_boolean()
4742 || cell.is_number()
4743 || is_canonical_bytes_envelope(cell)
4744 }
4745
4746 fn stored_row_is_representable(value: &Value) -> bool {
4747 value
4748 .as_object()
4749 .is_some_and(|row| row.values().all(Self::stored_cell_is_representable))
4750 }
4751
4752 fn stored_operation_is_representable(value: &Value) -> bool {
4753 value.get("table").is_some_and(Value::is_string)
4754 && value.get("rowId").is_some_and(Value::is_string)
4755 && value.get("baseVersion").is_none_or(|v| {
4756 v.as_i64()
4757 .is_some_and(|n| n.unsigned_abs() <= MAX_JS_SAFE_INTEGER)
4758 })
4759 && match value.get("op").and_then(Value::as_str) {
4760 Some("upsert") => {
4761 value
4762 .get("values")
4763 .is_some_and(Self::stored_row_is_representable)
4764 && value
4765 .get("insert")
4766 .is_none_or(|insert| *insert == Value::Bool(true))
4767 }
4768 Some("delete") => value.get("values").is_none() && value.get("insert").is_none(),
4769 _ => false,
4770 }
4771 }
4772
4773 fn outbox_op_is_journalable(operation: &OutboxOp) -> bool {
4777 serde_json::to_value(CommitOperation::from(operation))
4778 .is_ok_and(|value| Self::stored_operation_is_representable(&value))
4779 }
4780
4781 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
4782 let StoredCommitOutcomeRow {
4783 sequence,
4784 client_commit_id,
4785 status,
4786 recorded_at_ms,
4787 results_json,
4788 operations_json,
4789 resolution,
4790 resolved_at_ms,
4791 replacement_client_commit_id,
4792 } = row;
4793 if sequence.unsigned_abs() > MAX_JS_SAFE_INTEGER
4794 || recorded_at_ms.unsigned_abs() > MAX_JS_SAFE_INTEGER
4795 || resolved_at_ms.is_some_and(|value| value.unsigned_abs() > MAX_JS_SAFE_INTEGER)
4796 {
4797 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned());
4798 }
4799 let invalid = || "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
4800 let stored: Vec<Value> = serde_json::from_str(&results_json).map_err(|_| invalid())?;
4801 for result in &stored {
4802 let nested = match result.get("status").and_then(Value::as_str) {
4803 Some("conflict") => result.get("conflict"),
4804 Some("error") => result.get("rejection"),
4805 _ => None,
4806 };
4807 if nested
4808 .and_then(|record| record.get("operation"))
4809 .is_some_and(|op| !Self::stored_operation_is_representable(op))
4810 || (result.get("status").and_then(Value::as_str) == Some("conflict")
4811 && nested
4812 .and_then(|record| record.get("serverRow"))
4813 .is_some_and(|row| !Self::stored_row_is_representable(row)))
4814 {
4815 return Err(invalid());
4816 }
4817 }
4818 let results: Vec<CommitOperationOutcome> =
4819 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())?;
4820 let has_conflict = results
4821 .iter()
4822 .any(|result| matches!(result, CommitOperationOutcome::Conflict { .. }));
4823 let has_error = results
4824 .iter()
4825 .any(|result| matches!(result, CommitOperationOutcome::Error { .. }));
4826 let status = Self::parse_outcome_status(&status).map_err(|_| invalid())?;
4827 let redacted = results.is_empty() && operations_json.is_none();
4828 if match status {
4829 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => has_conflict || has_error,
4830 CommitOutcomeStatus::Conflict => !(has_conflict || redacted),
4831 CommitOutcomeStatus::Rejected => has_conflict || !(has_error || redacted),
4832 } {
4833 return Err(invalid());
4834 }
4835 let operations: Option<Vec<CommitOperation>> = operations_json
4836 .map(|raw| {
4837 let stored: Vec<Value> = serde_json::from_str(&raw).map_err(|_| invalid())?;
4838 if !stored.iter().all(Self::stored_operation_is_representable) {
4839 return Err(invalid());
4840 }
4841 serde_json::from_value(Value::Array(stored)).map_err(|_| invalid())
4842 })
4843 .transpose()?;
4844 for result in &results {
4845 match result {
4846 CommitOperationOutcome::Conflict { conflict }
4847 if conflict.server_version.unsigned_abs() > MAX_JS_SAFE_INTEGER =>
4848 {
4849 return Err("sync.local_corrupt: persisted commit outcome is invalid".to_owned())
4850 }
4851 CommitOperationOutcome::Error { rejection } => {
4852 if let Some(details) = &rejection.details {
4853 let encoded = serde_json::to_string(details).map_err(|_| {
4854 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4855 })?;
4856 RejectionDetails::parse(&encoded).map_err(|_| {
4857 "sync.local_corrupt: persisted commit outcome is invalid".to_owned()
4858 })?;
4859 }
4860 }
4861 _ => {}
4862 }
4863 }
4864 Ok(CommitOutcome {
4865 sequence,
4866 client_commit_id,
4867 status,
4868 recorded_at_ms,
4869 results,
4870 operations,
4871 retained_rows: None,
4872 resolution: Self::parse_outcome_resolution(&resolution)?,
4873 resolved_at_ms,
4874 replacement_client_commit_id,
4875 })
4876 }
4877
4878 fn unique_conflicts(
4880 &self,
4881 table: &TableSchema,
4882 values: &Map<String, Value>,
4883 base: bool,
4884 ) -> Result<Vec<RetainedUniqueConflict>, String> {
4885 let source = if base {
4886 base_table(&table.name)
4887 } else {
4888 visible_table(&table.name)
4889 };
4890 let mut conflicts = Vec::new();
4891 for index in table.indexes.iter().filter(|index| index.unique) {
4892 let mut params = vec![json_param_to_sql(
4893 values.get(&table.primary_key).ok_or("sync.local_corrupt")?,
4894 )?];
4895 for name in &index.columns {
4896 let column = table
4897 .columns
4898 .iter()
4899 .find(|column| column.name == *name)
4900 .ok_or("sync.local_corrupt")?;
4901 let cell = json_to_column_value(column, values.get(name))?;
4902 params.push(owned_sql_value(RowParam::Cell(&cell))?);
4903 }
4904 let sql = format!(
4905 "SELECT * FROM {source} WHERE {} != ? AND {}",
4906 quote_ident(&table.primary_key),
4907 index
4908 .columns
4909 .iter()
4910 .map(|column| format!("{} = ?", quote_ident(column)))
4911 .collect::<Vec<_>>()
4912 .join(" AND ")
4913 );
4914 let mut statement = self
4915 .conn
4916 .prepare_cached(&sql)
4917 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4918 let records = statement
4919 .query_map(rusqlite::params_from_iter(params), |record| {
4920 let values = table
4921 .columns
4922 .iter()
4923 .enumerate()
4924 .map(|(index, column)| {
4925 let value = sql_ref_to_json(column, record.get_ref(index)?);
4926 json_to_column_value(column, Some(&value)).map_err(|message| {
4927 rusqlite::Error::ToSqlConversionFailure(message.into())
4928 })
4929 })
4930 .collect::<Result<Row, _>>()?;
4931 Ok((values, record.get::<_, i64>(table.columns.len())?))
4932 })
4933 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4934 for record in records {
4935 let (row, version) =
4936 record.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
4937 let row_id = render_row_id_json(Some(&column_value_to_json(&row[table.pk_index])))?;
4938 conflicts.push(RetainedUniqueConflict {
4939 index: index.name.clone(),
4940 columns: index.columns.clone(),
4941 row_id,
4942 server_row: table
4943 .columns
4944 .iter()
4945 .zip(&row)
4946 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
4947 .collect(),
4948 server_version: version,
4949 });
4950 }
4951 }
4952 Ok(conflicts)
4953 }
4954
4955 fn attach_retained_rows(&self, mut outcome: CommitOutcome) -> Result<CommitOutcome, String> {
4956 if let Some((commit, initial)) = self
4957 .failed_commits
4958 .iter()
4959 .find(|(commit, _)| commit.client_commit_id == outcome.client_commit_id)
4960 {
4961 let mut rows = Vec::new();
4962 for operation in &commit.ops {
4963 let Some(table) = self.schema.table(&operation.table) else {
4965 continue;
4966 };
4967 let key =
4968 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
4969 .map_err(|_| "sync.local_corrupt".to_owned())?;
4970 let base = self.stored_row(table, &operation.row_id, true)?;
4971 rows.push(RetainedCommitRow {
4972 table: operation.table.clone(),
4973 row_id: operation.row_id.clone(),
4974 unique_conflicts: if operation.upsert {
4975 self.unique_conflicts(
4976 table,
4977 initial
4978 .get(&key)
4979 .and_then(Value::as_object)
4980 .or(operation.values.as_ref())
4981 .ok_or("sync.local_corrupt")?,
4982 true,
4983 )?
4984 } else {
4985 Vec::new()
4986 },
4987 local_row: if operation.upsert {
4988 initial
4989 .get(&key)
4990 .and_then(Value::as_object)
4991 .cloned()
4992 .or_else(|| operation.values.clone())
4993 } else {
4994 None
4995 },
4996 server_row: base.as_ref().map(|(values, _)| {
4997 table
4998 .columns
4999 .iter()
5000 .zip(values)
5001 .map(|(column, value)| {
5002 (column.name.clone(), column_value_to_json(value))
5003 })
5004 .collect()
5005 }),
5006 server_version: base.map(|(_, version)| version),
5007 });
5008 }
5009 outcome.retained_rows = Some(rows);
5010 }
5011 Ok(outcome)
5012 }
5013
5014 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
5015 let row = self
5016 .conn
5017 .query_row(
5018 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5019 resolution, resolved_at_ms, replacement_client_commit_id
5020 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
5021 rusqlite::params![client_commit_id],
5022 |row| {
5023 Ok(StoredCommitOutcomeRow {
5024 sequence: row.get(0)?,
5025 client_commit_id: row.get(1)?,
5026 status: row.get(2)?,
5027 recorded_at_ms: row.get(3)?,
5028 results_json: row.get(4)?,
5029 operations_json: row.get(5)?,
5030 resolution: row.get(6)?,
5031 resolved_at_ms: row.get(7)?,
5032 replacement_client_commit_id: row.get(8)?,
5033 })
5034 },
5035 )
5036 .optional()
5037 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5038 row.map(|row| self.attach_retained_rows(Self::outcome_from_row(row)?))
5039 .transpose()
5040 }
5041
5042 #[doc(hidden)]
5044 pub fn prepare_current_base(
5045 &self,
5046 client_commit_id: &str,
5047 sequence: i64,
5048 ) -> Result<CurrentBasePreparation, CurrentBaseProof> {
5049 let outcome = self
5050 .commit_outcome(client_commit_id)
5051 .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?;
5052 let valid = outcome.as_ref().is_some_and(|outcome| {
5053 outcome.sequence == sequence
5054 && outcome.resolution == CommitOutcomeResolution::Active
5055 && matches!(
5056 outcome.status,
5057 CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5058 )
5059 && outcome
5060 .operations
5061 .as_ref()
5062 .zip(outcome.retained_rows.as_ref())
5063 .is_some_and(|(operations, rows)| {
5064 !rows.is_empty() && operations.len() == rows.len()
5065 })
5066 });
5067 if !valid {
5068 return Err(CurrentBaseProof::failure(
5069 CurrentBaseProofReason::OutcomeUnavailable,
5070 ));
5071 }
5072 if !self.transport_enabled {
5073 return Err(CurrentBaseProof::failure(CurrentBaseProofReason::Offline));
5074 }
5075 Ok(CurrentBasePreparation {
5076 client_commit_id: client_commit_id.to_owned(),
5077 sequence,
5078 units: load_units(&self.conn)
5079 .map_err(|_| CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed))?,
5080 transport_epoch: self.transport_epoch,
5081 registration_epoch: self.registration_epoch,
5082 security_epoch: self.security_epoch,
5083 })
5084 }
5085
5086 #[doc(hidden)]
5088 pub fn finish_current_base(
5089 &mut self,
5090 prepared: CurrentBasePreparation,
5091 drive: SyncOutcome,
5092 ) -> CurrentBaseProof {
5093 if !self.transport_enabled || self.transport_epoch != prepared.transport_epoch {
5094 return CurrentBaseProof::failure(CurrentBaseProofReason::Offline);
5095 }
5096 if self.registration_epoch != prepared.registration_epoch {
5097 return CurrentBaseProof::failure(CurrentBaseProofReason::MissingCoverage);
5098 }
5099 if self.security_preflight || self.security_epoch != prepared.security_epoch {
5100 return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed);
5101 }
5102 let report = match drive {
5103 SyncOutcome::Ok(report) => report,
5104 SyncOutcome::BudgetExhausted(_) => {
5105 return CurrentBaseProof::failure(CurrentBaseProofReason::BudgetExhausted)
5106 }
5107 _ => return CurrentBaseProof::failure(CurrentBaseProofReason::SyncFailed),
5108 };
5109 if report.schema_floor.is_some() {
5110 return CurrentBaseProof::failure(CurrentBaseProofReason::SchemaFloor);
5111 }
5112 if self
5113 .conn
5114 .execute_batch("SAVEPOINT syncular_base_proof")
5115 .is_err()
5116 {
5117 return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5118 }
5119 let proof = (|| {
5120 let snapshot = snapshot_read_connection(
5121 &self.conn,
5122 &SnapshotReadRequest {
5123 units: prepared.units.iter().map(|unit| unit.id.clone()).collect(),
5124 commit_ids: vec![prepared.client_commit_id.clone()],
5125 ..Default::default()
5126 },
5127 )
5128 .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5129 let current = self
5130 .commit_outcome(&prepared.client_commit_id)
5131 .map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5132 let unavailable = if report.revoked.is_empty() {
5133 CurrentBaseProofReason::OutcomeUnavailable
5134 } else {
5135 CurrentBaseProofReason::Revoked
5136 };
5137 let Some(outcome) = current else {
5138 return Err(unavailable);
5139 };
5140 if !matches!(
5141 snapshot.deliveries.first(),
5142 Some(CommitDelivery::Known { .. })
5143 ) || outcome.sequence != prepared.sequence
5144 || outcome.resolution != CommitOutcomeResolution::Active
5145 || !matches!(
5146 outcome.status,
5147 CommitOutcomeStatus::Conflict | CommitOutcomeStatus::Rejected
5148 )
5149 {
5150 return Err(unavailable);
5151 }
5152 let (Some(operations), Some(rows)) = (outcome.operations, outcome.retained_rows) else {
5153 return Err(unavailable);
5154 };
5155 if rows.is_empty() || operations.len() != rows.len() {
5156 return Err(unavailable);
5157 }
5158 let after =
5159 load_units(&self.conn).map_err(|_| CurrentBaseProofReason::SnapshotFailed)?;
5160 let complete: HashSet<&str> = snapshot
5161 .units
5162 .iter()
5163 .filter_map(|unit| match unit {
5164 UnitCatchup::Known {
5165 id,
5166 bootstrap_complete: true,
5167 ..
5168 } => Some(id.as_str()),
5169 _ => None,
5170 })
5171 .collect();
5172 let candidates: Vec<&UnitRecord> = after
5175 .iter()
5176 .filter(|unit| {
5177 complete.contains(unit.id.as_str())
5178 && prepared.units.iter().any(|before| before.id == unit.id)
5179 })
5180 .collect();
5181 let covered = |table_name: &str, row: &Map<String, Value>| {
5182 let Some(table) = self.schema.table(table_name) else {
5183 return false;
5184 };
5185 candidates.iter().any(|unit| {
5186 covers(table, &unit.scopes)
5187 && pair_scopes(&unit.scopes, table_name, unit.echo.as_deref()).is_some_and(
5188 |scopes| {
5189 scopes.iter().all(|(variable, allowed)| {
5190 table
5191 .scope_column(variable)
5192 .and_then(|column| match row.get(column) {
5193 Some(Value::String(value)) => Some(value.clone()),
5194 Some(Value::Bool(value)) => Some(value.to_string()),
5195 Some(Value::Number(value)) => {
5196 value.as_f64().map(|number| {
5197 ryu_js::Buffer::new()
5198 .format_finite(number)
5199 .to_owned()
5200 })
5201 }
5202 _ => None,
5203 })
5204 .is_some_and(|value| {
5205 !value.is_empty() && allowed.contains(&value)
5206 })
5207 })
5208 },
5209 )
5210 })
5211 };
5212 for row in &rows {
5213 let Some(base) = row.server_row.as_ref().or(row.local_row.as_ref()) else {
5214 return Err(if report.revoked.is_empty() {
5215 CurrentBaseProofReason::MissingCoverage
5216 } else {
5217 CurrentBaseProofReason::Revoked
5218 });
5219 };
5220 if !covered(&row.table, base)
5221 || row
5222 .unique_conflicts
5223 .iter()
5224 .any(|conflict| !covered(&row.table, &conflict.server_row))
5225 {
5226 return Err(if report.revoked.is_empty() {
5227 CurrentBaseProofReason::MissingCoverage
5228 } else {
5229 CurrentBaseProofReason::Revoked
5230 });
5231 }
5232 }
5233 Ok(rows)
5234 })();
5235 if self
5236 .conn
5237 .execute_batch("RELEASE syncular_base_proof")
5238 .is_err()
5239 {
5240 return CurrentBaseProof::failure(CurrentBaseProofReason::SnapshotFailed);
5241 }
5242 match proof {
5243 Ok(rows) => CurrentBaseProof {
5244 proven: true,
5245 rows: Some(rows),
5246 reason: None,
5247 },
5248 Err(reason) => CurrentBaseProof::failure(reason),
5249 }
5250 }
5251
5252 pub fn prove_current_base(
5253 &mut self,
5254 transport: &mut dyn Transport,
5255 client_commit_id: &str,
5256 sequence: i64,
5257 max_rounds: Option<u32>,
5258 ) -> CurrentBaseProof {
5259 let prepared = match self.prepare_current_base(client_commit_id, sequence) {
5260 Ok(prepared) => prepared,
5261 Err(proof) => return proof,
5262 };
5263 let drive = self.sync_until_idle(transport, max_rounds);
5264 self.finish_current_base(prepared, drive)
5265 }
5266
5267 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
5268 if query.limit == Some(0) {
5269 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
5270 }
5271 let mut sql = String::from(
5272 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
5273 resolution, resolved_at_ms, replacement_client_commit_id
5274 FROM _syncular_commit_outcomes",
5275 );
5276 if query.active_only {
5277 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
5278 }
5279 sql.push_str(" ORDER BY seq DESC");
5280 if let Some(limit) = query.limit {
5281 sql.push_str(&format!(" LIMIT {limit}"));
5282 }
5283 let mut stmt = self
5284 .conn
5285 .prepare(&sql)
5286 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5287 let rows = stmt
5288 .query_map([], |row| {
5289 Ok(StoredCommitOutcomeRow {
5290 sequence: row.get(0)?,
5291 client_commit_id: row.get(1)?,
5292 status: row.get(2)?,
5293 recorded_at_ms: row.get(3)?,
5294 results_json: row.get(4)?,
5295 operations_json: row.get(5)?,
5296 resolution: row.get(6)?,
5297 resolved_at_ms: row.get(7)?,
5298 replacement_client_commit_id: row.get(8)?,
5299 })
5300 })
5301 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5302 let mut outcomes = Vec::new();
5303 for row in rows {
5304 outcomes.push(self.attach_retained_rows(Self::outcome_from_row(
5305 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
5306 )?)?);
5307 }
5308 Ok(outcomes)
5309 }
5310
5311 fn prune_commit_outcomes(&self) -> Result<(), String> {
5312 #[cfg(test)]
5313 self.outcome_prune_count
5314 .set(self.outcome_prune_count.get() + 1);
5315 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
5316 let count = self
5317 .conn
5318 .query_row(
5319 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
5320 [],
5321 |row| row.get::<_, i64>(0),
5322 )
5323 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5324 as usize;
5325 let excess = count.saturating_sub(max_entries);
5326 if excess == 0 {
5327 return Ok(());
5328 }
5329 let mut stmt = self
5330 .conn
5331 .prepare(
5332 "SELECT seq FROM _syncular_commit_outcomes
5333 WHERE status IN ('applied', 'cached') OR resolution != 'active'
5334 ORDER BY seq ASC LIMIT ?1",
5335 )
5336 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5337 let rows = stmt
5338 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
5339 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5340 let sequences = rows
5341 .collect::<Result<Vec<_>, _>>()
5342 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5343 drop(stmt);
5344 for sequence in sequences {
5345 self.conn
5346 .execute(
5347 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
5348 rusqlite::params![sequence],
5349 )
5350 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5351 }
5352 Ok(())
5353 }
5354
5355 pub fn units(&self) -> Result<Vec<UnitRecord>, String> {
5359 self.stored(load_units(&self.conn))
5360 }
5361
5362 pub fn unit(&self, id: &str) -> Result<Option<UnitRecord>, String> {
5363 self.stored(get_unit(&self.conn, id))
5364 }
5365
5366 pub fn subscribe(&mut self, scopes: UnitScopes) -> Result<String, String> {
5368 Ok(self.subscribe_many(vec![scopes])?.remove(0))
5369 }
5370
5371 pub fn subscribe_many(&mut self, units: Vec<UnitScopes>) -> Result<Vec<String>, String> {
5378 self.begin_observation("syncular_subscribe")?;
5379 let mut changed = false;
5380 let registered = (|| {
5381 let mut ids = Vec::with_capacity(units.len());
5382 for scopes in units {
5383 let (id, held) = self.hold_unit(scopes, true)?;
5384 changed |= held;
5385 ids.push(id);
5386 }
5387 self.conn
5388 .execute_batch("RELEASE syncular_subscribe")
5389 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5390 Ok(ids)
5391 })();
5392 let ids = match registered {
5393 Ok(ids) => ids,
5394 Err(error) => {
5395 self.rollback_observation("syncular_subscribe");
5396 return Err(error);
5397 }
5398 };
5399 if changed {
5400 self.registration_epoch = self.registration_epoch.wrapping_add(1);
5401 self.set_sync_needed(true, true);
5402 }
5403 Ok(ids)
5404 }
5405
5406 pub fn unsubscribe(&mut self, id: &str) -> Result<(), String> {
5409 let Some(unit) = self.stored(get_unit(&self.conn, id))? else {
5410 return Ok(());
5411 };
5412 if !unit.explicit {
5413 return Ok(());
5414 }
5415 if !self.stored(window_refs(&self.conn, id))?.is_empty() {
5416 return self.stored(save_unit(
5417 &self.conn,
5418 &UnitRecord {
5419 explicit: false,
5420 ..unit
5421 },
5422 ));
5423 }
5424 self.conn
5425 .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5426 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5427 self.registration_epoch = self.registration_epoch.wrapping_add(1);
5428 if unit.status == UnitStatus::Active {
5429 self.set_sync_needed(true, true);
5430 }
5431 Ok(())
5432 }
5433
5434 fn hold_unit(&mut self, scopes: UnitScopes, explicit: bool) -> Result<(String, bool), String> {
5438 if scopes.is_empty()
5439 || scopes.values().any(|value| value == "*")
5440 || !self
5441 .schema
5442 .tables
5443 .iter()
5444 .any(|table| covers(table, &scopes))
5445 {
5446 return Err("sync.invalid_subscription: a unit names one non-wildcard value per variable and covers a table".to_owned());
5447 }
5448 let id = ssp2::registration::unit_id(&scope_pairs(&scopes));
5449 let existing = self.stored(get_unit(&self.conn, &id))?;
5450 if let Some(existing) = existing
5451 .as_ref()
5452 .filter(|unit| unit.status != UnitStatus::Revoked)
5453 {
5454 if explicit && !existing.explicit {
5455 self.stored(save_unit(
5456 &self.conn,
5457 &UnitRecord {
5458 explicit,
5459 ..existing.clone()
5460 },
5461 ))?;
5462 }
5463 return Ok((id, false));
5464 }
5465 self.delete_pending_evict(&id)?;
5466 self.stored(save_unit(
5467 &self.conn,
5468 &UnitRecord {
5469 id: id.clone(),
5470 scopes,
5471 explicit: explicit || existing.is_some_and(|unit| unit.explicit),
5472 status: UnitStatus::Active,
5473 reason_code: None,
5474 cursor: Some(-1),
5475 bootstrap_state: None,
5476 seeded: Vec::new(),
5477 echo: None,
5478 },
5479 ))?;
5480 Ok((id, true))
5481 }
5482
5483 pub fn set_window(
5491 &mut self,
5492 base: &WindowBase,
5493 units: &[String],
5494 ) -> Result<CommandEffects, String> {
5495 let base_key = window_base_key(base);
5496 let live = self.load_window_units(&base_key)?;
5497 let mut changed = false;
5498 for value in units {
5499 if live.iter().any(|(held, _)| held == value) {
5500 continue;
5501 }
5502 let rollback = self.capture();
5503 self.begin_observation("syncular_window")?;
5504 let widened = (|| {
5505 let (id, registered) = self.hold_unit(window_unit_scopes(base, value), false)?;
5506 self.sync_needed |= registered;
5507 self.conn
5508 .execute(
5509 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
5510 rusqlite::params![base_key, value, id],
5511 )
5512 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5513 let mut batch = ChangeAccumulator::default();
5514 batch.window(&base_key, value);
5515 batch.status = true;
5516 self.finish_observation("syncular_window", batch)
5517 })();
5518 if let Err(error) = widened {
5519 self.rollback_observation("syncular_window");
5520 self.restore(rollback);
5521 return Err(error);
5522 }
5523 changed = true;
5524 self.registration_epoch = self.registration_epoch.wrapping_add(1);
5525 }
5526 for (value, id) in live {
5527 if units.contains(&value) {
5528 continue;
5529 }
5530 self.rebuild_overlay_if_dirty()?;
5531 let rollback = self.capture();
5532 self.begin_observation("syncular_window")?;
5533 let remaining = self.release_window_unit(&base_key, &value, &id);
5534 let remaining = match remaining {
5535 Ok(remaining) => remaining,
5536 Err(error) => {
5537 self.rollback_observation("syncular_window");
5538 self.restore(rollback);
5539 return Err(error);
5540 }
5541 };
5542 changed = true;
5543 self.registration_epoch = self.registration_epoch.wrapping_add(1);
5544 if remaining {
5545 self.drain_pending_evictions()?;
5546 }
5547 }
5548 Ok(if changed && self.transport_enabled {
5549 CommandEffects::interactive()
5550 } else {
5551 CommandEffects::none()
5552 })
5553 }
5554
5555 pub fn window_state(&self, base: &WindowBase) -> Result<WindowState, String> {
5559 persisted_window_state(&self.conn, base).map_err(|failure| failure.message)
5560 }
5561
5562 fn load_window_units(&self, base_key: &str) -> Result<Vec<(String, String)>, String> {
5564 let mut statement = self
5565 .conn
5566 .prepare_cached(
5567 "SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC",
5568 )
5569 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5570 let rows = statement
5571 .query_map([base_key], |row| Ok((row.get(0)?, row.get(1)?)))
5572 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5573 rows.collect::<Result<_, _>>()
5574 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5575 }
5576
5577 fn load_window_registry(&self) -> Result<Vec<(String, String)>, String> {
5579 let mut statement = self
5580 .conn
5581 .prepare_cached("SELECT base, unit FROM _syncular_windows ORDER BY base, unit")
5582 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5583 let rows = statement
5584 .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
5585 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5586 rows.collect::<Result<_, _>>()
5587 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5588 }
5589
5590 fn release_window_unit(
5598 &mut self,
5599 base_key: &str,
5600 value: &str,
5601 id: &str,
5602 ) -> Result<bool, String> {
5603 self.conn
5604 .execute(
5605 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
5606 rusqlite::params![base_key, value],
5607 )
5608 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5609 let mut batch = ChangeAccumulator::default();
5610 batch.window(base_key, value);
5611 batch.status = true;
5612 let unit = self.stored(get_unit(&self.conn, id))?;
5613 let mut remaining = false;
5614 if let Some(unit) = unit.filter(|unit| !unit.explicit) {
5615 if self.stored(window_refs(&self.conn, id))?.is_empty() {
5616 let units = self.stored(load_units(&self.conn))?;
5617 self.conn
5618 .execute("DELETE FROM _syncular_units WHERE id = ?1", [id])
5619 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5620 self.sync_needed = true;
5621 for table in self.schema.tables.clone() {
5622 let Some(effective) = unit
5623 .echo
5624 .as_deref()
5625 .filter(|_| covers(&table, &unit.scopes))
5626 .and_then(|echo| pair_scopes(&unit.scopes, &table.name, Some(echo)))
5627 else {
5628 continue;
5629 };
5630 let pinned = self.pinned_row_ids(&table.name);
5631 let owners = unit_owners(&units, &table, id);
5632 let (more, deferred) =
5633 self.evict_scope_rows(&table.name, &effective, &owners, &pinned)?;
5634 remaining |= more;
5635 if deferred {
5636 self.save_pending_evict(id, &table.name, &effective)?;
5637 }
5638 self.record_scope_map(&mut batch, &table.name, &effective);
5639 }
5640 }
5641 }
5642 self.finish_observation("syncular_window", batch)?;
5643 Ok(remaining)
5644 }
5645
5646 fn evict_scope_rows(
5651 &self,
5652 table_name: &str,
5653 effective: &[(String, Vec<String>)],
5654 owners: &[Vec<(String, Vec<String>)>],
5655 pinned: &std::collections::HashSet<String>,
5656 ) -> Result<(bool, bool), String> {
5657 let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
5658 let Some((scope, mut params)) = local_scope_predicate(table, effective, owners)
5659 .map_err(|()| "sync.scope_revoked".to_owned())?
5660 else {
5661 return Ok((false, false));
5662 };
5663 let scope_params = params.len();
5664 let pk = quote_ident(&table.primary_key);
5665 let predicate = if pinned.is_empty() {
5667 scope.clone()
5668 } else {
5669 params.extend(pinned.iter().cloned().map(SqlValue::Text));
5670 let holes = vec!["?"; pinned.len()].join(", ");
5671 format!("{scope} AND CAST({pk} AS TEXT) NOT IN ({holes})")
5672 };
5673 let base = base_table(table_name);
5674 let visible = visible_table(table_name);
5675 let mut ids: Vec<SqlValue> = Vec::new();
5676 for source in [&base, &visible] {
5677 let mut stmt = self
5678 .conn
5679 .prepare_cached(&format!(
5680 "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
5681 ))
5682 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5683 ids = stmt
5684 .query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
5685 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
5686 .collect::<Result<_, _>>()
5687 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5688 if !ids.is_empty() {
5689 break;
5690 }
5691 }
5692 if !ids.is_empty() {
5693 for id in &ids {
5694 self.conn
5695 .execute(
5696 "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2",
5697 rusqlite::params![table_name, id],
5698 )
5699 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5700 }
5701 let holes = vec!["?"; ids.len()].join(", ");
5702 for source in [&base, &visible] {
5703 self.conn
5704 .execute(
5705 &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
5706 rusqlite::params_from_iter(&ids),
5707 )
5708 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5709 }
5710 }
5711 let mut remaining = false;
5712 let mut deferred = false;
5713 for source in [&base, &visible] {
5714 remaining |= self
5715 .conn
5716 .query_row(
5717 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
5718 rusqlite::params_from_iter(¶ms),
5719 |row| row.get::<_, bool>(0),
5720 )
5721 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5722 deferred |= self
5723 .conn
5724 .query_row(
5725 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
5726 rusqlite::params_from_iter(¶ms[..scope_params]),
5727 |row| row.get::<_, bool>(0),
5728 )
5729 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5730 }
5731 Ok((remaining, deferred))
5732 }
5733
5734 fn drain_pending_evictions(&mut self) -> Result<(), String> {
5738 let pending = self.load_pending_evictions()?;
5739 if pending.is_empty() {
5740 return Ok(());
5741 }
5742 self.rebuild_overlay_if_dirty()?;
5743 for (unit_id, table_name, effective) in pending {
5744 let Some(table) = self.schema.table(&table_name).cloned() else {
5745 self.delete_pending_evict_table(&unit_id, &table_name)?;
5746 continue;
5747 };
5748 loop {
5749 std::thread::yield_now();
5750 self.begin_observation("syncular_evict")?;
5751 let chunk = (|| {
5752 let pinned = self.pinned_row_ids(&table_name);
5753 let owners =
5754 unit_owners(&self.stored(load_units(&self.conn))?, &table, &unit_id);
5755 let (remaining, deferred) =
5756 self.evict_scope_rows(&table_name, &effective, &owners, &pinned)?;
5757 if !deferred {
5758 self.delete_pending_evict_table(&unit_id, &table_name)?;
5759 }
5760 let mut batch = ChangeAccumulator::default();
5761 self.record_scope_map(&mut batch, &table_name, &effective);
5762 self.finish_observation("syncular_evict", batch)?;
5763 Ok(remaining)
5764 })();
5765 match chunk {
5766 Ok(true) => {}
5767 Ok(false) => break,
5768 Err(error) => {
5769 self.rollback_observation("syncular_evict");
5770 return Err(error);
5771 }
5772 }
5773 }
5774 }
5775 Ok(())
5776 }
5777
5778 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
5781 let mut pinned = std::collections::HashSet::new();
5782 for commit in self
5783 .outbox
5784 .iter()
5785 .map(|commit| &**commit)
5786 .chain(self.failed_commits.iter().map(|(commit, _)| commit))
5787 {
5788 for op in &commit.ops {
5789 if op.table == table {
5790 pinned.insert(op.row_id.clone());
5791 }
5792 }
5793 }
5794 pinned
5795 }
5796
5797 fn save_pending_evict(
5798 &self,
5799 unit_id: &str,
5800 table: &str,
5801 effective: &[(String, Vec<String>)],
5802 ) -> Result<(), String> {
5803 self.conn.execute(
5804 "INSERT OR REPLACE INTO _syncular_pending_evictions(unit_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
5805 rusqlite::params![unit_id, table, scope_map_to_json(effective).to_string()],
5806 ).map(|_| ()).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5807 }
5808
5809 fn delete_pending_evict(&self, unit_id: &str) -> Result<(), String> {
5811 self.conn
5812 .execute(
5813 "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1",
5814 [unit_id],
5815 )
5816 .map(|_| ())
5817 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5818 }
5819
5820 fn delete_pending_evict_table(&self, unit_id: &str, table: &str) -> Result<(), String> {
5821 self.conn
5822 .execute(
5823 "DELETE FROM _syncular_pending_evictions WHERE unit_id = ?1 AND tbl = ?2",
5824 [unit_id, table],
5825 )
5826 .map(|_| ())
5827 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
5828 }
5829
5830 fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
5831 let mut stmt = self
5832 .conn
5833 .prepare("SELECT unit_id, tbl, effective_scopes FROM _syncular_pending_evictions")
5834 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5835 let rows = stmt
5836 .query_map([], |row| {
5837 Ok((
5838 row.get::<_, String>(0)?,
5839 row.get::<_, String>(1)?,
5840 row.get::<_, String>(2)?,
5841 ))
5842 })
5843 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5844 rows.map(|row| {
5845 let (unit_id, table, json) =
5846 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5847 let value: Value = serde_json::from_str(&json)
5848 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5849 let effective = json_to_scope_map(&value)
5850 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
5851 Ok((unit_id, table, effective))
5852 })
5853 .collect()
5854 }
5855
5856 pub fn set_retain_failed_commits(&mut self, enabled: bool) -> Result<(), String> {
5858 self.retain_failed_commits = enabled;
5859 for (commit, _) in self.failed_commits.iter() {
5860 for operation in &commit.ops {
5861 self.overlay_dirty.table(&operation.table);
5862 }
5863 }
5864 self.failed_commits = self.load_failed_commits()?.into();
5865 for (commit, _) in self.failed_commits.iter() {
5866 for operation in &commit.ops {
5867 self.overlay_dirty.table(&operation.table);
5868 }
5869 }
5870 Ok(())
5871 }
5872
5873 fn load_failed_commits(&self) -> Result<Vec<FailedCommit>, String> {
5874 let mut statement = self.conn.prepare("SELECT client_commit_id, operations_json, initial_json FROM _syncular_failed_commits ORDER BY seq")
5875 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5876 let rows = statement
5877 .query_map([], |row| {
5878 Ok((
5879 row.get::<_, String>(0)?,
5880 row.get::<_, String>(1)?,
5881 row.get::<_, String>(2)?,
5882 ))
5883 })
5884 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5885 rows.map(|row| {
5886 let (id, operations, initial) =
5887 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5888 let operations: Vec<CommitOperation> =
5889 serde_json::from_str(&operations).map_err(|_| "sync.local_corrupt".to_owned())?;
5890 let initial =
5891 serde_json::from_str(&initial).map_err(|_| "sync.local_corrupt".to_owned())?;
5892 Ok((
5893 OutboxCommit {
5894 client_commit_id: id,
5895 ops: operations
5896 .into_iter()
5897 .map(|op| OutboxOp {
5898 upsert: op.op == "upsert",
5899 table: op.table,
5900 row_id: op.row_id,
5901 base_version: op.base_version,
5902 values: op.values,
5903 insert: op.insert,
5904 })
5905 .collect(),
5906 },
5907 initial,
5908 ))
5909 })
5910 .collect()
5911 }
5912
5913 fn retain_failed_commit(&mut self, id: &str, operations: &[OutboxOp]) -> Result<(), String> {
5914 let mut initial = Map::new();
5915 for operation in operations {
5916 let Some(table) = self.schema.table(&operation.table) else {
5918 continue;
5919 };
5920 let initial_row = match self.stored_row(table, &operation.row_id, true)? {
5921 Some(row) => Some(row),
5922 None => self.visible_row(table, &operation.row_id)?,
5923 };
5924 if let Some((values, _)) = initial_row {
5925 let mut intended: Map<String, Value> = table
5926 .columns
5927 .iter()
5928 .zip(values.iter())
5929 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
5930 .collect();
5931 if let Some(patch) = &operation.values {
5932 intended.extend(patch.clone());
5933 }
5934 initial.insert(
5935 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5936 .map_err(|_| "sync.local_corrupt".to_owned())?,
5937 Value::Object(intended),
5938 );
5939 }
5940 }
5941 self.conn.execute("INSERT INTO _syncular_failed_commits(client_commit_id, operations_json, initial_json) VALUES (?1, ?2, ?3)", rusqlite::params![id,
5942 serde_json::to_string(&operations.iter().map(CommitOperation::from).collect::<Vec<_>>()).map_err(|_| "sync.local_corrupt".to_owned())?,
5943 serde_json::to_string(&initial).map_err(|_| "sync.local_corrupt".to_owned())?])
5944 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5945 self.failed_commits.push((
5946 OutboxCommit {
5947 client_commit_id: id.to_owned(),
5948 ops: operations.to_vec(),
5949 },
5950 initial,
5951 ));
5952 Ok(())
5953 }
5954
5955 fn drop_failed_commits(
5956 &mut self,
5957 matches: impl Fn(&str, &Map<String, Value>) -> bool,
5958 ) -> Result<Vec<OutboxCommit>, String> {
5959 let mut doomed = self.drop_acknowledged_rows(&matches)?;
5960 for (commit, initial) in self.failed_commits.iter() {
5961 for operation in &commit.ops {
5962 let key =
5963 serde_json::to_string(&(operation.table.clone(), operation.row_id.clone()))
5964 .map_err(|_| "sync.local_corrupt".to_owned())?;
5965 if initial
5966 .get(&key)
5967 .and_then(Value::as_object)
5968 .is_some_and(|values| matches(&operation.table, values))
5969 {
5970 doomed.push(commit.clone());
5971 break;
5972 }
5973 }
5974 }
5975 for commit in &doomed {
5976 self.conn
5977 .execute(
5978 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
5979 rusqlite::params![commit.client_commit_id],
5980 )
5981 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5982 }
5983 for commit in &doomed {
5984 self.conn.execute("UPDATE _syncular_commit_outcomes SET operations_json = NULL, results_json = '[]' WHERE client_commit_id = ?1", rusqlite::params![commit.client_commit_id]).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
5985 }
5986 self.conflicts.retain(|record| {
5987 !doomed
5988 .iter()
5989 .any(|commit| commit.client_commit_id == record.client_commit_id)
5990 });
5991 self.rejections.retain(|record| {
5992 !doomed
5993 .iter()
5994 .any(|commit| commit.client_commit_id == record.client_commit_id)
5995 });
5996 self.failed_commits.retain(|(commit, _)| {
5997 !doomed
5998 .iter()
5999 .any(|dropped| dropped.client_commit_id == commit.client_commit_id)
6000 });
6001 for commit in &doomed {
6002 for operation in &commit.ops {
6003 self.overlay_dirty.table(&operation.table);
6004 }
6005 }
6006 Ok(doomed)
6007 }
6008
6009 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, ClientError> {
6013 self.mutate_with_commit_id(mutations, None)
6014 }
6015
6016 pub fn mutate_with_commit_id(
6020 &mut self,
6021 mutations: Vec<Mutation>,
6022 client_commit_id: Option<String>,
6023 ) -> Result<String, ClientError> {
6024 let prior = self.storage_failure.borrow_mut().take();
6025 if let Some(id) = &client_commit_id {
6026 let held = if id.is_empty() {
6027 Err("sync.invalid_request: clientCommitId is empty".to_owned())
6028 } else {
6029 self.conn
6030 .query_row(
6031 "SELECT EXISTS(SELECT 1 FROM _syncular_outbox WHERE commit_id=?1 UNION ALL SELECT 1 FROM _syncular_commit_outcomes WHERE client_commit_id=?1)",
6032 [id],
6033 |row| row.get(0),
6034 )
6035 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
6036 };
6037 match held {
6038 Ok(false) => {}
6039 Ok(true) => {
6040 return self.finish_authoring(
6041 prior,
6042 Err(ClientError {
6043 code: "client.commit_id_reused".to_owned(),
6044 message: "clientCommitId is held".to_owned(),
6045 details: Some(json!({ "clientCommitId": id })),
6046 retryable: false,
6047 }),
6048 )
6049 }
6050 Err(message) => return self.finish_authoring(prior, Err(message.into())),
6051 }
6052 }
6053 let outcome = (|| -> Result<String, ClientError> {
6054 if mutations.is_empty() {
6055 return Err(
6056 "sync.invalid_request: a commit must contain at least one operation (§6.1)"
6057 .to_owned()
6058 .into(),
6059 );
6060 }
6061 let mut ops = Vec::with_capacity(mutations.len());
6062 for mutation in mutations {
6063 match mutation {
6064 Mutation::Patch {
6065 table,
6066 values,
6067 base_version,
6068 } => {
6069 let schema_table = self
6070 .schema
6071 .table(&table)
6072 .ok_or_else(|| "sync.unknown_table".to_owned())?;
6073 let values = normalize_values_casing(schema_table, values)?;
6074 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6075 ops.push(self.prepare_patch(&table, &row_id, values, base_version)?);
6076 }
6077 Mutation::Upsert {
6078 table,
6079 values,
6080 base_version,
6081 } => {
6082 let schema_table = self.schema.table(&table).ok_or_else(|| {
6083 format!("sync.unknown_table: unknown table {table:?}")
6084 })?;
6085 let values = normalize_values_casing(schema_table, values)?;
6089 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
6090 let values = full_row_values(schema_table, values)?;
6093 self.refuse_sealed_row(schema_table, &row_id)?;
6094 self.validate_author_encode(schema_table, &row_id, &values)?;
6100 let insert = !self.visible_row_exists(schema_table, &row_id)?;
6104 ops.push(OutboxOp {
6105 upsert: true,
6106 table,
6107 row_id,
6108 base_version,
6109 values: Some(values),
6110 insert,
6111 });
6112 }
6113 Mutation::Delete {
6114 table,
6115 row_id,
6116 base_version,
6117 } => {
6118 let schema_table = self.schema.table(&table).ok_or_else(|| {
6119 format!("sync.unknown_table: unknown table {table:?}")
6120 })?;
6121 self.refuse_sealed_row(schema_table, &row_id)?;
6122 if row_id.contains('\0') {
6124 return Err("sync.invalid_request: rowId holds U+0000 (§2.4)"
6125 .to_owned()
6126 .into());
6127 }
6128 ops.push(OutboxOp {
6129 upsert: false,
6130 table,
6131 row_id,
6132 base_version,
6133 values: None,
6134 insert: false,
6135 });
6136 }
6137 }
6138 }
6139 Ok(self.record_outbox_commit(ops, client_commit_id)?)
6140 })();
6141 self.finish_authoring(prior, outcome)
6142 }
6143
6144 fn finish_authoring(
6150 &self,
6151 prior: Option<QueryReadFailure>,
6152 outcome: Result<String, ClientError>,
6153 ) -> Result<String, ClientError> {
6154 let fresh = self.storage_failure.borrow_mut().take();
6155 *self.storage_failure.borrow_mut() = fresh.clone().or(prior);
6156 outcome.map_err(|error| fresh.map(ClientError::from).unwrap_or(error))
6157 }
6158
6159 fn record_outbox_commit(
6160 &mut self,
6161 ops: Vec<OutboxOp>,
6162 client_commit_id: Option<String>,
6163 ) -> Result<String, String> {
6164 let commit = OutboxCommit {
6165 client_commit_id: client_commit_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()),
6166 ops,
6167 };
6168 self.begin_observation("syncular_mutation")?;
6169 let mut batch = ChangeAccumulator::default();
6170 for op in &commit.ops {
6171 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
6172 if let Some(values) = &op.values {
6173 if let Some(table) = self.schema.table(&op.table) {
6174 for scope in &table.scope_variables {
6175 if let Some(Value::String(value)) = values.get(&scope.column) {
6176 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
6177 precise = true;
6178 }
6179 }
6180 }
6181 }
6182 if !precise {
6183 batch.table(&op.table);
6184 }
6185 }
6186 if let Err(error) = self.persist_outbox_insert(&commit) {
6187 self.rollback_observation("syncular_mutation");
6188 return Err(error);
6189 }
6190 let was_dirty = self.overlay_dirty.snapshot();
6196 let id = commit.client_commit_id.clone();
6197 let appended = self.rebuild_overlay_if_dirty().and_then(|()| {
6198 commit
6199 .ops
6200 .iter()
6201 .try_for_each(|op| self.apply_outbox_op(op))?;
6202 self.outbox.push(Arc::new(commit));
6203 batch.status = true;
6204 self.finish_observation("syncular_mutation", batch)
6205 .inspect_err(|_| drop(self.outbox.pop()))
6206 });
6207 if let Err(error) = appended {
6208 self.rollback_observation("syncular_mutation");
6209 self.overlay_dirty.restore(was_dirty);
6210 return Err(error);
6211 }
6212 Ok(id)
6213 }
6214
6215 pub fn patch(
6221 &mut self,
6222 table: &str,
6223 row_id: &str,
6224 partial: Map<String, Value>,
6225 base_version: Option<i64>,
6226 ) -> Result<String, ClientError> {
6227 let prior = self.storage_failure.borrow_mut().take();
6228 let outcome = self.perform_patch(table, row_id, partial, base_version);
6229 self.finish_authoring(prior, outcome)
6230 }
6231
6232 fn perform_patch(
6235 &mut self,
6236 table: &str,
6237 row_id: &str,
6238 partial: Map<String, Value>,
6239 base_version: Option<i64>,
6240 ) -> Result<String, ClientError> {
6241 let operation = self.prepare_patch(table, row_id, partial, base_version)?;
6242 Ok(self.record_outbox_commit(vec![operation], None)?)
6243 }
6244
6245 fn prepare_patch(
6246 &self,
6247 table: &str,
6248 row_id: &str,
6249 partial: Map<String, Value>,
6250 base_version: Option<i64>,
6251 ) -> Result<OutboxOp, ClientError> {
6252 let schema_table = self
6253 .schema
6254 .table(table)
6255 .ok_or_else(|| format!("sync.unknown_table: unknown table {table:?}"))?;
6256 let partial = normalize_values_casing(schema_table, partial)?;
6257 if let Some(pk) = partial.get(&schema_table.primary_key) {
6258 if render_row_id_json(Some(pk)).ok().as_deref() != Some(row_id) {
6259 return Err(invalid_authoring(
6260 "patch cannot change the primary key",
6261 json!({ "table": table }),
6262 ));
6263 }
6264 }
6265 if !self.visible_row_exists(schema_table, row_id)? {
6266 return Err("sync.row_missing: a sparse patch requires a local row"
6267 .to_owned()
6268 .into());
6269 }
6270 self.refuse_sealed_row(schema_table, row_id)?;
6271 for scope in &schema_table.scope_variables {
6281 if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
6282 continue;
6283 }
6284 let Some(column) = schema_table
6285 .columns
6286 .iter()
6287 .find(|column| column.name == scope.column)
6288 else {
6289 continue;
6290 };
6291 let stored = self
6292 .conn
6293 .query_row(
6294 &format!(
6295 "SELECT {} FROM {} WHERE {}",
6296 quote_ident(&scope.column),
6297 visible_table(&schema_table.name),
6298 row_id_predicate(schema_table)
6299 ),
6300 rusqlite::params![row_id],
6301 |row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
6302 )
6303 .optional()
6304 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6305 let incoming = json_to_column_value(column, partial.get(&scope.column))
6310 .ok()
6311 .map(|value| column_value_to_json(&value));
6312 let equal = stored.as_ref().is_some_and(|stored| {
6316 stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
6317 });
6318 if !equal {
6319 return Err(invalid_authoring(
6320 "patch cannot write a scope column (§3.4)",
6321 json!({ "table": table, "column": scope.column }),
6322 ));
6323 }
6324 }
6325 let mut values = partial;
6326 for scope in &schema_table.scope_variables {
6329 values.remove(&scope.column);
6330 }
6331 let pk_value = match schema_table.columns[schema_table.pk_index].ty {
6332 ColumnType::Integer => match row_id.parse::<i64>() {
6334 Ok(v)
6335 if v.to_string() == row_id
6336 && (-ssp2::primitives::I64_SAFE_MAX..=ssp2::primitives::I64_SAFE_MAX)
6337 .contains(&v) =>
6338 {
6339 Value::from(v)
6340 }
6341 _ => {
6342 return Err(format!(
6343 "sync.invalid_request: table {table:?}: invalid integer primary key {row_id:?}"
6344 )
6345 .into())
6346 }
6347 },
6348 ColumnType::Boolean => match row_id {
6349 "true" => Value::Bool(true),
6350 "false" => Value::Bool(false),
6351 _ => {
6352 return Err(format!(
6353 "sync.invalid_request: table {table:?}: invalid boolean primary key {row_id:?}"
6354 )
6355 .into())
6356 }
6357 },
6358 _ => Value::from(row_id.to_owned()),
6359 };
6360 values.insert(schema_table.primary_key.clone(), pk_value);
6361 if let Some(column) = schema_table
6363 .columns
6364 .iter()
6365 .find(|column| !column.nullable && values.get(&column.name) == Some(&Value::Null))
6366 {
6367 return Err(invalid_authoring(
6368 "a non-nullable column is null (§6.1)",
6369 json!({ "table": table, "column": column.name }),
6370 ));
6371 }
6372 self.validate_author_encode(schema_table, row_id, &values)?;
6378 Ok(OutboxOp {
6379 upsert: true,
6380 table: table.to_owned(),
6381 row_id: row_id.to_owned(),
6382 base_version,
6383 values: Some(values),
6384 insert: false,
6385 })
6386 }
6387
6388 fn stored_key_fallback(
6393 &self,
6394 table: &TableSchema,
6395 row_id: &str,
6396 ) -> Result<Option<Row>, String> {
6397 let Some(selector) = self.encryption.key_id_columns.get(&table.name) else {
6398 return Ok(None);
6399 };
6400 let Some(index) = table
6401 .columns
6402 .iter()
6403 .position(|column| &column.name == selector)
6404 else {
6405 return Ok(None);
6406 };
6407 let sql = format!(
6408 "SELECT {} FROM {} WHERE {}",
6409 quote_ident(selector),
6410 visible_table(&table.name),
6411 row_id_predicate(table)
6412 );
6413 let key = self
6414 .conn
6415 .query_row(&sql, rusqlite::params![row_id], |row| {
6416 row.get::<_, Option<String>>(0)
6417 })
6418 .optional()
6419 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6420 .flatten()
6421 .filter(|key| !key.is_empty());
6422 let Some(key) = key else {
6423 return Ok(None);
6424 };
6425 let mut fallback: Row = vec![None; table.columns.len()];
6426 fallback[index] = Some(ColumnValue::String(key));
6427 Ok(Some(fallback))
6428 }
6429
6430 fn validate_author_encode(
6435 &self,
6436 table: &TableSchema,
6437 row_id: &str,
6438 values: &Map<String, Value>,
6439 ) -> Result<(), String> {
6440 match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
6441 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
6442 result => result.map(|_| ()),
6443 }
6444 }
6445
6446 fn drop_unencodable_outbox(
6460 &mut self,
6461 with_encryption: bool,
6462 batch: Option<&mut ChangeAccumulator>,
6463 ) -> Result<bool, String> {
6464 let mut failures = Vec::new();
6465 for commit in self.outbox.iter() {
6466 for op in &commit.ops {
6467 let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6468 continue;
6469 };
6470 let Some(values) = op.presented_values(table) else {
6473 continue;
6474 };
6475 let fallback = self.stored_key_fallback(table, &op.row_id)?;
6476 let failure = match encode_sparse_row_json(
6477 table,
6478 &op.row_id,
6479 &values,
6480 &self.encryption,
6481 fallback.as_ref(),
6482 ) {
6483 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => {
6484 with_encryption.then_some((error, ENCRYPT_FAILED_CODE))
6485 }
6486 Err(error) if error.starts_with(INVALID_REQUEST_CODE) => Some((
6491 LEGACY_VALUE_REFUSAL_MESSAGE.to_owned(),
6492 OUTBOX_INCOMPATIBLE_CODE,
6493 )),
6494 _ => None,
6495 };
6496 if let Some((error, code)) = failure {
6497 failures.push((commit.clone(), error, code));
6498 break;
6499 }
6500 }
6501 }
6502 if failures.is_empty() {
6503 return Ok(false);
6504 }
6505 let rollback = self.capture();
6511 let owns_observation = batch.is_none();
6512 let owns_transaction = self.conn.is_autocommit();
6513 let mut domains = ChangeAccumulator::default();
6514 if owns_observation {
6515 self.begin_observation("syncular_value_recovery")?;
6516 }
6517 let dropped = (|| -> Result<(), String> {
6518 for (commit, message, code) in failures {
6519 if self.retain_failed_commits && code == ENCRYPT_FAILED_CODE {
6520 self.retain_failed_commit(&commit.client_commit_id, &commit.ops)?;
6521 }
6522 let representable = commit.ops.iter().all(Self::outbox_op_is_journalable);
6523 let rejection = RejectionRecord {
6524 client_commit_id: commit.client_commit_id.clone(),
6525 op_index: 0,
6526 code: code.to_owned(),
6527 message,
6528 retryable: false,
6529 details: (code == OUTBOX_INCOMPATIBLE_CODE).then(|| RejectionDetails {
6530 field_paths: None,
6531 reason: Some(INVALID_STORED_VALUES_REASON.to_owned()),
6532 required_action: None,
6533 references: None,
6534 }),
6535 operation: representable
6536 .then(|| commit.ops.first())
6537 .flatten()
6538 .map(CommitOperation::from),
6539 };
6540 self.persist_commit_outcome(
6541 &commit.client_commit_id,
6542 CommitOutcomeStatus::Rejected,
6543 &[CommitOperationOutcome::Error {
6544 rejection: rejection.clone(),
6545 }],
6546 representable.then_some(commit.ops.as_slice()),
6547 )?;
6548 self.delete_outbox_persisted(&commit.client_commit_id)?;
6549 self.outbox
6550 .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
6551 self.rejections.push(rejection);
6552 for operation in &commit.ops {
6553 domains.table(operation.table.as_str());
6554 }
6555 }
6556 domains.status = true;
6557 domains.rejections = true;
6558 domains.outcomes = true;
6559 self.prune_commit_outcomes()?;
6560 self.overlay_dirty.set(true);
6562 self.rebuild_overlay()?;
6563 Ok(())
6564 })();
6565 let result = match batch {
6566 Some(batch) => {
6567 if dropped.is_ok() {
6568 for table in domains.tables.keys() {
6569 batch.table(table.as_str());
6570 }
6571 batch.status |= domains.status;
6572 batch.rejections |= domains.rejections;
6573 batch.outcomes |= domains.outcomes;
6574 }
6575 dropped
6576 }
6577 None => {
6578 dropped.and_then(|()| self.finish_observation("syncular_value_recovery", domains))
6579 }
6580 };
6581 if let Err(error) = result {
6582 if owns_observation {
6583 if owns_transaction {
6584 if let Err(rollback) = self.conn.execute_batch("ROLLBACK") {
6585 let mut failure = self.storage_failure.borrow_mut();
6586 let failure =
6587 failure.get_or_insert_with(|| QueryReadFailure::from(error.clone()));
6588 failure.rollback_failure = Some(Box::new(QueryReadFailure::from(rollback)));
6589 }
6590 } else {
6591 self.rollback_observation("syncular_value_recovery");
6592 }
6593 }
6594 self.restore(rollback);
6595 return Err(error);
6596 }
6597 Ok(true)
6598 }
6599
6600 pub fn pending_commit_ids(&self) -> Vec<String> {
6601 self.outbox
6602 .iter()
6603 .map(|c| c.client_commit_id.clone())
6604 .collect()
6605 }
6606
6607 pub fn pending_payloads(&self) -> Result<Vec<Vec<u8>>, String> {
6611 let mut payloads = Vec::new();
6612 for commit in self.outbox.iter() {
6613 for op in &commit.ops {
6614 let Some(table) = self.schema.table(&op.table).filter(|_| op.upsert) else {
6615 continue;
6616 };
6617 let Some(values) = op.presented_values(table) else {
6618 continue;
6619 };
6620 let fallback = self.stored_key_fallback(table, &op.row_id)?;
6621 if let Ok(payload) = encode_sparse_row_json(
6622 table,
6623 &op.row_id,
6624 &values,
6625 &self.encryption,
6626 fallback.as_ref(),
6627 ) {
6628 payloads.push(payload);
6629 }
6630 }
6631 }
6632 Ok(payloads)
6633 }
6634
6635 pub fn conflicts(&self) -> &[ConflictRecord] {
6636 &self.conflicts
6637 }
6638
6639 pub fn rejections(&self) -> &[RejectionRecord] {
6640 &self.rejections
6641 }
6642
6643 pub fn resolve_commit_outcome(
6644 &mut self,
6645 input: ResolveCommitOutcomeInput,
6646 ) -> Result<CommitOutcome, String> {
6647 let current = self
6648 .commit_outcome(&input.client_commit_id)?
6649 .ok_or_else(|| {
6650 format!(
6651 "sync.outcome_not_found: no durable outcome exists for {:?}",
6652 input.client_commit_id
6653 )
6654 })?;
6655 if current.resolution != CommitOutcomeResolution::Active {
6656 return Ok(current);
6657 }
6658 if input.resolution == CommitOutcomeResolution::Active {
6659 return Err("sync.invalid_request: resolution must leave active state".to_owned());
6660 }
6661 match input.resolution {
6662 CommitOutcomeResolution::Superseded => {
6663 let replacement = input
6664 .replacement_client_commit_id
6665 .as_deref()
6666 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
6667 .ok_or_else(|| {
6668 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
6669 .to_owned()
6670 })?;
6671 let _ = replacement;
6672 }
6673 _ if input.replacement_client_commit_id.is_some() => {
6674 return Err(
6675 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
6676 .to_owned(),
6677 );
6678 }
6679 _ => {}
6680 }
6681 let allowed = match current.status {
6682 CommitOutcomeStatus::Conflict => matches!(
6683 input.resolution,
6684 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
6685 ),
6686 CommitOutcomeStatus::Rejected => {
6687 matches!(
6688 input.resolution,
6689 CommitOutcomeResolution::Superseded
6690 | CommitOutcomeResolution::ResolvedKeepServer
6691 )
6692 }
6693 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
6694 input.resolution == CommitOutcomeResolution::Dismissed
6695 }
6696 };
6697 if !allowed {
6698 return Err(format!(
6699 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
6700 input.resolution, current.status
6701 ));
6702 }
6703
6704 let rollback = self.capture();
6705 self.begin_observation("syncular_outcome_resolution")?;
6706 let result = (|| {
6707 self.conn
6708 .execute(
6709 "UPDATE _syncular_commit_outcomes
6710 SET resolution = ?1, resolved_at_ms = ?2,
6711 replacement_client_commit_id = ?3
6712 WHERE client_commit_id = ?4 AND resolution = 'active'",
6713 rusqlite::params![
6714 Self::outcome_resolution_name(input.resolution),
6715 self.clock_now_ms(),
6716 input.replacement_client_commit_id,
6717 input.client_commit_id
6718 ],
6719 )
6720 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6721 let resolved = self
6722 .commit_outcome(¤t.client_commit_id)?
6723 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
6724 self.conn
6725 .execute(
6726 "DELETE FROM _syncular_failed_commits WHERE client_commit_id = ?1",
6727 rusqlite::params![input.client_commit_id],
6728 )
6729 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6730 for (commit, _) in self.failed_commits.iter() {
6731 if commit.client_commit_id == input.client_commit_id {
6732 for operation in &commit.ops {
6733 self.overlay_dirty.table(&operation.table);
6734 }
6735 }
6736 }
6737 self.failed_commits
6738 .retain(|(commit, _)| commit.client_commit_id != input.client_commit_id);
6739 self.prune_commit_outcomes()?;
6740 Ok(resolved)
6741 })();
6742 let resolved = match result {
6743 Ok(outcome) => outcome,
6744 Err(error) => {
6745 self.rollback_observation("syncular_outcome_resolution");
6746 self.restore(rollback);
6747 return Err(error);
6748 }
6749 };
6750 self.conflicts
6751 .retain(|record| record.client_commit_id != current.client_commit_id);
6752 self.rejections
6753 .retain(|record| record.client_commit_id != current.client_commit_id);
6754 let mut batch = ChangeAccumulator {
6755 conflicts: current.status == CommitOutcomeStatus::Conflict,
6756 rejections: current.status == CommitOutcomeStatus::Rejected,
6757 outcomes: true,
6758 ..ChangeAccumulator::default()
6759 };
6760 if let Some(operations) = ¤t.operations {
6761 for operation in operations {
6762 batch.table(&operation.table);
6763 }
6764 }
6765 if let Err(error) = self
6766 .rebuild_overlay_if_dirty()
6767 .and_then(|()| self.finish_observation("syncular_outcome_resolution", batch))
6768 {
6769 self.rollback_observation("syncular_outcome_resolution");
6770 self.restore(rollback);
6771 return Err(error);
6772 }
6773 Ok(resolved)
6774 }
6775
6776 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
6777 self.schema_floor.as_ref()
6778 }
6779
6780 pub fn lease_state(&self) -> Option<&LeaseState> {
6782 self.lease_state.as_ref()
6783 }
6784
6785 fn record_lease_error(&mut self, code: &str) -> Result<(), String> {
6788 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
6789 return Ok(());
6790 }
6791 let mut next = self.lease_state.clone().unwrap_or_default();
6792 next.error_code = Some(code.to_owned());
6793 self.set_lease_state(Some(next))
6794 }
6795
6796 fn set_lease_state(&mut self, next: Option<LeaseState>) -> Result<(), String> {
6797 if self.lease_state == next {
6798 return Ok(());
6799 }
6800 self.begin_observation("syncular_lease")?;
6801 let rollback = self.capture();
6802 self.lease_state = next;
6803 let persisted = match &self.lease_state {
6804 Some(lease) => serde_json::to_string(lease)
6805 .map_err(|error| error.to_string())
6806 .and_then(|json| self.set_meta(LEASE_STATE_KEY, &json)),
6807 None => self.delete_meta(LEASE_STATE_KEY),
6808 };
6809 let batch = ChangeAccumulator {
6810 status: true,
6811 ..ChangeAccumulator::default()
6812 };
6813 if let Err(error) =
6814 persisted.and_then(|()| self.finish_observation("syncular_lease", batch))
6815 {
6816 self.rollback_observation("syncular_lease");
6817 self.restore(rollback);
6818 return Err(error);
6819 }
6820 Ok(())
6821 }
6822
6823 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) -> Result<(), String> {
6824 if self.schema_floor == next {
6825 return Ok(());
6826 }
6827 self.begin_observation("syncular_schema_floor")?;
6828 let rollback = self.capture();
6829 self.stopped = next.is_some();
6830 self.schema_floor = next;
6831 let persisted = match &self.schema_floor {
6832 Some(floor) => serde_json::to_string(floor)
6833 .map_err(|error| error.to_string())
6834 .and_then(|json| self.set_meta(SCHEMA_FLOOR_KEY, &json)),
6835 None => self.delete_meta(SCHEMA_FLOOR_KEY),
6836 };
6837 let batch = ChangeAccumulator {
6838 status: true,
6839 ..ChangeAccumulator::default()
6840 };
6841 if let Err(error) =
6842 persisted.and_then(|()| self.finish_observation("syncular_schema_floor", batch))
6843 {
6844 self.rollback_observation("syncular_schema_floor");
6845 self.restore(rollback);
6846 return Err(error);
6847 }
6848 Ok(())
6849 }
6850
6851 fn set_upgrading(&mut self, value: bool) {
6852 if self.upgrading == value {
6853 return;
6854 }
6855 if self.begin_observation("syncular_upgrading").is_err() {
6856 return;
6857 }
6858 self.upgrading = value;
6859 let batch = ChangeAccumulator {
6860 status: true,
6861 ..ChangeAccumulator::default()
6862 };
6863 if self
6864 .finish_observation("syncular_upgrading", batch)
6865 .is_err()
6866 {
6867 self.rollback_observation("syncular_upgrading");
6868 }
6869 }
6870
6871 pub fn sync_needed(&self) -> bool {
6872 self.sync_needed
6873 }
6874
6875 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
6876 let schema_table = self
6877 .schema
6878 .table(table)
6879 .ok_or_else(|| format!("unknown table {table:?}"))?;
6880 let sql = format!(
6881 "SELECT * FROM {} ORDER BY {} ASC",
6882 visible_table(table),
6883 quote_ident(&schema_table.primary_key)
6884 );
6885 let mut stmt = self
6886 .conn
6887 .prepare(&sql)
6888 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6889 let mut rows = stmt
6890 .query([])
6891 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6892 let mut out = Vec::new();
6893 while let Some(row) = rows
6894 .next()
6895 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6896 {
6897 let mut values = Map::new();
6898 for (i, column) in schema_table.columns.iter().enumerate() {
6899 let value = row
6900 .get_ref(i)
6901 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6902 values.insert(column.name.clone(), sql_ref_to_json(column, value));
6903 }
6904 let version: i64 = row
6905 .get(schema_table.columns.len())
6906 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
6907 let row_id = match values.get(&schema_table.primary_key) {
6908 Some(Value::String(s)) => s.clone(),
6909 Some(Value::Number(n)) => n.to_string(),
6910 Some(Value::Bool(b)) => b.to_string(),
6911 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
6912 };
6913 let sealed_key = if schema_table.has_encrypted_columns() {
6914 row.get(schema_table.columns.len() + 1)
6915 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6916 } else {
6917 None
6918 };
6919 out.push(RowState {
6920 row_id,
6921 version,
6922 values,
6923 sealed_key,
6924 });
6925 }
6926 Ok(out)
6927 }
6928
6929 #[cfg(feature = "crdt-yjs")]
6945 fn crdt_column_bytes(
6946 &self,
6947 table: &str,
6948 row_id: &str,
6949 column: &str,
6950 ) -> Result<Option<Vec<u8>>, String> {
6951 let schema_table = self
6952 .schema
6953 .table(table)
6954 .ok_or_else(|| format!("unknown table {table:?}"))?;
6955 let col = schema_table
6956 .columns
6957 .iter()
6958 .find(|c| c.name == column)
6959 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
6960 if col.ty != ColumnType::Crdt {
6961 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
6962 }
6963 let sql = format!(
6964 "SELECT {} FROM {} WHERE {}",
6965 quote_ident(column),
6966 visible_table(table),
6967 row_id_predicate(schema_table)
6968 );
6969 let bytes: Option<Vec<u8>> = self
6970 .conn
6971 .prepare_cached(&sql)
6972 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
6973 .query_row(rusqlite::params![row_id], |row| {
6974 row.get::<_, Option<Vec<u8>>>(0)
6975 })
6976 .map_err(|e| match e {
6977 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
6978 other => Self::sqlite_failure(&self.storage_failure, other),
6979 })?;
6980 Ok(bytes)
6981 }
6982
6983 #[cfg(feature = "crdt-yjs")]
6987 pub fn crdt_text(
6988 &self,
6989 table: &str,
6990 row_id: &str,
6991 column: &str,
6992 name: &str,
6993 ) -> Result<String, String> {
6994 let bytes = self
6995 .crdt_column_bytes(table, row_id, column)?
6996 .unwrap_or_default();
6997 crate::crdt::text(&bytes, name)
6998 }
6999
7000 #[cfg(feature = "crdt-yjs")]
7004 pub fn crdt_insert_text(
7005 &mut self,
7006 table: &str,
7007 row_id: &str,
7008 column: &str,
7009 name: &str,
7010 index: u32,
7011 value: &str,
7012 ) -> Result<String, ClientError> {
7013 let prior = self.storage_failure.borrow_mut().take();
7014 let outcome = (|| -> Result<String, ClientError> {
7015 let current = self
7016 .crdt_column_bytes(table, row_id, column)?
7017 .unwrap_or_default();
7018 let update = crate::crdt::insert_text(¤t, name, index, value)?;
7019 self.perform_crdt_push_update(table, row_id, column, &update)
7020 })();
7021 self.finish_authoring(prior, outcome)
7022 }
7023
7024 #[cfg(feature = "crdt-yjs")]
7028 pub fn crdt_delete_text(
7029 &mut self,
7030 table: &str,
7031 row_id: &str,
7032 column: &str,
7033 name: &str,
7034 index: u32,
7035 len: u32,
7036 ) -> Result<String, ClientError> {
7037 let prior = self.storage_failure.borrow_mut().take();
7038 let outcome = (|| -> Result<String, ClientError> {
7039 let current = self
7040 .crdt_column_bytes(table, row_id, column)?
7041 .unwrap_or_default();
7042 let update = crate::crdt::delete_text(¤t, name, index, len)?;
7043 self.perform_crdt_push_update(table, row_id, column, &update)
7044 })();
7045 self.finish_authoring(prior, outcome)
7046 }
7047
7048 #[cfg(feature = "crdt-yjs")]
7053 pub fn crdt_apply_update(
7054 &mut self,
7055 table: &str,
7056 row_id: &str,
7057 column: &str,
7058 update: &[u8],
7059 ) -> Result<String, ClientError> {
7060 let prior = self.storage_failure.borrow_mut().take();
7061 let outcome = (|| -> Result<String, ClientError> {
7062 let current = self
7063 .crdt_column_bytes(table, row_id, column)?
7064 .unwrap_or_default();
7065 let next = crate::crdt::apply_update(¤t, update)?;
7066 self.perform_crdt_push_update(table, row_id, column, &next)
7067 })();
7068 self.finish_authoring(prior, outcome)
7069 }
7070
7071 #[cfg(feature = "crdt-yjs")]
7078 fn perform_crdt_push_update(
7079 &mut self,
7080 table: &str,
7081 row_id: &str,
7082 column: &str,
7083 crdt_bytes: &[u8],
7084 ) -> Result<String, ClientError> {
7085 let mut bytes_obj = Map::new();
7086 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7087 let mut partial = Map::new();
7088 partial.insert(column.to_owned(), Value::Object(bytes_obj));
7089 self.perform_patch(table, row_id, partial, None)
7090 }
7091
7092 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7107 query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
7108 }
7109
7110 #[cfg(feature = "bench-internals")]
7112 #[doc(hidden)]
7113 pub fn benchmark_connection(&mut self) -> &mut Connection {
7114 &mut self.conn
7115 }
7116
7117 #[cfg(feature = "bench-internals")]
7119 #[doc(hidden)]
7120 pub fn benchmark_phases(
7121 &self,
7122 enabled: Option<bool>,
7123 reset: bool,
7124 ) -> Result<Option<Value>, String> {
7125 self.benchmark_phases.configure(enabled, reset)?;
7126 Ok(self.benchmark_phases.snapshot())
7127 }
7128
7129 pub fn authority_snapshot(&self) -> Result<Value, String> {
7131 validate_authority_reads(&self.schema, &self.limits.authority_reads)?;
7132 if self.limits.authority_reads.is_empty() {
7133 return Err("client.authority_read_forbidden: no authority reads were declared at client creation".into());
7134 }
7135 self.conn
7136 .execute_batch("SAVEPOINT authority_snapshot")
7137 .map_err(|e| e.to_string())?;
7138 let result = (|| {
7139 let revision = self
7140 .get_meta(LOCAL_REVISION_KEY)?
7141 .ok_or("sync.local_corrupt: authority revision is missing")?;
7142 let parsed_revision = revision
7143 .parse::<u64>()
7144 .map_err(|_| "sync.local_corrupt: authority revision is invalid")?;
7145 if parsed_revision.to_string() != revision {
7146 return Err("sync.local_corrupt: authority revision is invalid".into());
7147 }
7148 let mut tables = Vec::new();
7149 for read in &self.limits.authority_reads {
7150 let table = self
7151 .schema
7152 .table(&read.table)
7153 .ok_or("client.authority_read_forbidden: authority table is not declared")?;
7154 let scalar = |column: &str, value: &Value| -> Result<String, String> {
7155 let ty = table
7156 .columns
7157 .iter()
7158 .find(|c| c.name == column)
7159 .expect("validated authority column")
7160 .ty;
7161 if ty == ColumnType::Boolean {
7162 if let Some(value) = value.as_bool() {
7163 return Ok(value.to_string());
7164 }
7165 return match value.as_i64() {
7166 Some(0) => Ok("false".into()),
7167 Some(1) => Ok("true".into()),
7168 _ => Err("sync.local_corrupt: authority boolean is invalid".into()),
7169 };
7170 }
7171 match value {
7172 Value::String(s) => Ok(s.clone()),
7173 Value::Number(n) => Ok(n.to_string()),
7174 Value::Object(o) if ty == ColumnType::Integer && o.len() == 1 => o
7175 .get("$bigint")
7176 .and_then(Value::as_str)
7177 .map(str::to_owned)
7178 .ok_or("sync.local_corrupt: authority integer is invalid".into()),
7179 _ => {
7180 Err("sync.local_corrupt: authority scalar is missing or invalid".into())
7181 }
7182 }
7183 };
7184 let matches = |row: &QueryRow| -> Result<bool, String> {
7185 for (variable, values) in &read.scopes {
7186 let column = table
7187 .scope_column(variable)
7188 .expect("validated authority scope");
7189 let value = row
7190 .get(column)
7191 .ok_or("sync.local_corrupt: authority scope column is missing")?;
7192 if value.is_null() || !values.contains(&scalar(column, value)?) {
7193 return Ok(false);
7194 }
7195 }
7196 Ok(true)
7197 };
7198 let selected = read
7199 .columns
7200 .iter()
7201 .map(|c| quote_ident(c))
7202 .collect::<Vec<_>>()
7203 .join(",");
7204 let mut intent = BTreeSet::new();
7205 let mut relevant = BTreeSet::new();
7206 for row in query_connection(&self.conn,
7207 "SELECT id,values_json FROM (SELECT json_extract(value,'$.table') AS tbl,json_extract(value,'$.rowId') AS id,json_extract(value,'$.values') AS values_json FROM _syncular_outbox,json_each(ops_json) UNION ALL SELECT json_extract(value,'$.table'),json_extract(value,'$.rowId'),json_extract(value,'$.values') FROM _syncular_failed_commits,json_each(operations_json) UNION ALL SELECT tbl,id,intent_json FROM _syncular_acked_rows) WHERE tbl=?",
7208 &[Value::String(read.table.clone())]).map_err(|e| e.to_string())? {
7209 let id = row.get("id").and_then(Value::as_str).ok_or("sync.local_corrupt: authority intent id is invalid")?.to_owned();
7210 intent.insert(id.clone());
7211 if let Some(raw) = row.get("values_json").and_then(Value::as_str) {
7212 let values: QueryRow = serde_json::from_str(raw).map_err(|_| "sync.local_corrupt: authority intent values are invalid")?;
7213 if read.scopes.keys().all(|v| values.contains_key(table.scope_column(v).expect("validated scope"))) && matches(&values)? {relevant.insert(id);}
7214 }
7215 }
7216 let mut rows = BTreeMap::new();
7217 for full in [base_table(&read.table), visible_table(&read.table)] {
7218 for mut values in query_connection(
7219 &self.conn,
7220 &format!(
7221 "SELECT {selected},_syncular_version AS authority_version FROM {full}"
7222 ),
7223 &[],
7224 )
7225 .map_err(|e| e.to_string())?
7226 {
7227 if !matches(&values)? {
7228 continue;
7229 }
7230 let id = scalar(
7231 &table.primary_key,
7232 values
7233 .get(&table.primary_key)
7234 .expect("selected primary key"),
7235 )?;
7236 relevant.insert(id.clone());
7237 if full == base_table(&read.table) {
7238 let version = values
7239 .remove("authority_version")
7240 .expect("selected version");
7241 let version = version
7242 .as_i64()
7243 .ok_or("sync.local_corrupt: authority server version is invalid")?;
7244 if version >= 0 {
7245 rows.insert(id.clone(), json!({"values": values, "version": version, "hasLocalIntent": intent.contains(&id)}));
7246 }
7247 }
7248 }
7249 }
7250 intent.retain(|id| relevant.contains(id));
7251 let rows = rows.into_values().collect::<Vec<_>>();
7252 let replica = self.stored(replica_cursor(&self.conn))?;
7254 let persisted = self
7255 .stored(load_units(&self.conn))?
7256 .into_iter()
7257 .filter(|unit| covers(table, &unit.scopes))
7258 .map(|unit| {
7259 let effective = unit.echo.as_deref().and_then(|echo| {
7260 pair_scopes(&unit.scopes, &read.table, Some(echo))
7261 });
7262 let requested = read
7263 .scopes
7264 .keys()
7265 .map(|v| (v.clone(), json!(unit.scopes.get(v).into_iter().collect::<Vec<_>>())))
7266 .collect::<Map<_, _>>();
7267 let complete = effective
7268 .as_ref()
7269 .is_some_and(|scopes| scopes.len() == read.scopes.len())
7270 && unit.status == UnitStatus::Active
7271 && !unit.bootstrapping();
7272 let exposed = effective.map(|scopes| {
7273 read.scopes
7274 .keys()
7275 .map(|v| {
7276 let held = scopes.iter().find(|(variable, _)| variable == v);
7277 (v.clone(), json!(held.map_or(&Vec::new(), |(_, values)| values)))
7278 })
7279 .collect::<Map<_, _>>()
7280 });
7281 json!({"requestedScopes": requested, "status": unit.status, "cursor": unit.cursor.unwrap_or(replica), "effectiveScopes": exposed, "complete": complete})
7282 })
7283 .collect::<Vec<_>>();
7284 fn covered(selectors: &[(&String, &Vec<String>)], candidates: &[&Value]) -> bool {
7285 match selectors.split_first() {
7286 None => !candidates.is_empty(),
7287 Some(((variable, values), rest)) => values.iter().all(|value| {
7288 covered(
7289 rest,
7290 &candidates
7291 .iter()
7292 .copied()
7293 .filter(|p| {
7294 p["effectiveScopes"][variable.as_str()]
7295 .as_array()
7296 .is_some_and(|held| {
7297 held.contains(&Value::String(value.clone()))
7298 || held.contains(&Value::String("*".into()))
7299 })
7300 })
7301 .collect::<Vec<_>>(),
7302 )
7303 }),
7304 }
7305 }
7306 let coverage = if covered(
7307 &read.scopes.iter().collect::<Vec<_>>(),
7308 &persisted
7309 .iter()
7310 .filter(|p| p["complete"] == true)
7311 .collect::<Vec<_>>(),
7312 ) {
7313 "complete"
7314 } else if persisted.iter().any(|p| {
7315 p["status"] == "active"
7316 && read.scopes.iter().all(|(v, values)| {
7317 values.iter().any(|value| {
7318 p["requestedScopes"][v].as_array().is_some_and(|held| {
7319 held.contains(&Value::String(value.clone()))
7320 })
7321 })
7322 })
7323 }) {
7324 "pending"
7325 } else {
7326 "missing"
7327 };
7328 tables.push(json!({"table": read.table, "rows": rows, "localIntentRowIds": intent, "scopes": read.scopes, "coverage": coverage, "persisted": persisted}));
7329 }
7330 Ok(
7331 json!({"revision": revision, "complete": tables.iter().all(|t| t["coverage"] == "complete"), "tables": tables}),
7332 )
7333 })();
7334 match result {
7335 Ok(snapshot) => {
7336 self.conn
7337 .execute_batch("RELEASE authority_snapshot")
7338 .map_err(|e| e.to_string())?;
7339 Ok(snapshot)
7340 }
7341 Err(error) => {
7342 self.conn
7343 .execute_batch("ROLLBACK TO authority_snapshot; RELEASE authority_snapshot")
7344 .map_err(|e| format!("{error}; rollback failed: {e}"))?;
7345 Err(error)
7346 }
7347 }
7348 }
7349
7350 pub fn query_snapshot(
7354 &mut self,
7355 sql: &str,
7356 params: &[QueryValue],
7357 coverage: &[WindowCoverage],
7358 owner: Option<QueryOwner<'_>>,
7359 ) -> Result<QuerySnapshot, String> {
7360 let result = snapshot_connection(&self.conn, sql, params, coverage);
7361 if let Some(owner) = owner {
7362 self.record_query_read(owner, result.as_ref().err());
7363 }
7364 result.map_err(|failure| failure.to_string())
7365 }
7366
7367 pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
7371 let position = self
7372 .query_failures
7373 .iter()
7374 .position(|entry| entry.id == owner.id);
7375 let Some(failure) = failure else {
7376 if let Some(index) = position {
7377 self.query_failures.remove(index);
7378 }
7379 return;
7380 };
7381 let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
7382 .into_iter()
7383 .map(str::to_owned)
7384 .collect();
7385 let code = failure.code.unwrap_or("client.query_failed").to_owned();
7386 if position.is_some_and(|index| {
7387 let entry = &self.query_failures[index];
7388 entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
7389 }) {
7390 return;
7391 }
7392 if let Some(index) = position {
7393 self.query_failures.remove(index);
7394 }
7395 self.query_failures.push(DiagnosticQueryFailure {
7396 id: owner.id.to_owned(),
7397 tables,
7398 code,
7399 sqlite_code: failure.sqlite_code,
7400 at_ms: self.clock_now_ms(),
7401 });
7402 if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
7403 self.query_failures.remove(0);
7404 }
7405 }
7406
7407 fn build_request(&self, url_capable: bool) -> Result<(Message, RequestMeta), Box<SyncOutcome>> {
7410 #[cfg(feature = "bench-internals")]
7411 let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7412 let log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7413 Box::new(SyncOutcome::Failed {
7414 error_code: "storage.failed".into(),
7415 message,
7416 details: None,
7417 recommended_action: None,
7418 })
7419 })?;
7420 let max_commits = self.limits.max_push_commits_per_request;
7421 let max_ops = self
7422 .limits
7423 .max_push_operations_per_request
7424 .unwrap_or(PUSH_OPS_PER_REQUEST);
7425 let byte_cap = self.limits.max_push_request_bytes;
7426 let accept = self.limits.accept.unwrap_or(if url_capable {
7429 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
7430 } else {
7431 DEFAULT_ACCEPT
7432 });
7433 let storage_failed = |message: String| {
7434 Box::new(SyncOutcome::Failed {
7435 error_code: "storage.failed".into(),
7436 message,
7437 details: None,
7438 recommended_action: None,
7439 })
7440 };
7441 let header = Frame::ReqHeader {
7442 client_id: self.client_id.clone(),
7443 schema_version: self.schema.version,
7444 log_epoch: log_epoch.clone(),
7445 outbox_id: self.get_meta(OUTBOX_ID_KEY).map_err(storage_failed)?,
7446 };
7447 let mut lead = vec![header];
7451 lead.extend(
7452 self.round_failure
7453 .iter()
7454 .map(|code| Frame::RoundFailure { code: code.clone() }),
7455 );
7456 let registered: Vec<UnitRecord> = self
7457 .stored(load_units(&self.conn))
7458 .map_err(storage_failed)?
7459 .into_iter()
7460 .filter(|unit| unit.status == UnitStatus::Active)
7461 .collect();
7462 let registration_digest = ssp2::registration::registration_digest(
7463 registered
7464 .iter()
7465 .map(|unit| scope_pairs(&unit.scopes))
7466 .collect::<Vec<_>>()
7467 .iter()
7468 .map(Vec::as_slice),
7469 );
7470 let echo_digest = self
7471 .get_meta(ECHO_DIGEST_KEY)
7472 .map_err(storage_failed)?
7473 .filter(|hex| !hex.is_empty())
7474 .map(|hex| {
7475 crate::values::hex_to_bytes(&hex)
7476 .ok()
7477 .and_then(|bytes| <[u8; 32]>::try_from(bytes).ok())
7478 .ok_or_else(|| {
7479 "sync.local_corrupt: persisted echo digest is invalid".to_owned()
7480 })
7481 })
7482 .transpose()
7483 .map_err(storage_failed)?;
7484 let pull = Frame::PullHeader {
7485 limit_commits: self.limits.limit_commits.unwrap_or(0),
7486 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
7487 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
7488 accept,
7489 cursor: self
7490 .stored(replica_cursor(&self.conn))
7491 .map_err(storage_failed)?,
7492 registration_digest,
7493 echo_digest,
7494 };
7495 let registration_digest = bytes_to_hex(®istration_digest);
7496 let travels = self
7499 .get_meta(REGISTERED_DIGEST_KEY)
7500 .map_err(storage_failed)?
7501 .as_deref()
7502 != Some(registration_digest.as_str());
7503 let mut sub_frames: Vec<Frame> = Vec::new();
7504 if travels {
7505 sub_frames.extend(registered.iter().map(|unit| Frame::Unit {
7506 scopes: scope_pairs(&unit.scopes),
7507 }));
7508 }
7509 sub_frames.extend(registered.iter().filter_map(|unit| {
7511 Some(Frame::Progress {
7512 id: unit.id.clone(),
7513 cursor: unit.cursor?,
7514 bootstrap_state: unit.bootstrap_state.clone().map(RawJson),
7515 })
7516 }));
7517 let fixed_bytes = match byte_cap {
7520 Some(_) => {
7521 let mut frames = lead.clone();
7522 frames.push(pull.clone());
7523 frames.extend(sub_frames.iter().cloned());
7524 encode_message(&Message {
7525 wire_version: WIRE_VERSION,
7526 msg_kind: MsgKind::Request,
7527 frames,
7528 })
7529 .len()
7530 }
7531 None => 0,
7532 };
7533 if let Some(cap) = byte_cap {
7534 if fixed_bytes > cap {
7535 return Err(Self::push_capacity_outcome("bytes", cap, fixed_bytes, None));
7536 }
7537 }
7538 let base_bytes = match byte_cap {
7539 Some(_) => {
7540 let mut frames = lead.clone();
7541 frames.push(pull.clone());
7542 encode_message(&Message {
7543 wire_version: WIRE_VERSION,
7544 msg_kind: MsgKind::Request,
7545 frames,
7546 })
7547 .len()
7548 }
7549 None => 0,
7550 };
7551 let total_outbox = if log_epoch.is_some() {
7552 self.outbox.len()
7553 } else {
7554 0
7555 };
7556 let mut push_frames: Vec<Frame> = Vec::new();
7557 let mut pushed_ids = Vec::new();
7558 let mut ops_in_request = 0usize;
7559 let mut bytes_in_request = 0usize;
7560 let mut deferred_commits = 0usize;
7561 #[cfg(feature = "bench-internals")]
7562 let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7563 for (index, commit) in self.outbox.iter().take(total_outbox).enumerate() {
7564 if max_commits.is_some_and(|cap| push_frames.len() >= cap) {
7568 deferred_commits = total_outbox - index;
7569 break;
7570 }
7571 if ops_in_request + commit.ops.len() > max_ops {
7572 if push_frames.is_empty() {
7573 return Err(Self::push_capacity_outcome(
7574 "operations",
7575 max_ops,
7576 commit.ops.len(),
7577 Some(commit.client_commit_id.as_str()),
7578 ));
7579 }
7580 deferred_commits = total_outbox - index;
7581 break;
7582 }
7583 if self.outbox[..index]
7586 .iter()
7587 .flat_map(|earlier| &earlier.ops)
7588 .any(|earlier| commit.ops.iter().any(|op| chains_on(earlier, op)))
7589 {
7590 deferred_commits = total_outbox - index;
7591 break;
7592 }
7593 let operations = commit
7594 .ops
7595 .iter()
7596 .map(|op| {
7597 let payload = match self
7598 .schema
7599 .table(&op.table)
7600 .and_then(|table| Some((table, op.presented_values(table)?)))
7601 {
7602 Some((table, values)) => {
7603 let fallback =
7613 self.stored_key_fallback(table, &op.row_id)
7614 .map_err(|message| {
7615 Box::new(SyncOutcome::Failed {
7616 error_code: "storage.failed".into(),
7617 message,
7618 details: None,
7619 recommended_action: None,
7620 })
7621 })?;
7622 encode_sparse_row_json(
7623 table,
7624 &op.row_id,
7625 &values,
7626 &self.encryption,
7627 fallback.as_ref(),
7628 )
7629 .ok()
7630 }
7631 None => None,
7632 };
7633 Ok(ssp2::model::Operation {
7634 table: op.table.clone(),
7635 row_id: op.row_id.clone(),
7636 op: if op.upsert { Op::Upsert } else { Op::Delete },
7637 base_version: op.base_version,
7638 payload,
7639 })
7640 })
7641 .collect::<Result<Vec<_>, Box<SyncOutcome>>>()?;
7642 let client_seq = self
7644 .conn
7645 .prepare_cached("SELECT seq FROM _syncular_outbox WHERE commit_id = ?1")
7646 .and_then(|mut statement| {
7647 statement.query_row([&commit.client_commit_id], |row| row.get(0))
7648 })
7649 .map_err(|error| {
7650 storage_failed(Self::sqlite_failure(&self.storage_failure, error))
7651 })?;
7652 let frame = Frame::PushCommit {
7653 client_commit_id: commit.client_commit_id.clone(),
7654 client_seq,
7655 operations,
7656 };
7657 let frame_bytes = match byte_cap {
7658 Some(_) => {
7659 let mut frames = lead.clone();
7660 frames.push(frame.clone());
7661 frames.push(pull.clone());
7662 encode_message(&Message {
7663 wire_version: WIRE_VERSION,
7664 msg_kind: MsgKind::Request,
7665 frames,
7666 })
7667 .len()
7668 - base_bytes
7669 }
7670 None => 0,
7671 };
7672 if let Some(cap) = byte_cap {
7676 if !push_frames.is_empty() && fixed_bytes + bytes_in_request + frame_bytes > cap {
7677 deferred_commits = total_outbox - index;
7678 break;
7679 }
7680 }
7681 push_frames.push(frame);
7682 pushed_ids.push(commit.client_commit_id.clone());
7683 ops_in_request += commit.ops.len();
7684 bytes_in_request += frame_bytes;
7685 }
7686 #[cfg(feature = "bench-internals")]
7687 drop(outbox_phase);
7688 if let Some(cap) = byte_cap {
7689 if fixed_bytes + bytes_in_request > cap {
7690 let blocked = if push_frames.is_empty() {
7693 None
7694 } else {
7695 self.outbox.first().map(|c| c.client_commit_id.as_str())
7696 };
7697 return Err(Self::push_capacity_outcome(
7698 "bytes",
7699 cap,
7700 fixed_bytes + bytes_in_request,
7701 blocked,
7702 ));
7703 }
7704 }
7705 let mut frames = lead;
7706 frames.extend(push_frames);
7707 frames.push(pull);
7708 frames.extend(sub_frames);
7709 let message = Message {
7710 wire_version: WIRE_VERSION,
7711 msg_kind: MsgKind::Request,
7712 frames,
7713 };
7714 Ok((
7715 message,
7716 RequestMeta {
7717 pushed_ids,
7718 registration_digest,
7719 accept,
7720 deferred_commits,
7721 applied: BTreeMap::new(),
7722 deferred: BTreeSet::new(),
7723 },
7724 ))
7725 }
7726
7727 fn push_capacity_outcome(
7729 kind: &str,
7730 limit: usize,
7731 size: usize,
7732 client_commit_id: Option<&str>,
7733 ) -> Box<SyncOutcome> {
7734 let mut details = Map::new();
7735 details.insert("kind".to_owned(), Value::from(kind));
7736 details.insert("limit".to_owned(), Value::from(limit as u64));
7737 details.insert("size".to_owned(), Value::from(size as u64));
7738 if let Some(id) = client_commit_id {
7739 details.insert("clientCommitId".to_owned(), Value::from(id));
7740 }
7741 Box::new(SyncOutcome::Failed {
7742 error_code: "client.push_request_too_large".into(),
7743 message: "the push request exceeds a configured capacity limit".into(),
7744 details: Some(Value::Object(details)),
7745 recommended_action: None,
7746 })
7747 }
7748
7749 pub fn progress(&self) -> ProgressObserver {
7753 self.progress.clone()
7754 }
7755
7756 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
7757 self.sync_bounded(transport, None)
7758 }
7759
7760 fn sync_bounded(
7762 &mut self,
7763 transport: &mut dyn Transport,
7764 idle_bound: Option<(u64, u32)>,
7765 ) -> SyncOutcome {
7766 let mut prepared = match self.prepare_sync_round(transport.supports_url_fetch()) {
7767 Ok(prepared) => prepared,
7768 Err(outcome) => return *outcome,
7769 };
7770 prepared.idle_bound = idle_bound;
7771 let mut next = prepared;
7776 loop {
7777 match self.apply_sync_round(next.exchange(transport)) {
7778 crate::AppliedSyncRound::Continue(prepared) => next = prepared,
7779 crate::AppliedSyncRound::Complete {
7780 outcome, controls, ..
7781 } => {
7782 if self.transport_enabled {
7783 for text in controls {
7784 let _ = transport.realtime_send(&text);
7785 }
7786 } else {
7787 self.disconnect_realtime(transport);
7788 }
7789 return outcome;
7790 }
7791 }
7792 }
7793 }
7794
7795 fn finish_sync_round(&mut self, started_at_ms: i64, mut outcome: SyncOutcome) -> SyncOutcome {
7796 if let SyncOutcome::Failed {
7797 error_code,
7798 message,
7799 details,
7800 ..
7801 } = &mut outcome
7802 {
7803 if let Some((failure, code)) = self
7804 .storage_failure
7805 .borrow()
7806 .as_ref()
7807 .and_then(|failure| failure.code.map(|code| (failure, code)))
7808 {
7809 *error_code = code.to_owned();
7810 *message = match code {
7811 "client.storage_busy" => "local SQLite storage is busy",
7812 "client.storage_full" => "local SQLite storage is full",
7813 "client.storage_io" => "local SQLite storage I/O failed",
7814 "client.storage_corrupt" => "local SQLite storage is corrupt",
7815 _ => unreachable!("unrecognized classified storage failure"),
7816 }
7817 .to_owned();
7818 *details = failure.details();
7819 }
7820 }
7821 let details = match &outcome {
7822 SyncOutcome::Failed { details, .. } => details.as_ref(),
7823 _ => None,
7824 };
7825 let cause_kind = details
7826 .and_then(|details| details.get("causeKind"))
7827 .and_then(Value::as_str)
7828 .and_then(|cause| match cause {
7829 "network" => Some(crate::TransportCauseKind::Network),
7830 "connect" => Some(crate::TransportCauseKind::Connect),
7831 "tls" => Some(crate::TransportCauseKind::Tls),
7832 "io" => Some(crate::TransportCauseKind::Io),
7833 "request" => Some(crate::TransportCauseKind::Request),
7834 "protocol" => Some(crate::TransportCauseKind::Protocol),
7835 "timeout" => Some(crate::TransportCauseKind::Timeout),
7836 "round_deadline" => Some(crate::TransportCauseKind::RoundDeadline),
7837 "aborted" => Some(crate::TransportCauseKind::Aborted),
7838 "body" => Some(crate::TransportCauseKind::Body),
7839 "status" => Some(crate::TransportCauseKind::Status),
7840 "redirect" => Some(crate::TransportCauseKind::Redirect),
7841 "unknown" => Some(crate::TransportCauseKind::Unknown),
7842 _ => None,
7843 });
7844 let http_status = if matches!(
7845 cause_kind,
7846 Some(
7847 crate::TransportCauseKind::Status
7848 | crate::TransportCauseKind::Body
7849 | crate::TransportCauseKind::Redirect
7850 )
7851 ) {
7852 details
7853 .and_then(|details| details.get("httpStatus"))
7854 .and_then(Value::as_f64)
7855 .filter(|status| status.fract() == 0.0 && (100.0..=599.0).contains(status))
7856 .map(|status| status as u16)
7857 } else {
7858 None
7859 };
7860 let retry_delay_ms = self.round_retry_delay_ms;
7861 self.progress.update(|p| match &outcome {
7862 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report)
7863 if report.failed.is_empty() =>
7864 {
7865 p.state = ProgressState::Complete
7866 }
7867 SyncOutcome::Ok(_) | SyncOutcome::BudgetExhausted(_) => {
7868 p.state = ProgressState::Failed;
7869 p.error_code = Some("sync.scope_revoked".into());
7870 }
7871 SyncOutcome::Failed { error_code, .. } => {
7872 p.state = ProgressState::Failed;
7873 p.error_code = Some(Self::diagnostic_code(error_code));
7874 p.cause_kind = cause_kind;
7875 p.http_status = http_status;
7876 p.retry_delay_ms = retry_delay_ms;
7877 }
7878 SyncOutcome::RealtimeUnavailable { .. } => {
7879 p.state = ProgressState::Failed;
7880 p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
7881 p.retry_delay_ms = retry_delay_ms;
7882 }
7883 });
7884 let completed_at_ms = self.clock_now_ms();
7885 self.last_round = Some(match &outcome {
7886 SyncOutcome::Ok(report) | SyncOutcome::BudgetExhausted(report) => DiagnosticLastRound {
7887 status: "succeeded".to_owned(),
7888 started_at_ms,
7889 completed_at_ms,
7890 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7891 counters: Some(DiagnosticRoundCounters {
7892 pushed: report.pushed,
7893 applied: report.applied.len(),
7894 rejected: report.rejected.len(),
7895 retryable: report.retryable.len(),
7896 conflicts: report.conflicts,
7897 commits_applied: report.commits_applied,
7898 segment_rows_applied: report.segment_rows_applied,
7899 bootstrapping: report.bootstrapping.len(),
7900 resets: report.resets.len(),
7901 revoked: report.revoked.len(),
7902 failed: report.failed.len(),
7903 deferred_commits: report.deferred_commits,
7904 }),
7905 error_code: None,
7906 cause_kind: None,
7907 http_status: None,
7908 },
7909 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
7910 status: "failed".to_owned(),
7911 started_at_ms,
7912 completed_at_ms,
7913 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7914 counters: None,
7915 error_code: Some(Self::diagnostic_code(error_code)),
7916 cause_kind,
7917 http_status,
7918 },
7919 SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
7920 status: "failed".to_owned(),
7921 started_at_ms,
7922 completed_at_ms,
7923 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
7924 counters: None,
7925 error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
7926 cause_kind: None,
7927 http_status: None,
7928 },
7929 });
7930 outcome
7931 }
7932
7933 pub fn prepare_sync_round(
7936 &mut self,
7937 url_capable: bool,
7938 ) -> Result<crate::PreparedSyncRound, Box<SyncOutcome>> {
7939 if !self.transport_enabled {
7940 return Err(Box::new(SyncOutcome::Failed {
7941 error_code: "sync.offline".into(),
7942 message: "transport gate is closed".into(),
7943 details: None,
7944 recommended_action: None,
7945 }));
7946 }
7947 if self.active_round.is_some() {
7948 return Err(Box::new(SyncOutcome::Failed {
7949 error_code: "client.round_in_progress".into(),
7950 message: "a sync round is already in flight".into(),
7951 details: None,
7952 recommended_action: None,
7953 }));
7954 }
7955 self.storage_failure.borrow_mut().take();
7956 self.progress.start();
7957 self.round_retry_delay_ms = None;
7958 let started_at_ms = self.clock_now_ms();
7959 let prepared = (|| {
7960 if self.stopped {
7961 return Err(Box::new(SyncOutcome::Ok(SyncReport {
7962 schema_floor: self.schema_floor.clone(),
7963 ..SyncReport::default()
7964 })));
7965 }
7966 if self.security_preflight {
7967 return Err(Box::new(SyncOutcome::Failed {
7968 error_code: SECURITY_PREFLIGHT_REQUIRED_CODE.into(),
7969 message: "security preflight must complete before syncing".into(),
7970 details: None,
7971 recommended_action: None,
7972 }));
7973 }
7974 self.set_sync_needed(false, false);
7975 self.drain_pending_evictions()
7976 .and_then(|_| self.drop_incompatible_outbox())
7977 .and_then(|_| self.drop_unencodable_outbox(true, None))
7978 .map_err(|message| {
7979 Box::new(SyncOutcome::Failed {
7980 error_code: "storage.failed".into(),
7981 message,
7982 details: None,
7983 recommended_action: None,
7984 })
7985 })?;
7986 self.reveal_sealed_rows().map_err(|error| {
7988 Box::new(SyncOutcome::Failed {
7989 error_code: error.code,
7990 message: error.message,
7991 details: error.details,
7992 recommended_action: None,
7993 })
7994 })?;
7995 let has_log_epoch = self.get_meta(LOG_EPOCH_KEY).map_err(|message| {
7996 Box::new(SyncOutcome::Failed {
7997 error_code: "storage.failed".into(),
7998 message,
7999 details: None,
8000 recommended_action: None,
8001 })
8002 })?;
8003 let uploads = if has_log_epoch.is_some() && self.schema_has_blobs() {
8004 self.pending_blob_uploads().map_err(|e| {
8005 Box::new(SyncOutcome::Failed {
8006 error_code: e.code,
8007 message: e.message,
8008 details: e.details,
8009 recommended_action: None,
8010 })
8011 })?
8012 } else {
8013 Vec::new()
8014 };
8015 if self.realtime_state != RealtimeState::Connected
8016 && self.realtime_policy == RealtimePolicy::Required
8017 {
8018 let retry_delay_ms = self.schedule_background_retry();
8019 self.realtime_retry_delay_ms = Some(retry_delay_ms);
8020 return Err(Box::new(SyncOutcome::RealtimeUnavailable {
8021 state: self.realtime_state,
8022 reason_code: self.realtime_reason_code.clone(),
8023 retry_delay_ms,
8024 }));
8025 }
8026 let (message, mut meta) = self.build_request(url_capable)?;
8027 let units = self.stored(load_units(&self.conn)).map_err(|message| {
8028 Box::new(SyncOutcome::Failed {
8029 error_code: "storage.failed".into(),
8030 message,
8031 details: None,
8032 recommended_action: None,
8033 })
8034 })?;
8035 for unit in &units {
8036 let applied =
8037 self.stored(applied_rows(&self.conn, &unit.id))
8038 .map_err(|message| {
8039 Box::new(SyncOutcome::Failed {
8040 error_code: "storage.failed".into(),
8041 message,
8042 details: None,
8043 recommended_action: None,
8044 })
8045 })?;
8046 if !applied.is_empty() {
8047 meta.applied.insert(unit.id.clone(), applied);
8048 }
8049 }
8050 let id = uuid::Uuid::new_v4();
8051 self.active_round = Some(id);
8052 let phase = if uploads.is_empty() {
8053 crate::round::RoundPhase::Sync
8054 } else {
8055 crate::round::RoundPhase::UploadGrant
8056 };
8057 Ok(crate::PreparedSyncRound {
8058 id,
8059 started_at_ms,
8060 message,
8061 meta,
8062 uploads: uploads.into(),
8063 phase,
8064 encoded: None,
8065 continuation: None,
8066 completed: None,
8067 realtime: self.realtime_state == RealtimeState::Connected,
8068 scopes: units
8070 .into_iter()
8071 .map(|unit| {
8072 (
8073 unit.id,
8074 canonical_scope_json(&unit_scope_lists(&unit.scopes)),
8075 )
8076 })
8077 .collect(),
8078 #[cfg(feature = "bench-internals")]
8079 benchmark_phases: self.benchmark_phases.clone(),
8080 progress: self.progress.clone(),
8081 fixed_now: self
8082 .now_ms
8083 .as_ref()
8084 .map(|clock| clock.load(std::sync::atomic::Ordering::Relaxed)),
8085 round_deadline_at: None,
8086 idle_bound: None,
8087 })
8088 })();
8089 prepared.map_err(|outcome| Box::new(self.finish_sync_round(started_at_ms, *outcome)))
8090 }
8091
8092 pub fn cancel_sync_round(&mut self) {
8095 self.active_round = None;
8096 }
8097
8098 pub fn apply_sync_round(
8101 &mut self,
8102 completed: crate::CompletedSyncRound,
8103 ) -> crate::AppliedSyncRound {
8104 let mut prepared = completed.prepared;
8105 if self.active_round != Some(prepared.id) || self.security_preflight {
8106 return crate::AppliedSyncRound::Complete {
8107 outcome: SyncOutcome::Failed {
8108 error_code: "client.round_cancelled".into(),
8109 message: "sync round belongs to a released client context".into(),
8110 details: None,
8111 recommended_action: None,
8112 },
8113 controls: Vec::new(),
8114 more: false,
8115 bootstrap_advanced: false,
8116 };
8117 }
8118 self.storage_failure.borrow_mut().take();
8119 let (response, mut downloads, transport_failed) = match completed.exchange {
8120 crate::round::ExchangeResult::Upload { id, result } => {
8121 let result = match result {
8122 Err(error)
8127 if error
8128 .details
8129 .as_ref()
8130 .and_then(|details| details.get("retryable"))
8131 == Some(&Value::Bool(false)) =>
8132 {
8133 self.conn
8134 .prepare(
8135 "SELECT commit_id FROM _syncular_blob_commit_refs WHERE blob_id = ?",
8136 )
8137 .and_then(|mut statement| {
8138 statement
8139 .query_map(rusqlite::params![id], |row| row.get::<_, String>(0))?
8140 .collect::<Result<HashSet<_>, _>>()
8141 })
8142 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
8143 .and_then(|refused| {
8144 let commits = self
8145 .outbox
8146 .iter()
8147 .filter(|commit| refused.contains(&commit.client_commit_id))
8148 .cloned()
8149 .collect::<Vec<_>>();
8150 if !commits.is_empty() {
8151 self.drop_outbox_commits(&commits, &error.code, &error.message)?;
8152 }
8153 prepared.message.frames.retain(|frame| {
8154 !matches!(frame, Frame::PushCommit { client_commit_id, .. }
8155 if refused.contains(client_commit_id))
8156 });
8157 prepared.meta.pushed_ids.retain(|id| !refused.contains(id));
8158 prepared.continuation = Some(crate::round::RoundPhase::UploadGrant);
8159 Ok(())
8160 })
8161 .map_err(|message| TransportError::new("client.failed", message))
8162 }
8163 result => result,
8164 };
8165 let result = result.and_then(|()| {
8166 self.conn
8167 .execute(
8168 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
8169 rusqlite::params![id],
8170 )
8171 .map(|_| ())
8172 .map_err(|error| {
8173 TransportError::new(
8174 "client.failed",
8175 Self::sqlite_failure(&self.storage_failure, error),
8176 )
8177 })
8178 });
8179 match result {
8180 Ok(()) => return crate::AppliedSyncRound::Continue(prepared),
8181 Err(TransportError {
8182 code,
8183 message,
8184 details,
8185 }) => {
8186 self.active_round = None;
8187 if Self::unreachable_transport(&details) {
8188 self.set_sync_needed(true, false);
8191 } else if Self::retryable_failure_code(&code) {
8192 self.schedule_background_retry();
8193 }
8194 let recommended_action = recommended_action_of(&details);
8195 let outcome = self.finish_sync_round(
8196 prepared.started_at_ms,
8197 SyncOutcome::Failed {
8198 error_code: code,
8199 message,
8200 details,
8201 recommended_action,
8202 },
8203 );
8204 return crate::AppliedSyncRound::Complete {
8205 outcome,
8206 controls: Vec::new(),
8207 more: false,
8208 bootstrap_advanced: false,
8209 };
8210 }
8211 }
8212 }
8213 crate::round::ExchangeResult::Reply {
8214 response,
8215 downloads,
8216 transport_failed,
8217 } => (response, downloads, transport_failed),
8218 };
8219 self.active_round = None;
8220 let before = self.resume_tokens();
8221 let outcome = (|| {
8222 let response = match response {
8223 Ok(response) => {
8224 self.round_failure = None;
8227 response
8228 }
8229 Err(TransportError {
8230 code,
8231 message,
8232 details,
8233 }) => {
8234 if code != "sync.idle_timeout" && ssp2::model::valid_round_failure_code(&code) {
8240 self.round_failure = Some(code.clone());
8241 }
8242 if transport_failed {
8243 if prepared.realtime {
8244 self.set_realtime_state(RealtimeState::Lost, Some(&code));
8245 }
8246 if let Err(message) = self.record_lease_error(&code) {
8247 return SyncOutcome::Failed {
8248 error_code: "storage.failed".to_owned(),
8249 message,
8250 details: None,
8251 recommended_action: None,
8252 };
8253 }
8254 }
8255 if Self::retryable_failure_code(&code) && !Self::unreachable_transport(&details)
8256 {
8257 let delay = self.schedule_background_retry();
8258 if prepared.realtime && transport_failed {
8259 self.realtime_retry_delay_ms = Some(delay);
8260 }
8261 }
8262 let recommended_action = recommended_action_of(&details);
8263 return SyncOutcome::Failed {
8264 error_code: code,
8265 message,
8266 details,
8267 recommended_action,
8268 };
8269 }
8270 };
8271 let mut outcome = self.process_response(&mut downloads, response, &prepared.meta);
8272 if let SyncOutcome::Ok(report) = &mut outcome {
8273 report.deferred_commits = prepared.meta.deferred_commits;
8274 }
8275 match &outcome {
8276 SyncOutcome::Ok(_) => self.reset_background_retry(),
8277 SyncOutcome::Failed {
8278 error_code,
8279 details,
8280 ..
8281 } if Self::retryable_failure_code(error_code)
8282 && !Self::unreachable_transport(details) =>
8283 {
8284 self.schedule_background_retry();
8285 }
8286 _ => {}
8287 }
8288 if prepared.meta.deferred_commits > 0 {
8289 self.set_sync_needed(true, true);
8290 }
8291 self.previous_version_lifetime_check();
8292 outcome
8293 })();
8294 match &outcome {
8295 SyncOutcome::Ok(report) if !report.bootstrapping.is_empty() => {
8298 self.set_sync_needed(true, false)
8299 }
8300 SyncOutcome::Failed { details, .. } if Self::unreachable_transport(details) => {
8303 self.set_sync_needed(true, false)
8304 }
8305 _ => {}
8306 }
8307 let more = self.transport_enabled
8308 && matches!(&outcome, SyncOutcome::Ok(report)
8309 if report.schema_floor.is_none()
8310 && (!report.bootstrapping.is_empty()
8311 || report.commits_applied > 0
8312 || report.segment_rows_applied > 0
8313 || !report.resets.is_empty()
8314 || report.deferred_commits > 0
8315 || self.sync_needed));
8316 let bootstrap_advanced = matches!(&outcome, SyncOutcome::Ok(report) if report.segment_rows_applied > 0 && !report.bootstrapping.is_empty())
8317 && before != self.resume_tokens();
8318 crate::AppliedSyncRound::Complete {
8319 outcome: self.finish_sync_round(prepared.started_at_ms, outcome),
8320 controls: if self.transport_enabled {
8321 downloads.controls
8322 } else {
8323 Vec::new()
8324 },
8325 more,
8326 bootstrap_advanced,
8327 }
8328 }
8329
8330 #[allow(clippy::type_complexity)]
8334 fn resume_tokens(&self) -> Option<Vec<(String, Option<String>, Option<String>)>> {
8335 self.conn
8336 .prepare_cached(
8337 "SELECT id, bootstrap_state, applied FROM _syncular_units ORDER BY rowid",
8338 )
8339 .and_then(|mut stmt| {
8340 stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
8341 .collect()
8342 })
8343 .ok()
8344 }
8345
8346 pub const SYNC_IDLE_TIMEOUT_MS: u64 = 60_000;
8348
8349 pub fn sync_until_idle(
8350 &mut self,
8351 transport: &mut dyn Transport,
8352 max_rounds: Option<u32>,
8353 ) -> SyncOutcome {
8354 self.sync_until_idle_with_deadline(transport, max_rounds, Some(Self::SYNC_IDLE_TIMEOUT_MS))
8355 }
8356
8357 pub fn sync_until_idle_with_deadline(
8364 &mut self,
8365 transport: &mut dyn Transport,
8366 max_rounds: Option<u32>,
8367 timeout_ms: Option<u64>,
8368 ) -> SyncOutcome {
8369 if max_rounds == Some(0) {
8372 return SyncOutcome::Failed {
8373 details: None,
8374 error_code: "sync.invalid_request".into(),
8375 message: "maxRounds must be a positive integer".into(),
8376 recommended_action: None,
8377 };
8378 }
8379 if timeout_ms.is_some_and(|ms| ms == 0 || i64::try_from(ms).is_err()) {
8380 return SyncOutcome::Failed {
8381 details: None,
8382 error_code: "sync.invalid_request".into(),
8383 message: "timeoutMs must be a positive integer".into(),
8384 recommended_action: None,
8385 };
8386 }
8387 let started_at_ms = self.clock_now_ms();
8388 let rounds = max_rounds.unwrap_or(20);
8389 let mut aggregate = SyncReport::default();
8390 let mut spent = 0;
8391 let mut completed = false;
8392 while spent < rounds {
8393 if completed
8396 && timeout_ms.is_some_and(|ms| {
8397 self.clock_now_ms() >= started_at_ms.saturating_add(ms as i64)
8398 })
8399 {
8400 return SyncOutcome::BudgetExhausted(aggregate);
8401 }
8402 let idle_bound = timeout_ms.map(|ms| (ms, spent));
8403 spent += 1;
8404 let before = self.resume_tokens();
8405 match self.sync_bounded(transport, idle_bound) {
8406 SyncOutcome::Failed {
8407 error_code,
8408 message,
8409 details,
8410 recommended_action,
8411 } => {
8412 return SyncOutcome::Failed {
8413 details,
8414 error_code,
8415 message,
8416 recommended_action,
8417 };
8418 }
8419 outcome @ SyncOutcome::RealtimeUnavailable { .. } => return outcome,
8420 SyncOutcome::BudgetExhausted(_) => {
8421 unreachable!("sync() never exhausts a round budget")
8422 }
8423 SyncOutcome::Ok(report) => {
8424 completed = true;
8425 if max_rounds.is_none()
8426 && report.segment_rows_applied > 0
8427 && !report.bootstrapping.is_empty()
8428 && before != self.resume_tokens()
8429 {
8430 spent = 0;
8431 }
8432 let stop = !self.transport_enabled() || report.schema_floor.is_some();
8435 aggregate.merge(&report);
8436 if stop {
8437 return SyncOutcome::Ok(aggregate);
8438 }
8439 let more = !report.bootstrapping.is_empty()
8445 || report.commits_applied > 0
8446 || report.segment_rows_applied > 0
8447 || !report.resets.is_empty()
8448 || report.deferred_commits > 0
8449 || self.sync_needed;
8450 if !more {
8451 return SyncOutcome::Ok(aggregate);
8452 }
8453 }
8454 }
8455 }
8456 SyncOutcome::BudgetExhausted(aggregate)
8459 }
8460
8461 fn process_response(
8462 &mut self,
8463 transport: &mut dyn Transport,
8464 response: Message,
8465 meta: &RequestMeta,
8466 ) -> SyncOutcome {
8467 #[cfg(feature = "bench-internals")]
8468 let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8469 let mut report = SyncReport {
8470 pushed: meta.pushed_ids.len() as u32,
8471 ..SyncReport::default()
8472 };
8473 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8474 HashMap::new();
8475 let pushed_ids = meta
8476 .pushed_ids
8477 .iter()
8478 .map(String::as_str)
8479 .collect::<HashSet<_>>();
8480 let mut last_final_push_result_id: Option<String> = None;
8481 for frame in &response.frames {
8482 match frame {
8483 Frame::PushResultDetails {
8484 client_commit_id,
8485 entries,
8486 } => {
8487 let details = rejection_details_by_commit
8488 .entry(client_commit_id.clone())
8489 .or_default();
8490 for entry in entries {
8491 let parsed = match RejectionDetails::parse(&entry.details.0) {
8492 Ok(value) => value,
8493 Err(message) => {
8494 return SyncOutcome::Failed {
8495 details: None,
8496 error_code: "sync.invalid_request".to_owned(),
8497 message,
8498 recommended_action: None,
8499 };
8500 }
8501 };
8502 details.insert(entry.op_index, parsed);
8503 }
8504 }
8505 Frame::PushResult {
8506 client_commit_id,
8507 status,
8508 results,
8509 ..
8510 } if pushed_ids.contains(client_commit_id.as_str())
8511 && Self::push_result_is_final(*status, results) =>
8512 {
8513 last_final_push_result_id = Some(client_commit_id.clone());
8514 }
8515 _ => {}
8516 }
8517 }
8518 let mut frames = response.frames.into_iter().peekable();
8519 match frames.next() {
8520 Some(Frame::RespHeader {
8521 required_schema_version,
8522 latest_schema_version,
8523 log_epoch,
8524 reset_required,
8525 }) => {
8526 if let Some(required) = required_schema_version {
8527 let floor = SchemaFloor {
8530 required_schema_version: Some(required),
8531 latest_schema_version,
8532 };
8533 if let Err(message) = self.set_schema_floor(Some(floor.clone())) {
8534 return SyncOutcome::Failed {
8535 details: None,
8536 error_code: "storage.failed".to_owned(),
8537 message,
8538 recommended_action: None,
8539 };
8540 }
8541 report.schema_floor = Some(floor);
8542 return SyncOutcome::Ok(report);
8543 }
8544 if reset_required {
8545 if frames.next().is_some() {
8546 return SyncOutcome::Failed {
8547 details: None,
8548 error_code: "client.invalid_host_response".to_owned(),
8549 message: "log-epoch reset response contains body frames".to_owned(),
8550 recommended_action: None,
8551 };
8552 }
8553 match self.run_log_epoch_reset(&log_epoch) {
8554 Ok(resets) => {
8555 report.resets = resets;
8556 return SyncOutcome::Ok(report);
8557 }
8558 Err(message) => {
8559 return SyncOutcome::Failed {
8560 details: None,
8561 error_code: "sync.local_corrupt".to_owned(),
8562 message,
8563 recommended_action: None,
8564 };
8565 }
8566 }
8567 }
8568 let stored_epoch = match self.get_meta(LOG_EPOCH_KEY) {
8569 Ok(stored) => stored,
8570 Err(message) => {
8571 return SyncOutcome::Failed {
8572 details: None,
8573 error_code: "storage.failed".to_owned(),
8574 message,
8575 recommended_action: None,
8576 };
8577 }
8578 };
8579 if stored_epoch.as_deref() != Some(log_epoch.as_str()) {
8580 return SyncOutcome::Failed {
8581 details: None,
8582 error_code: "client.invalid_host_response".to_owned(),
8583 message: "server changed logEpoch without requiring a reset".to_owned(),
8584 recommended_action: None,
8585 };
8586 }
8587 }
8588 _ => {
8589 return SyncOutcome::Failed {
8590 details: None,
8591 error_code: "sync.invalid_request".to_owned(),
8592 message: "response does not start with RESP_HEADER".to_owned(),
8593 recommended_action: None,
8594 };
8595 }
8596 }
8597
8598 if let Err(message) = self.begin_coalesced_apply() {
8599 return SyncOutcome::Failed {
8600 details: None,
8601 error_code: "storage.failed".into(),
8602 message,
8603 recommended_action: None,
8604 };
8605 }
8606 let mut failure: Option<(String, String, Option<String>, Option<Value>)> = None;
8607 let mut live: Option<(LiveStatus, String, Vec<UnitRecord>)> = None;
8610 let mut live_ended = false;
8611 while let Some(frame) = frames.next() {
8612 match frame {
8613 frame @ Frame::PushResult { .. } => {
8614 let successful = matches!(
8615 &frame,
8616 Frame::PushResult {
8617 status: PushStatus::Applied | PushStatus::Cached,
8618 ..
8619 }
8620 );
8621 let mut results = vec![frame];
8622 if successful {
8623 while matches!(
8624 frames.peek(),
8625 Some(Frame::PushResult {
8626 status: PushStatus::Applied | PushStatus::Cached,
8627 ..
8628 })
8629 ) {
8630 if let Some(next) = frames.next() {
8631 results.push(next);
8632 }
8633 }
8634 }
8635 if self
8636 .handle_push_results(
8637 &results,
8638 &pushed_ids,
8639 &rejection_details_by_commit,
8640 &mut report,
8641 last_final_push_result_id.as_deref(),
8642 )
8643 .is_err()
8644 {
8645 failure = Some((
8646 "client.outcome_persistence_failed".to_owned(),
8647 "local commit outcome could not be persisted".to_owned(),
8648 None,
8649 None,
8650 ));
8651 break;
8652 }
8653 }
8654 Frame::PushResultDetails { .. } => {}
8655 Frame::SubStart {
8656 id,
8657 status,
8658 reason_code,
8659 bootstrap,
8660 fenced,
8661 } => {
8662 let mut body = Vec::new();
8665 let mut sub_end: Option<(i64, Option<String>)> = None;
8666 let mut error: Option<Frame> = None;
8667 for inner in frames.by_ref() {
8668 match inner {
8669 Frame::SubEnd {
8670 next_cursor,
8671 bootstrap_state,
8672 } => {
8673 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8674 break;
8675 }
8676 inner @ Frame::Error { .. } => {
8677 error = Some(inner);
8678 break;
8679 }
8680 Frame::Unknown { .. } => {}
8681 other => body.push(other),
8682 }
8683 }
8684 let ended = sub_end.is_some();
8685 let section = SectionStart {
8686 id,
8687 status,
8688 reason_code,
8689 bootstrap,
8690 fenced,
8691 };
8692 match self.process_section(
8693 transport,
8694 §ion,
8695 body,
8696 sub_end,
8697 meta,
8698 &mut report,
8699 ) {
8700 Err(SectionError::Abort(code, message)) => {
8701 failure = Some((code, message, None, None));
8702 break;
8703 }
8704 Err(SectionError::Transport(error)) => {
8705 failure = Some((error.code, error.message, None, error.details));
8706 break;
8707 }
8708 _ => {}
8709 }
8710 if ended {
8711 self.progress.update(|p| {
8713 p.unit_id = None;
8714 p.table = None;
8715 p.segment_id = None;
8716 });
8717 }
8718 match error {
8719 Some(Frame::Error {
8720 code,
8721 message,
8722 recommended_action,
8723 details,
8724 ..
8725 }) => {
8726 failure = Some(Self::error_frame_failure(
8727 code,
8728 message,
8729 recommended_action,
8730 details,
8731 ));
8732 break;
8733 }
8734 Some(_) => unreachable!("only an ERROR closes a section early"),
8735 None if !ended => {
8736 failure = Some((
8737 "sync.invalid_request".to_owned(),
8738 "unit section without SUB_END".to_owned(),
8739 None,
8740 None,
8741 ));
8742 break;
8743 }
8744 None => {}
8745 }
8746 }
8747 Frame::LiveStart {
8748 status,
8749 reason_code,
8750 } => {
8751 let receivers = match self.stored(load_units(&self.conn)) {
8752 Ok(units) => units
8753 .into_iter()
8754 .filter(|unit| {
8755 unit.status == UnitStatus::Active
8756 && unit.echo.is_some()
8757 && !unit.bootstrapping()
8758 })
8759 .collect(),
8760 Err(message) => {
8761 failure = Some(("storage.failed".to_owned(), message, None, None));
8762 break;
8763 }
8764 };
8765 live = Some((status, reason_code, receivers));
8766 }
8767 Frame::Commit {
8768 commit_seq,
8769 tables,
8770 changes,
8771 ..
8772 } => {
8773 let Some((LiveStatus::Active, _, receivers)) = &live else {
8774 continue;
8775 };
8776 let receivers = receivers.clone();
8777 match self.apply_live_commit(
8778 &receivers,
8779 &tables,
8780 changes,
8781 commit_seq,
8782 AppliedServerCommitSource::Pull,
8783 ) {
8784 Ok(applied) => report.commits_applied += u32::from(applied),
8785 Err(error) => {
8786 failure = Some((error.code, error.message, None, error.details));
8787 break;
8788 }
8789 }
8790 }
8791 Frame::LiveEnd {
8792 through_seq,
8793 echo_digest,
8794 } => {
8795 let Some((status, reason_code, _)) = live.take() else {
8796 continue;
8797 };
8798 live_ended = true;
8799 live = Some((status, String::new(), Vec::new()));
8800 if let Err(message) = self.finish_live_section(
8801 status,
8802 &reason_code,
8803 through_seq,
8804 echo_digest,
8805 &meta.registration_digest,
8806 &mut report,
8807 ) {
8808 failure = Some(("storage.failed".to_owned(), message, None, None));
8809 break;
8810 }
8811 }
8812 Frame::Lease {
8813 lease_id,
8814 expires_at_ms,
8815 } => {
8816 if let Err(message) = self.set_lease_state(Some(LeaseState {
8819 lease_id: Some(lease_id),
8820 expires_at_ms: Some(expires_at_ms),
8821 error_code: None,
8822 })) {
8823 failure = Some(("storage.failed".to_owned(), message, None, None));
8824 break;
8825 }
8826 }
8827 Frame::Error {
8828 code,
8829 message,
8830 recommended_action,
8831 details,
8832 ..
8833 } => {
8834 failure = Some(Self::error_frame_failure(
8837 code,
8838 message,
8839 recommended_action,
8840 details,
8841 ));
8842 break;
8843 }
8844 Frame::Unknown { .. } => {}
8845 _ => {
8846 failure = Some((
8847 "sync.invalid_request".to_owned(),
8848 "unexpected frame in response".to_owned(),
8849 None,
8850 None,
8851 ));
8852 break;
8853 }
8854 }
8855 }
8856
8857 let applied = match failure {
8858 Some(failure) => Err(failure),
8859 None => self
8864 .rebuild_overlay_if_dirty()
8865 .and_then(|()| self.drain_pending_evictions())
8866 .map_err(|message| ("storage.failed".to_owned(), message, None, None)),
8867 };
8868 if let Err(message) = self.finish_coalesced_apply() {
8871 return SyncOutcome::Failed {
8872 details: None,
8873 error_code: "storage.failed".into(),
8874 message,
8875 recommended_action: None,
8876 };
8877 }
8878 if let Err((error_code, message, recommended_action, details)) = applied {
8879 return SyncOutcome::Failed {
8880 details,
8881 error_code,
8882 message,
8883 recommended_action,
8884 };
8885 }
8886 let catching_up = match self.stored(load_units(&self.conn)) {
8887 Ok(units) => units
8888 .into_iter()
8889 .filter(|unit| unit.status == UnitStatus::Active && unit.cursor.is_some())
8890 .map(|unit| unit.id)
8891 .collect::<Vec<_>>(),
8892 Err(message) => {
8893 return SyncOutcome::Failed {
8894 details: None,
8895 error_code: "storage.failed".into(),
8896 message,
8897 recommended_action: None,
8898 }
8899 }
8900 };
8901 let status = live.map(|(status, _, _)| status);
8905 if !live_ended || status != Some(LiveStatus::Active) || !catching_up.is_empty() {
8906 self.set_sync_needed(true, status == Some(LiveStatus::Unregistered));
8907 }
8908 self.ack_after_pull(transport);
8909 if self.upgrading && catching_up.is_empty() {
8912 self.set_upgrading(false);
8913 }
8914 report.bootstrapping = catching_up;
8915 SyncOutcome::Ok(report)
8916 }
8917
8918 fn error_frame_failure(
8920 code: String,
8921 message: String,
8922 recommended_action: String,
8923 details: Option<RawJson>,
8924 ) -> (String, String, Option<String>, Option<Value>) {
8925 let recommended_action = (!recommended_action.is_empty()).then_some(recommended_action);
8926 let details = details
8927 .and_then(|details| serde_json::from_str::<Value>(&details.0).ok())
8928 .filter(Value::is_object);
8929 (code, message, recommended_action, details)
8930 }
8931
8932 fn finish_live_section(
8937 &mut self,
8938 status: LiveStatus,
8939 reason_code: &str,
8940 through_seq: i64,
8941 echo_digest: Option<[u8; 32]>,
8942 registration_digest: &str,
8943 report: &mut SyncReport,
8944 ) -> Result<(), String> {
8945 let rollback = self.capture();
8946 let resets = report.resets.len();
8947 self.begin_observation("syncular_live_end")?;
8948 let finished = (|| {
8949 let mut batch = ChangeAccumulator::default();
8950 let before = self.stored(replica_cursor(&self.conn))?;
8951 let mut cursor = if status == LiveStatus::Unregistered {
8952 before
8953 } else {
8954 -1
8955 };
8956 let joined = before.max(through_seq);
8957 for unit in self.stored(load_units(&self.conn))? {
8958 if unit.status != UnitStatus::Active || unit.bootstrapping() {
8959 continue;
8960 }
8961 match status {
8962 LiveStatus::Reset if unit.cursor.is_none() => {
8965 for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
8966 batch.window(&base_key, &value);
8967 }
8968 report.resets.push(unit.id.clone());
8969 self.stored(save_unit(
8970 &self.conn,
8971 &UnitRecord {
8972 reason_code: Some(reason_code.to_owned()),
8973 cursor: Some(-1),
8974 seeded: Vec::new(),
8975 ..unit
8976 },
8977 ))?;
8978 }
8979 LiveStatus::Active => {
8981 let next = unit.cursor.unwrap_or(joined).max(through_seq);
8982 if next >= joined {
8983 cursor = joined;
8984 }
8985 if unit.cursor.is_some() {
8986 self.stored(save_unit(
8987 &self.conn,
8988 &UnitRecord {
8989 cursor: (next < joined).then_some(next),
8990 ..unit
8991 },
8992 ))?;
8993 }
8994 }
8995 _ => {}
8996 }
8997 }
8998 self.set_meta(REPLICA_CURSOR_KEY, &cursor.to_string())?;
8999 self.set_meta(
9000 ECHO_DIGEST_KEY,
9001 &echo_digest
9002 .map(|digest| bytes_to_hex(&digest))
9003 .unwrap_or_default(),
9004 )?;
9005 if status == LiveStatus::Unregistered {
9006 self.delete_meta(REGISTERED_DIGEST_KEY)?;
9007 } else {
9008 self.set_meta(REGISTERED_DIGEST_KEY, registration_digest)?;
9009 }
9010 self.finish_observation("syncular_live_end", batch)
9011 })();
9012 if finished.is_err() {
9013 self.rollback_observation("syncular_live_end");
9014 self.restore(rollback);
9015 report.resets.truncate(resets);
9016 }
9017 finished
9018 }
9019
9020 fn handle_push_results(
9023 &mut self,
9024 frames: &[Frame],
9025 pushed_ids: &HashSet<&str>,
9026 rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
9027 report: &mut SyncReport,
9028 last_final_push_result_id: Option<&str>,
9029 ) -> Result<(), String> {
9030 self.begin_observation("syncular_push_results")?;
9031 let rollback = self.capture();
9032 let conflict_count = self.conflicts.len();
9033 let applied_count = report.applied.len();
9034 let rejected_count = report.rejected.len();
9035 let retryable_count = report.retryable.len();
9036 let mut drained = Vec::new();
9037 let mut batch = ChangeAccumulator::default();
9038 let persisted = (|| {
9039 for frame in frames {
9040 let Frame::PushResult {
9041 client_commit_id,
9042 status,
9043 results,
9044 commit_seq,
9045 } = frame
9046 else {
9047 return Err("unexpected acknowledgement frame".to_owned());
9048 };
9049 if !pushed_ids.contains(client_commit_id.as_str()) {
9050 continue;
9051 }
9052 let Some(index) = self
9053 .outbox
9054 .iter()
9055 .position(|commit| commit.client_commit_id == *client_commit_id)
9056 else {
9057 continue;
9058 };
9059 let status = *status;
9060 let rejection_details = rejection_details_by_commit.get(client_commit_id);
9061 let operations = self.outbox[index].ops.clone();
9062 match status {
9063 PushStatus::Applied | PushStatus::Cached => {
9064 let sequence =
9067 commit_seq.ok_or_else(|| "ACK lacks a commit sequence".to_owned())?;
9068 let affected: HashSet<_> = operations
9069 .iter()
9070 .map(|op| (op.table.as_str(), op.row_id.as_str()))
9071 .collect();
9072 let prior = self.acknowledged_operations(Some(&affected), None)?;
9073 for (op_index, operation) in operations.iter().enumerate() {
9074 let delivered: bool = self.conn.query_row("SELECT EXISTS(SELECT 1 FROM _syncular_row_deliveries WHERE tbl=?1 AND id=?2 AND commit_seq>=?3)", rusqlite::params![operation.table, operation.row_id, sequence], |row| row.get(0))
9075 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9076 if delivered {
9077 continue;
9078 }
9079 let table = self
9080 .schema
9081 .table(&operation.table)
9082 .ok_or_else(|| "sync.unknown_table".to_owned())?;
9083 let mut intended: Map<String, Value> = self
9084 .stored_row(table, &operation.row_id, true)?
9085 .map(|(row, _)| {
9086 table
9087 .columns
9088 .iter()
9089 .zip(&row)
9090 .map(|(column, value)| {
9091 (column.name.clone(), column_value_to_json(value))
9092 })
9093 .collect()
9094 })
9095 .unwrap_or_default();
9096 let mut scope_image = intended.clone();
9097 for op in prior
9098 .iter()
9099 .chain(
9100 self.failed_commits
9101 .iter()
9102 .flat_map(|(commit, _)| &commit.ops),
9103 )
9104 .chain(
9105 self.outbox
9106 .iter()
9107 .take(index)
9108 .flat_map(|commit| &commit.ops),
9109 )
9110 .chain(operations.iter().take(op_index + 1))
9111 {
9112 if op.table != operation.table || op.row_id != operation.row_id {
9113 continue;
9114 }
9115 if let Some(values) = &op.values {
9116 intended.extend(values.clone());
9117 } else {
9118 scope_image = intended.clone();
9119 intended.clear();
9120 }
9121 }
9122 self.conn.execute("INSERT INTO _syncular_acked_rows(commit_id,idx,tbl,id,commit_seq,op_json,intent_json) VALUES (?1,?2,?3,?4,?5,?6,?7)",
9123 rusqlite::params![client_commit_id, op_index as i64, operation.table, operation.row_id, sequence,
9124 serde_json::to_string(&CommitOperation::from(operation)).map_err(|_| "sync.local_corrupt".to_owned())?,
9125 serde_json::to_string(if operation.upsert { &intended } else { &scope_image }).map_err(|_| "sync.local_corrupt".to_owned())?])
9126 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9127 }
9128 let journal_results = results
9129 .iter()
9130 .map(|result| {
9131 let op_index = match result {
9132 OpResult::Applied { op_index }
9133 | OpResult::Conflict { op_index, .. }
9134 | OpResult::Error { op_index, .. } => *op_index,
9135 };
9136 CommitOperationOutcome::Applied { op_index }
9137 })
9138 .collect::<Vec<_>>();
9139 let outcome_status = if status == PushStatus::Applied {
9140 CommitOutcomeStatus::Applied
9141 } else {
9142 CommitOutcomeStatus::Cached
9143 };
9144 let persisted = self
9145 .persist_commit_outcome(
9146 client_commit_id,
9147 outcome_status,
9148 &journal_results,
9149 None,
9150 )
9151 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9152 persisted?;
9153 self.sync_needed = true;
9154 batch.status = true;
9155 batch.outcomes = true;
9156 }
9157 PushStatus::Rejected => {
9158 if results.iter().any(|result| {
9159 matches!(
9160 result,
9161 OpResult::Error {
9162 code,
9163 retryable: true,
9164 ..
9165 } if code == "sync.idempotency_cache_miss"
9166 )
9167 }) {
9168 report.retryable.push(client_commit_id.to_owned());
9171 continue;
9172 }
9173
9174 let mut journal_results = Vec::with_capacity(results.len());
9175 let mut conflicts = Vec::new();
9176 let mut rejections = Vec::new();
9177 for result in results {
9178 match result {
9179 OpResult::Applied { op_index } => {
9180 journal_results.push(CommitOperationOutcome::Applied {
9181 op_index: *op_index,
9182 });
9183 }
9184 OpResult::Conflict {
9185 op_index,
9186 code,
9187 message,
9188 server_version,
9189 server_row,
9190 conflict_columns: conflict_columns_bitmap,
9191 } => {
9192 let operation = operations
9193 .get(*op_index as usize)
9194 .map(CommitOperation::from);
9195 let (table, row_id) = operation
9196 .as_ref()
9197 .map(|op| (op.table.clone(), op.row_id.clone()))
9198 .unwrap_or_default();
9199 let server_row_json = self
9200 .schema
9201 .table(&table)
9202 .and_then(|t| {
9203 decode_row_bytes(t, server_row, &self.encryption)
9204 .ok()
9205 .map(|(row, _)| (t, row))
9206 })
9207 .map(|(t, row)| {
9208 let mut map = Map::new();
9209 for (i, column) in t.columns.iter().enumerate() {
9210 map.insert(
9211 column.name.clone(),
9212 column_value_to_json(
9213 row.get(i).unwrap_or(&None),
9214 ),
9215 );
9216 }
9217 map
9218 })
9219 .unwrap_or_default();
9220 let conflict_columns = self
9223 .schema
9224 .table(&table)
9225 .map(|t| {
9226 t.columns
9227 .iter()
9228 .enumerate()
9229 .filter(|(index, _)| {
9230 conflict_columns_bitmap
9231 .get(index / 8)
9232 .is_some_and(|byte| {
9233 byte & (1 << (index % 8)) != 0
9234 })
9235 })
9236 .map(|(_, column)| column.name.clone())
9237 .collect::<Vec<_>>()
9238 })
9239 .unwrap_or_default();
9240 let conflict = ConflictRecord {
9241 client_commit_id: client_commit_id.to_owned(),
9242 op_index: *op_index,
9243 table,
9244 row_id,
9245 code: code.clone(),
9246 message: message.clone(),
9247 server_version: *server_version,
9248 server_row: server_row_json,
9249 conflict_columns,
9250 operation,
9251 };
9252 journal_results.push(CommitOperationOutcome::Conflict {
9253 conflict: conflict.clone(),
9254 });
9255 conflicts.push(conflict);
9256 }
9257 OpResult::Error {
9258 op_index,
9259 code,
9260 message,
9261 retryable,
9262 } => {
9263 let rejection = RejectionRecord {
9264 client_commit_id: client_commit_id.to_owned(),
9265 op_index: *op_index,
9266 code: code.clone(),
9267 message: message.clone(),
9268 retryable: *retryable,
9269 details: rejection_details
9270 .and_then(|details| details.get(op_index))
9271 .cloned(),
9272 operation: operations
9273 .get(*op_index as usize)
9274 .map(CommitOperation::from),
9275 };
9276 journal_results.push(CommitOperationOutcome::Error {
9277 rejection: rejection.clone(),
9278 });
9279 rejections.push(rejection);
9280 }
9281 }
9282 }
9283 if self.retain_failed_commits {
9284 self.retain_failed_commit(client_commit_id, &operations)?;
9285 }
9286 let outcome_status = if conflicts.is_empty() {
9287 CommitOutcomeStatus::Rejected
9288 } else {
9289 CommitOutcomeStatus::Conflict
9290 };
9291 let persisted = self
9292 .persist_commit_outcome(
9293 client_commit_id,
9294 outcome_status,
9295 &journal_results,
9296 Some(&operations),
9297 )
9298 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
9299 persisted?;
9300 batch.conflicts = !conflicts.is_empty();
9301 batch.rejections = !rejections.is_empty();
9302 batch.status = true;
9303 batch.outcomes = true;
9304 self.conflicts.extend(conflicts);
9305 self.rejections.extend(rejections);
9306 }
9307 }
9308 drained.push(self.outbox.remove(index));
9309 if batch.status {
9310 for operation in &operations {
9311 if !self.record_row_scopes(
9312 &mut batch,
9313 &operation.table,
9314 &operation.row_id,
9315 false,
9316 ) {
9317 batch.table(&operation.table);
9318 }
9319 }
9320 }
9321 match status {
9322 PushStatus::Applied | PushStatus::Cached => {
9323 report.applied.push(client_commit_id.to_owned())
9324 }
9325 PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
9326 }
9327 }
9328 if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
9329 self.prune_commit_outcomes()?;
9330 }
9331 self.reconcile_rows(&drained)?;
9332 self.rebuild_overlay_if_dirty()?;
9333 self.finish_observation("syncular_push_results", batch)
9334 })();
9335 if let Err(error) = persisted {
9336 self.rollback_observation("syncular_push_results");
9337 self.restore(rollback);
9338 report.applied.truncate(applied_count);
9339 report.rejected.truncate(rejected_count);
9340 report.retryable.truncate(retryable_count);
9341 return Err(error);
9342 }
9343 if !rollback.sync_needed && self.sync_needed {
9344 self.sync_intent_queue.push_back(SyncIntent::Interactive);
9345 }
9346 report.conflicts += (self.conflicts.len() - conflict_count) as u32;
9347 Ok(())
9348 }
9349
9350 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
9351 status != PushStatus::Rejected
9352 || !results.iter().any(|result| {
9353 matches!(
9354 result,
9355 OpResult::Error {
9356 code,
9357 retryable: true,
9358 ..
9359 } if code == "sync.idempotency_cache_miss"
9360 )
9361 })
9362 }
9363
9364 fn process_section(
9372 &mut self,
9373 transport: &mut dyn Transport,
9374 start: &SectionStart,
9375 body: Vec<Frame>,
9376 sub_end: Option<(i64, Option<String>)>,
9377 meta: &RequestMeta,
9378 report: &mut SyncReport,
9379 ) -> Result<(), SectionError> {
9380 let storage = |message| SectionError::Abort("storage.failed".to_owned(), message);
9381 let unit = self
9382 .stored(get_unit(&self.conn, &start.id))
9383 .map_err(storage)?
9384 .filter(|unit| unit.status == UnitStatus::Active);
9385 self.progress.update(|p| {
9387 p.phase = ProgressPhase::Import;
9388 p.unit_id = unit.as_ref().map(|unit| unit.id.clone());
9389 p.table = None;
9390 p.segment_id = None;
9391 p.bytes_received = 0;
9392 p.bytes_total = None;
9393 p.rows_processed = 0;
9394 p.rows_total = None;
9395 p.import_write_ms = None;
9396 p.import_commit_ms = None;
9397 });
9398 let Some(unit) = unit else {
9399 return Ok(());
9400 };
9401 let echo: Vec<EchoRecord> = start
9402 .fenced
9403 .iter()
9404 .map(|record| EchoRecord {
9405 table: record.table.clone(),
9406 status: record.status.name().to_owned(),
9407 scopes: record.scopes.iter().cloned().collect(),
9408 })
9409 .collect();
9410 let applied_record = self
9412 .stored(applied_rows(&self.conn, &unit.id))
9413 .map_err(storage)?;
9414 let mut seeded: BTreeSet<String> = if unit.bootstrap_state.is_none() {
9416 BTreeSet::new()
9417 } else {
9418 unit.seeded.iter().cloned().collect()
9419 };
9420 let applied = (|| {
9421 for frame in body {
9422 match frame {
9423 Frame::SegmentInline { payload } => {
9424 let segment = decode_rows_segment(&payload).map_err(|e| {
9425 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9426 })?;
9427 self.progress.update(|p| {
9428 p.phase = ProgressPhase::Import;
9429 p.unit_id = Some(unit.id.clone());
9430 p.table = Some(segment.table.clone());
9431 p.segment_id = None;
9432 p.bytes_received = payload.len() as u64;
9433 p.bytes_total = Some(payload.len() as u64);
9434 p.rows_processed = 0;
9435 p.rows_total =
9436 Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9437 p.import_write_ms = None;
9438 p.import_commit_ms = None;
9439 });
9440 let (pair, clear, effective) =
9441 self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9442 report.segment_rows_applied +=
9443 self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9444 seeded.insert(segment.table);
9445 }
9446 Frame::SegmentRef {
9447 segment_id,
9448 byte_length,
9449 media_type,
9450 table,
9451 row_count,
9452 as_of_commit_seq,
9453 scope_digest,
9454 row_cursor,
9455 next_row_cursor,
9456 url,
9457 url_expires_at_ms,
9458 ..
9459 } => {
9460 let advertised = match media_type {
9463 MediaType::Rows => {
9464 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9465 || meta.accept & ACCEPT_INLINE_ROWS != 0
9466 }
9467 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9468 };
9469 if !advertised {
9470 return Err(SectionError::Abort(
9471 "sync.invalid_request".to_owned(),
9472 format!(
9473 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9474 media_type.name()
9475 ),
9476 ));
9477 }
9478 let resumed = applied_record.get(&segment_id).copied();
9481 if resumed.is_some() {
9482 seeded.insert(table.clone());
9483 }
9484 if resumed == Some(row_count) {
9485 continue;
9486 }
9487 if meta
9488 .deferred
9489 .contains(&(unit.id.clone(), segment_id.clone()))
9490 {
9491 return Err(SectionError::Deferred);
9492 }
9493 self.progress.update(|p| {
9494 p.phase = ProgressPhase::Download;
9495 p.unit_id = Some(unit.id.clone());
9496 p.table = Some(table.clone());
9497 p.segment_id = Some(segment_id.clone());
9498 p.bytes_received = 0;
9499 p.bytes_total = u64::try_from(byte_length).ok();
9500 p.rows_processed = 0;
9501 p.rows_total = u64::try_from(row_count).ok();
9502 p.import_write_ms = None;
9503 p.import_commit_ms = None;
9504 });
9505 let observer = self.progress.clone();
9506 let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9507 let bytes = if let Some(url) = url {
9508 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9513 return Err(SectionError::Abort(
9514 "sync.invalid_request".to_owned(),
9515 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9516 .to_owned(),
9517 ));
9518 }
9519 if url_expires_at_ms.is_some_and(|exp| {
9521 exp <= transport
9522 .segment_fetch_started_at(&url)
9523 .unwrap_or_else(|| self.clock_now_ms())
9524 }) {
9525 return Err(SectionError::Abort(
9526 "sync.segment_expired".to_owned(),
9527 format!(
9528 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9529 ),
9530 ));
9531 }
9532 transport
9533 .fetch_url(
9534 &url,
9535 u64::try_from(byte_length).unwrap_or(0),
9536 &mut on_progress,
9537 )
9538 .map_err(|e| self.segment_fetch_failed(e))?
9539 } else {
9540 let requested_scopes_json =
9542 canonical_scope_json(&unit_scope_lists(&unit.scopes));
9543 transport
9544 .download_segment(
9545 &SegmentRequest {
9546 segment_id: segment_id.clone(),
9547 table: table.clone(),
9548 requested_scopes_json,
9549 byte_length: u64::try_from(byte_length).unwrap_or(0),
9550 },
9551 &mut on_progress,
9552 )
9553 .map_err(|e| {
9554 if e.code == "sync.forbidden" {
9557 SectionError::Refused
9558 } else {
9559 self.segment_fetch_failed(e)
9560 }
9561 })?
9562 };
9563 if bytes.len() as i64 > byte_length {
9567 self.schedule_background_retry();
9568 return Err(SectionError::Abort(
9569 "sync.invalid_request".to_owned(),
9570 "segment exceeds its byteLength".to_owned(),
9571 ));
9572 }
9573 on_progress(bytes.len() as u64);
9574 let digest = Sha256::digest(&bytes);
9575 let expected = segment_id
9576 .strip_prefix("sha256:")
9577 .unwrap_or(segment_id.as_str());
9578 if bytes_to_hex(&digest) != expected {
9579 self.schedule_background_retry();
9581 return Err(SectionError::Abort(
9582 "sync.invalid_request".to_owned(),
9583 "segment bytes do not match the content address (§5.1)".to_owned(),
9584 ));
9585 }
9586 self.progress.update(|p| p.phase = ProgressPhase::Import);
9587 if media_type == MediaType::Sqlite {
9588 if row_cursor.is_some() || next_row_cursor.is_some() {
9591 return Err(SectionError::Abort(
9592 "sync.invalid_request".to_owned(),
9593 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9594 .to_owned(),
9595 ));
9596 }
9597 let (pair, clear, effective) =
9598 self.segment_pair(&unit, &echo, &seeded, &table)?;
9599 report.segment_rows_applied += self.apply_sqlite_segment(
9600 &pair,
9601 &bytes,
9602 clear,
9603 &effective,
9604 &unit.id,
9605 &segment_id,
9606 resumed.unwrap_or(0),
9607 row_count,
9608 as_of_commit_seq,
9609 &scope_digest,
9610 )?;
9611 seeded.insert(table);
9612 } else {
9613 let segment = decode_rows_segment(&bytes).map_err(|e| {
9614 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9615 })?;
9616 let (pair, clear, effective) =
9617 self.segment_pair(&unit, &echo, &seeded, &segment.table)?;
9618 report.segment_rows_applied +=
9619 self.apply_segment(&pair, &segment, clear, &effective, &unit.id)?;
9620 seeded.insert(segment.table);
9621 self.stored(record_applied_rows(
9623 &self.conn,
9624 &unit.id,
9625 &segment_id,
9626 row_count,
9627 ))
9628 .map_err(storage)?;
9629 }
9630 }
9631 _ => {
9632 return Err(SectionError::Abort(
9633 "sync.invalid_request".to_owned(),
9634 "unexpected frame inside a unit section".to_owned(),
9635 ));
9636 }
9637 }
9638 }
9639 Ok(())
9640 })();
9641 match applied {
9642 Ok(()) => {}
9643 Err(SectionError::Refused | SectionError::Deferred) => return Ok(()),
9645 Err(SectionError::FailClosed) => {
9646 self.begin_observation("syncular_section_failure")
9649 .map_err(storage)?;
9650 let failed = (|| {
9651 let mut batch = ChangeAccumulator::default();
9652 for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9653 batch.window(&base_key, &value);
9654 }
9655 batch.status = true;
9656 self.stored(save_unit(
9657 &self.conn,
9658 &UnitRecord {
9659 status: UnitStatus::Failed,
9660 reason_code: Some("sync.scope_revoked".to_owned()),
9661 ..unit.clone()
9662 },
9663 ))?;
9664 self.finish_observation("syncular_section_failure", batch)
9665 })();
9666 if let Err(message) = failed {
9667 self.rollback_observation("syncular_section_failure");
9668 return Err(storage(message));
9669 }
9670 report.failed.push(unit.id);
9671 return Ok(());
9672 }
9673 Err(error) => return Err(error),
9674 }
9675 let Some((next_cursor, bootstrap_state)) = sub_end else {
9676 return Ok(());
9677 };
9678 let rollback = self.capture();
9679 let counts = (
9680 report.resets.len(),
9681 report.revoked.len(),
9682 report.failed.len(),
9683 );
9684 self.begin_observation("syncular_sub_end")
9685 .map_err(storage)?;
9686 let finished = self
9687 .finish_section(start, echo, next_cursor, bootstrap_state, seeded, report)
9688 .and_then(|batch| {
9689 self.rebuild_overlay_if_dirty()?;
9690 self.finish_observation("syncular_sub_end", batch)
9691 });
9692 if let Err(message) = finished {
9693 self.rollback_observation("syncular_sub_end");
9694 self.restore(rollback);
9695 report.resets.truncate(counts.0);
9696 report.revoked.truncate(counts.1);
9697 report.failed.truncate(counts.2);
9698 return Err(storage(message));
9699 }
9700 Ok(())
9701 }
9702
9703 #[allow(clippy::type_complexity)]
9706 fn segment_pair(
9707 &self,
9708 unit: &UnitRecord,
9709 echo: &[EchoRecord],
9710 seeded: &BTreeSet<String>,
9711 name: &str,
9712 ) -> Result<(TableSchema, bool, Vec<(String, Vec<String>)>), SectionError> {
9713 let table = self.schema.table(name).cloned().ok_or_else(|| {
9714 SectionError::Abort(
9715 "sync.schema_mismatch".to_owned(),
9716 format!("segment table {name:?} is not in the client schema"),
9717 )
9718 })?;
9719 let effective = covers(&table, &unit.scopes)
9720 .then(|| pair_scopes(&unit.scopes, name, Some(echo)))
9721 .flatten()
9722 .ok_or_else(|| {
9723 SectionError::Abort(
9724 "sync.invalid_request".to_owned(),
9725 "a segment names no active pair of its unit".to_owned(),
9726 )
9727 })?;
9728 Ok((table, !seeded.contains(name), effective))
9729 }
9730
9731 fn finish_section(
9738 &mut self,
9739 start: &SectionStart,
9740 echo: Vec<EchoRecord>,
9741 next_cursor: i64,
9742 bootstrap_state: Option<String>,
9743 seeded: BTreeSet<String>,
9744 report: &mut SyncReport,
9745 ) -> Result<ChangeAccumulator, String> {
9746 #[cfg(feature = "bench-internals")]
9747 let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9748 let mut batch = ChangeAccumulator::default();
9749 let Some(unit) = self
9751 .stored(get_unit(&self.conn, &start.id))?
9752 .filter(|unit| unit.status == UnitStatus::Active)
9753 else {
9754 return Ok(batch);
9755 };
9756 for (base_key, value) in self.stored(window_refs(&self.conn, &unit.id))? {
9757 batch.window(&base_key, &value);
9758 }
9759 if start.status == SubStatus::Reset {
9760 report.resets.push(unit.id.clone());
9763 self.stored(save_unit(
9764 &self.conn,
9765 &UnitRecord {
9766 reason_code: Some(start.reason_code.clone()),
9767 cursor: Some(-1),
9768 bootstrap_state: None,
9769 seeded: Vec::new(),
9770 ..unit
9771 },
9772 ))?;
9773 return Ok(batch);
9774 }
9775 let revoked = start.status == SubStatus::Revoked;
9776 let mut units = None;
9777 let mut failed = false;
9778 let mut rearmed = false;
9779 for table in self.schema.tables.clone() {
9780 if !covers(&table, &unit.scopes) {
9781 continue;
9782 }
9783 let last = unit
9784 .echo
9785 .as_deref()
9786 .and_then(|last| pair_scopes(&unit.scopes, &table.name, Some(last)));
9787 let next = (!revoked)
9788 .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9789 .flatten();
9790 rearmed |= unit.echo.is_some() && last.is_none() && next.is_some();
9792 if let (Some(last), None) = (&last, &next) {
9795 let units = match units {
9796 Some(ref units) => units,
9797 None => units.insert(self.stored(load_units(&self.conn))?),
9798 };
9799 let owners = unit_owners(units, &table, &unit.id);
9800 failed |= !self.purge_pair(&mut batch, &table, last, &owners)?;
9801 }
9802 }
9803 if revoked || failed {
9804 if revoked {
9805 report.revoked.push(unit.id.clone());
9806 }
9807 if failed {
9808 report.failed.push(unit.id.clone());
9809 }
9810 batch.status = true;
9811 self.stored(save_unit(
9812 &self.conn,
9813 &UnitRecord {
9814 status: if failed {
9815 UnitStatus::Failed
9816 } else {
9817 UnitStatus::Revoked
9818 },
9819 reason_code: Some(if failed {
9820 "sync.scope_revoked".to_owned()
9821 } else {
9822 start.reason_code.clone()
9823 }),
9824 cursor: Some(-1),
9825 bootstrap_state: None,
9826 seeded: Vec::new(),
9827 echo: None,
9828 ..unit
9829 },
9830 ))?;
9831 return Ok(batch);
9832 }
9833 let completed = start.bootstrap && !rearmed && bootstrap_state.is_none();
9834 let (cursor, bootstrap_state, seeded) = if rearmed {
9835 (Some(-1), None, Vec::new())
9836 } else if start.bootstrap {
9837 let seeded = if bootstrap_state.is_none() {
9838 Vec::new()
9839 } else {
9840 seeded.into_iter().collect()
9841 };
9842 (Some(next_cursor), bootstrap_state, seeded)
9843 } else {
9844 (
9845 unit.cursor,
9846 unit.bootstrap_state.clone(),
9847 unit.seeded.clone(),
9848 )
9849 };
9850 self.stored(save_unit(
9851 &self.conn,
9852 &UnitRecord {
9853 reason_code: None,
9854 echo: Some(echo.clone()),
9855 cursor,
9856 bootstrap_state,
9857 seeded,
9858 ..unit.clone()
9859 },
9860 ))?;
9861 if !completed {
9862 return Ok(batch);
9863 }
9864 for table in self.schema.tables.clone() {
9867 let Some(effective) = covers(&table, &unit.scopes)
9868 .then(|| pair_scopes(&unit.scopes, &table.name, Some(&echo)))
9869 .flatten()
9870 else {
9871 continue;
9872 };
9873 let mut statement = self
9874 .conn
9875 .prepare("SELECT id,intent_json FROM _syncular_acked_rows WHERE tbl=?1 AND commit_seq<=?2")
9876 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9877 let rows = statement
9878 .query_map(rusqlite::params![table.name, next_cursor], |row| {
9879 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
9880 })
9881 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9882 let mut covered = HashSet::new();
9883 for row in rows {
9884 let (id, intent) =
9885 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9886 let values: Map<String, Value> =
9887 serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
9888 if owns_local_row(&table, &effective, &[], &values) {
9889 covered.insert(id);
9890 }
9891 }
9892 drop(statement);
9893 for id in covered {
9894 self.conn
9895 .execute(
9896 "DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3",
9897 rusqlite::params![table.name, id, next_cursor],
9898 )
9899 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
9900 self.overlay_dirty.table(&table.name);
9901 batch.table(&table.name);
9902 }
9903 }
9904 Ok(batch)
9905 }
9906
9907 fn purge_pair(
9913 &mut self,
9914 batch: &mut ChangeAccumulator,
9915 table: &TableSchema,
9916 last: &[(String, Vec<String>)],
9917 owners: &[Vec<(String, Vec<String>)>],
9918 ) -> Result<bool, String> {
9919 if local_scope_predicate(table, last, owners).is_err() {
9920 return Ok(false);
9921 }
9922 for commit in self.drop_failed_commits(|name, values| {
9923 name == table.name && owns_local_row(table, last, owners, values)
9924 })? {
9925 batch.conflicts = true;
9926 batch.rejections = true;
9927 batch.outcomes = true;
9928 for operation in commit.ops {
9929 batch.table(&operation.table);
9930 }
9931 }
9932 if !self.purge_scope_rows(&table.name, last, owners)? {
9933 return Err("storage.failed: the revoked scope purge failed".to_owned());
9934 }
9935 self.record_scope_map(batch, &table.name, last);
9936 if self.drop_doomed_outbox(table, last, owners)? {
9937 batch.status = true;
9938 batch.rejections = true;
9939 batch.outcomes = true;
9940 }
9941 self.delete_unreferenced_cached_blobs()?;
9943 Ok(true)
9944 }
9945
9946 fn apply_live_commit(
9951 &mut self,
9952 receivers: &[UnitRecord],
9953 tables: &[String],
9954 changes: Vec<ssp2::model::Change>,
9955 commit_seq: i64,
9956 source: AppliedServerCommitSource,
9957 ) -> Result<bool, TransportError> {
9958 let changes: Vec<ssp2::model::Change> = changes
9959 .into_iter()
9960 .filter(|change| {
9961 let Some(table) = tables
9963 .get(change.table_index as usize)
9964 .and_then(|name| self.schema.table(name))
9965 else {
9966 return true;
9967 };
9968 receivers.iter().any(|unit| {
9969 covers(table, &unit.scopes)
9970 && pair_scopes(&unit.scopes, &table.name, unit.echo.as_deref()).is_some_and(
9971 |scopes| {
9972 scopes.iter().all(|(variable, values)| {
9973 change
9974 .scopes
9975 .iter()
9976 .find(|(key, _)| key == variable)
9977 .is_some_and(|(_, value)| values.contains(value))
9978 })
9979 },
9980 )
9981 })
9982 })
9983 .collect();
9984 if changes.is_empty() {
9985 return Ok(false);
9986 }
9987 let affected_tables = changes
9988 .iter()
9989 .filter_map(|change| tables.get(change.table_index as usize))
9990 .cloned()
9991 .collect::<BTreeSet<_>>();
9992 let previous_batches = self.change_queue.len();
9993 self.apply_commit_frame(tables, &changes, commit_seq)?;
9994 if self.change_queue.len() > previous_batches {
9995 if let Some(batch) = self.change_queue.back_mut() {
9996 batch
9997 .applied_commits
9998 .extend(
9999 affected_tables
10000 .into_iter()
10001 .map(|table| AppliedServerCommit {
10002 table,
10003 commit_seq,
10004 source,
10005 }),
10006 );
10007 }
10008 }
10009 Ok(true)
10010 }
10011
10012 fn apply_commit_frame(
10014 &mut self,
10015 tables: &[String],
10016 changes: &[ssp2::model::Change],
10017 sequence: i64,
10018 ) -> Result<(), TransportError> {
10019 #[cfg(feature = "bench-internals")]
10020 let _phase = self.benchmark_phases.start(Phase::CommitApply);
10021 let rollback = self.capture();
10022 self.coalesce_unit(changes.len())
10023 .and_then(|()| self.begin_observation("syncular_commit"))
10024 .map_err(|message| TransportError::new("storage.failed", message))?;
10025 let applied = (|| {
10026 let mut batch = ChangeAccumulator::default();
10027 self.record_commit_changes(&mut batch, tables, changes);
10028 for change in changes {
10031 let Some(table) = tables.get(change.table_index as usize) else {
10032 continue;
10033 };
10034 let acknowledged = self
10035 .conn
10036 .prepare_cached("SELECT op_json FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq=?3")
10037 .and_then(|mut statement| {
10038 statement
10039 .query_map(rusqlite::params![table, change.row_id, sequence], |row| row.get::<_, String>(0))?
10040 .collect::<Result<Vec<_>, _>>()
10041 })
10042 .map_err(|error| TransportError::new("storage.failed", Self::sqlite_failure(&self.storage_failure, error)))?
10043 .iter()
10044 .map(|op| serde_json::from_str::<CommitOperation>(op).map(OutboxOp::from))
10045 .collect::<Result<Vec<_>, _>>()
10046 .map_err(|_| TransportError::new("storage.failed", "sync.local_corrupt: invalid acknowledged operation".to_owned()))?;
10047 for index in 0..self.outbox.len() {
10048 if !self.outbox[index]
10049 .ops
10050 .iter()
10051 .any(|op| acknowledged.iter().any(|earlier| chains_on(earlier, op)))
10052 {
10053 continue;
10054 }
10055 let commit = Arc::make_mut(&mut self.outbox[index]);
10056 for op in &mut commit.ops {
10057 if acknowledged.iter().any(|earlier| chains_on(earlier, op)) {
10058 op.base_version = Some(change.row_version);
10059 }
10060 }
10061 let ops = serde_json::to_string(
10062 &commit
10063 .ops
10064 .iter()
10065 .map(CommitOperation::from)
10066 .collect::<Vec<_>>(),
10067 )
10068 .map_err(|_| {
10069 TransportError::new(
10070 "storage.failed",
10071 "sync.local_corrupt: unencodable outbox".to_owned(),
10072 )
10073 })?;
10074 let id = commit.client_commit_id.clone();
10075 self.conn
10076 .execute(
10077 "UPDATE _syncular_outbox SET ops_json=?1 WHERE commit_id=?2",
10078 rusqlite::params![ops, id],
10079 )
10080 .map_err(|error| {
10081 TransportError::new(
10082 "storage.failed",
10083 Self::sqlite_failure(&self.storage_failure, error),
10084 )
10085 })?;
10086 }
10087 }
10088 {
10089 let storage_failed = |error| {
10090 TransportError::new(
10091 "storage.failed",
10092 Self::sqlite_failure(&self.storage_failure, error),
10093 )
10094 };
10095 let pinned: HashSet<(&str, &str)> = self
10096 .outbox
10097 .iter()
10098 .flat_map(|commit| &commit.ops)
10099 .map(|op| (op.table.as_str(), op.row_id.as_str()))
10100 .collect();
10101 for change in changes {
10102 let table = tables.get(change.table_index as usize).ok_or_else(|| {
10103 TransportError::new(
10104 "sync.invalid_request",
10105 "change tableIndex out of range".to_owned(),
10106 )
10107 })?;
10108 if pinned.contains(&(table.as_str(), change.row_id.as_str())) {
10109 self.conn.prepare_cached("INSERT INTO _syncular_row_deliveries(tbl,id,commit_seq) VALUES (?1,?2,?3) ON CONFLICT(tbl,id) DO UPDATE SET commit_seq=MAX(commit_seq,excluded.commit_seq)")
10110 .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10111 .map_err(storage_failed)?;
10112 }
10113 self.conn.prepare_cached("DELETE FROM _syncular_acked_rows WHERE tbl=?1 AND id=?2 AND commit_seq<=?3")
10114 .and_then(|mut statement| statement.execute(rusqlite::params![table, change.row_id, sequence]))
10115 .map_err(storage_failed)?;
10116 }
10117 }
10118 self.apply_commit_changes(tables, changes)?;
10119 self.rebuild_overlay_if_dirty()
10120 .map_err(|message| TransportError::new("storage.failed", message))?;
10121 self.finish_observation("syncular_commit", batch)
10122 .map_err(|message| TransportError::new("storage.failed", message))
10123 })();
10124 if applied.is_err() {
10125 self.rollback_observation("syncular_commit");
10126 self.restore(rollback);
10127 }
10128 applied
10129 }
10130
10131 fn apply_commit_changes(
10132 &mut self,
10133 tables: &[String],
10134 changes: &[ssp2::model::Change],
10135 ) -> Result<(), TransportError> {
10136 let changed_tables: BTreeSet<&str> = changes
10141 .iter()
10142 .filter_map(|change| tables.get(change.table_index as usize))
10143 .map(String::as_str)
10144 .collect();
10145 let mut mirror_visible = !self.overlay_dirty.get();
10146 for name in &changed_tables {
10147 if mirror_visible
10148 && self
10149 .schema
10150 .table(name)
10151 .is_some_and(|table| table.indexes.iter().any(|index| index.unique))
10152 {
10153 let (pending, protected) = self
10154 .table_intent(name)
10155 .map_err(|message| TransportError::new("storage.failed", message))?;
10156 mirror_visible = !pending && !protected;
10157 }
10158 }
10159 for change in changes {
10160 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
10161 TransportError::new(
10162 "sync.invalid_request",
10163 "change tableIndex out of range".to_owned(),
10164 )
10165 })?;
10166 let table = self.schema.table(table_name).ok_or_else(|| {
10167 TransportError::new(
10168 "sync.schema_mismatch",
10169 format!("change targets unknown table {table_name:?}"),
10170 )
10171 })?;
10172 let landed = match change.op {
10173 Op::Upsert => {
10174 let payload = change.row.as_ref().ok_or_else(|| {
10175 TransportError::new(
10176 "sync.invalid_request",
10177 "upsert change without row payload".to_owned(),
10178 )
10179 })?;
10180 let (row, sealed) = {
10182 #[cfg(feature = "bench-internals")]
10183 let _phase = self.benchmark_phases.start(Phase::RowDecode);
10184 decode_row_bytes(table, payload, &self.encryption)?
10186 };
10187 let table_name = table.name.clone();
10188 let landed = self
10190 .write_base_row(&table_name, &row, change.row_version, sealed.as_ref())
10191 .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10192 if mirror_visible && landed {
10193 self.write_row(
10194 &visible_table(&table_name),
10195 &table_name,
10196 &row,
10197 change.row_version,
10198 sealed.as_ref(),
10199 )
10200 .map_err(|m| TransportError::new("sync.invalid_request", m))?;
10201 }
10202 landed
10203 }
10204 Op::Delete => self
10207 .delete_base_row(table_name, &change.row_id, change.row_version)
10208 .map_err(|m| TransportError::new("sync.invalid_request", m))?,
10209 };
10210 self.overlay_dirty.table(table_name);
10214 if mirror_visible && !(landed && change.op == Op::Upsert) {
10215 let predicate = row_id_predicate(table);
10216 let visible = visible_table(table_name);
10217 let mut statements = vec![format!("DELETE FROM {visible} WHERE {predicate}")];
10218 if !landed {
10219 statements.push(format!(
10220 "INSERT INTO {visible} SELECT * FROM {} WHERE {predicate}",
10221 base_table(table_name)
10222 ));
10223 }
10224 for statement in &statements {
10225 self.conn
10226 .prepare_cached(statement)
10227 .and_then(|mut statement| {
10228 statement.execute(rusqlite::params![change.row_id])
10229 })
10230 .map_err(|error| {
10231 TransportError::new("sync.invalid_request", error.to_string())
10232 })?;
10233 }
10234 }
10235 }
10236 if mirror_visible {
10237 let changed_rows: HashSet<(&str, &str)> = changes
10238 .iter()
10239 .map(|change| {
10240 (
10241 tables[change.table_index as usize].as_str(),
10242 change.row_id.as_str(),
10243 )
10244 })
10245 .collect();
10246 self.replay_overlay(&changed_tables, Some(&changed_rows))
10247 .map_err(|message| TransportError::new("storage.failed", message))?;
10248 self.overlay_dirty.set(false);
10249 }
10250 Ok(())
10251 }
10252
10253 fn apply_segment(
10258 &mut self,
10259 table: &TableSchema,
10260 segment: &RowsSegment,
10261 first_page: bool,
10262 effective: &[(String, Vec<String>)],
10263 unit_id: &str,
10264 ) -> Result<u32, SectionError> {
10265 let column = if segment.columns.len() != table.wire_columns.len() {
10270 Some(Value::from("count"))
10271 } else {
10272 segment
10273 .columns
10274 .iter()
10275 .zip(table.wire_columns.iter())
10276 .position(|(a, b)| a.name != b.name || a.ty != b.ty || a.nullable != b.nullable)
10277 .map(Value::from)
10278 };
10279 if segment.table != table.name
10280 || segment.schema_version != self.schema.version
10281 || column.is_some()
10282 {
10283 let mut details = Map::new();
10284 details.insert("table".to_owned(), Value::from(table.name.as_str()));
10285 if let Some(column) = column {
10286 details.insert("column".to_owned(), column);
10287 }
10288 return Err(SectionError::Transport(TransportError {
10289 code: "sync.schema_mismatch".to_owned(),
10290 message: "rows segment columns do not match the generated schema (§5.2)".to_owned(),
10291 details: Some(Value::Object(details)),
10292 }));
10293 }
10294 let mut applied = 0u32;
10295 let empty = Vec::new();
10296 let blocks = if segment.blocks.is_empty() {
10297 std::slice::from_ref(&empty)
10298 } else {
10299 &segment.blocks
10300 };
10301 for (index, block) in blocks.iter().enumerate() {
10302 let clear = first_page && index == 0;
10303 if !clear && block.is_empty() {
10304 continue;
10305 }
10306 let rollback = self.capture();
10307 let (pending, protected) = self
10308 .table_intent(&table.name)
10309 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10310 let reconcile_pending =
10311 !protected && pending && table.indexes.iter().any(|index| index.unique);
10312 let mirror_visible = !protected && !reconcile_pending;
10313 self.coalesce_unit(block.len())
10314 .and_then(|()| self.begin_observation("syncular_segment_block"))
10315 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10316 let outcome = (|| {
10317 self.rebuild_overlay_if_dirty()
10318 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10319 let mut changed_keys = Vec::new();
10320 let mut batch = ChangeAccumulator::default();
10321 if clear && self.clear_scope(table, effective, unit_id)? || !block.is_empty() {
10322 batch.table(&table.name);
10323 }
10324 for (row_index, row) in block.iter().enumerate() {
10325 let decrypted;
10326 let mut sealed = None;
10327 let values = if table.has_encrypted_columns() {
10328 let mut values = row.values.clone();
10329 sealed = crate::values::decrypt_segment_row(
10330 table,
10331 &mut values,
10332 &self.encryption,
10333 )
10334 .map_err(SectionError::Transport)?;
10335 decrypted = values;
10336 &decrypted
10337 } else {
10338 &row.values
10339 };
10340 if !self
10341 .write_base_row(&table.name, values, row.server_version, sealed.as_ref())
10342 .map_err(|message| {
10343 SectionError::Abort("sync.invalid_request".into(), message)
10344 })?
10345 {
10346 continue;
10347 }
10348 if reconcile_pending {
10349 changed_keys.push(
10350 owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
10351 |message| SectionError::Abort("storage.failed".into(), message),
10352 )?,
10353 );
10354 }
10355 if mirror_visible {
10356 self.write_row(
10357 &visible_table(&table.name),
10358 &table.name,
10359 values,
10360 row.server_version,
10361 sealed.as_ref(),
10362 )
10363 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10364 }
10365 let processed = u64::from(applied) + row_index as u64 + 1;
10366 if processed.is_multiple_of(1024) {
10367 self.progress.update(|p| p.rows_processed = processed);
10368 }
10369 }
10370 self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10371 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10372 self.finish_observation("syncular_segment_block", batch)
10373 .map_err(|message| SectionError::Abort("storage.failed".into(), message))
10374 })();
10375 if let Err(error) = outcome {
10376 self.rollback_observation("syncular_segment_block");
10377 self.restore(rollback);
10378 return Err(error);
10379 }
10380 applied += block.len() as u32;
10381 self.progress
10382 .update(|p| p.rows_processed = u64::from(applied));
10383 std::thread::yield_now();
10384 }
10385 self.progress
10386 .update(|p| p.rows_processed = u64::from(applied));
10387 Ok(applied)
10388 }
10389
10390 #[allow(clippy::too_many_arguments)]
10398 fn apply_sqlite_segment(
10399 &mut self,
10400 table: &TableSchema,
10401 bytes: &[u8],
10402 first_page: bool,
10403 effective: &[(String, Vec<String>)],
10404 unit_id: &str,
10405 segment_id: &str,
10406 resumed: i64,
10407 row_count: i64,
10408 as_of_commit_seq: i64,
10409 scope_digest: &str,
10410 ) -> Result<u32, SectionError> {
10411 let invalid = |detail: &str| {
10412 SectionError::Abort(
10413 "sync.invalid_request".to_owned(),
10414 format!("sqlite segment rejected: {detail} (§5.3)"),
10415 )
10416 };
10417 if table.has_encrypted_columns() {
10420 return Err(invalid("table has encrypted columns"));
10421 }
10422
10423 let directory = self
10427 .previous_version_replica_path()
10428 .and_then(|replica| std::path::Path::new(&replica).parent().map(Into::into))
10429 .unwrap_or_else(std::env::temp_dir);
10430 let path = directory.join(format!(".syncular-image-{}.tmp", uuid::Uuid::new_v4()));
10431 let outcome = (|| {
10432 let mut options = std::fs::OpenOptions::new();
10433 options.write(true).create_new(true);
10434 #[cfg(unix)]
10435 std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600);
10436 std::io::Write::write_all(
10437 &mut options
10438 .open(&path)
10439 .map_err(|_| invalid("image temp file write failed"))?,
10440 bytes,
10441 )
10442 .map_err(|_| invalid("image temp file write failed"))?;
10443 let img = rusqlite::Connection::open_with_flags(
10444 &path,
10445 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
10446 )
10447 .map_err(|_| invalid("bytes do not open as a SQLite database"))?;
10448 self.apply_sqlite_image(
10449 &img,
10450 table,
10451 first_page,
10452 effective,
10453 unit_id,
10454 segment_id,
10455 resumed,
10456 row_count,
10457 as_of_commit_seq,
10458 scope_digest,
10459 )
10460 })();
10461 let _ = std::fs::remove_file(&path);
10462 outcome
10463 }
10464
10465 #[allow(clippy::too_many_arguments)]
10466 fn apply_sqlite_image(
10467 &mut self,
10468 img: &rusqlite::Connection,
10469 table: &crate::schema::TableSchema,
10470 first_fresh_page: bool,
10471 effective: &[(String, Vec<String>)],
10472 unit_id: &str,
10473 segment_id: &str,
10474 resumed: i64,
10475 row_count: i64,
10476 as_of_commit_seq: i64,
10477 scope_digest: &str,
10478 ) -> Result<u32, SectionError> {
10479 let invalid = |detail: String| {
10480 SectionError::Abort(
10481 "sync.invalid_request".to_owned(),
10482 format!("sqlite segment rejected: {detail} (§5.3)"),
10483 )
10484 };
10485
10486 type MetaRow = (i64, String, i64, i64, String, i64, i64);
10489 let meta: MetaRow = img
10490 .query_row(
10491 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
10492 \"scopeDigest\", \"rowCount\",
10493 (SELECT count(*) FROM _syncular_segment)
10494 FROM _syncular_segment",
10495 [],
10496 |row| {
10497 Ok((
10498 row.get(0)?,
10499 row.get(1)?,
10500 row.get(2)?,
10501 row.get(3)?,
10502 row.get(4)?,
10503 row.get(5)?,
10504 row.get(6)?,
10505 ))
10506 },
10507 )
10508 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
10509 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
10510 if meta_count != 1 {
10511 return Err(invalid(format!(
10512 "_syncular_segment must contain exactly one row, found {meta_count}"
10513 )));
10514 }
10515 if format != 1 {
10516 return Err(invalid(format!("format {format}")));
10517 }
10518 if meta_table != table.name {
10519 return Err(invalid(format!("image table {meta_table:?}")));
10520 }
10521 if schema_version != i64::from(self.schema.version) {
10522 return Err(invalid(format!("schemaVersion {schema_version}")));
10523 }
10524 if pin != as_of_commit_seq {
10525 return Err(invalid(format!("asOfCommitSeq {pin}")));
10526 }
10527 if digest != scope_digest {
10528 return Err(invalid("scopeDigest mismatch".to_owned()));
10529 }
10530 if meta_rows != row_count {
10531 return Err(invalid(format!("rowCount {meta_rows}")));
10532 }
10533
10534 let mut names: Vec<String> = Vec::new();
10536 let mut primary_keys: Vec<String> = Vec::new();
10537 {
10538 let mut stmt = img
10539 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
10540 .map_err(|_| invalid("image data table missing".to_owned()))?;
10541 let mut rows = stmt
10542 .query([])
10543 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
10544 while let Some(row) = rows
10545 .next()
10546 .map_err(|_| invalid("image data table unreadable".to_owned()))?
10547 {
10548 if row
10549 .get::<_, i64>(5)
10550 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?
10551 > 0
10552 {
10553 primary_keys.push(row.get(1).map_err(|error| {
10554 invalid(Self::sqlite_failure(&self.storage_failure, error))
10555 })?);
10556 }
10557 names.push(
10558 row.get::<_, String>(1)
10559 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
10560 );
10561 }
10562 }
10563 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
10564 expected.push("_syncular_version");
10565 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
10566 return Err(SectionError::Transport(TransportError {
10567 code: "sync.schema_mismatch".to_owned(),
10568 message: "sqlite segment columns do not match the generated schema (§5.3)"
10569 .to_owned(),
10570 details: Some(serde_json::json!({ "table": table.name })),
10571 }));
10572 }
10573
10574 if primary_keys != vec![table.primary_key.clone()] {
10575 return Err(invalid(
10576 "image primary key does not match generated schema".into(),
10577 ));
10578 }
10579 let actual_count: i64 = img
10580 .query_row(
10581 &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
10582 [],
10583 |row| row.get(0),
10584 )
10585 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10586 if actual_count != row_count {
10587 return Err(invalid("image row count does not match descriptor".into()));
10588 }
10589 let insert = self.insert_row_sql(&base_table(&table.name), table);
10590 let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
10591 let column_list = names
10592 .iter()
10593 .map(|n| quote_ident(n))
10594 .collect::<Vec<_>>()
10595 .join(", ");
10596 let mut stmt = img
10599 .prepare(&format!(
10600 "SELECT {column_list} FROM {} ORDER BY {} LIMIT -1 OFFSET ?1",
10601 quote_ident(&table.name),
10602 quote_ident(&table.primary_key)
10603 ))
10604 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10605 let mut rows = stmt
10606 .query([resumed])
10607 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10608 let resumed = u32::try_from(resumed)
10609 .ok()
10610 .filter(|rows| i64::from(*rows) <= actual_count)
10611 .ok_or_else(|| invalid("applied-segment record exceeds the image".into()))?;
10612 let mut applied = resumed;
10613 let mut limit = if first_fresh_page {
10614 1024
10615 } else {
10616 self.image_chunk_rows
10617 .get(&table.name)
10618 .copied()
10619 .unwrap_or(1024)
10620 };
10621 let (mut write, mut commit) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
10622 loop {
10623 let clear = applied == 0 && first_fresh_page;
10624 let rollback = self.capture();
10625 let (pending, protected) = self
10626 .table_intent(&table.name)
10627 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10628 let reconcile_pending =
10629 !protected && pending && table.indexes.iter().any(|index| index.unique);
10630 let mirror_visible = !protected && !reconcile_pending;
10631 let chunk = if pending || protected {
10638 limit.min(1024)
10639 } else {
10640 limit
10641 };
10642 self.coalesce_unit(chunk as usize)
10643 .and_then(|()| self.begin_observation("syncular_image_chunk"))
10644 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10645 let started = std::time::Instant::now();
10646 let result = (|| {
10647 self.rebuild_overlay_if_dirty()
10648 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10649 let mut changed_keys = Vec::new();
10650 let mut batch = ChangeAccumulator::default();
10651 if clear && self.clear_scope(table, effective, unit_id)? {
10652 batch.table(&table.name);
10653 }
10654 let mut processed = 0u32;
10655 {
10656 let mut ins = self.conn.prepare_cached(&insert).map_err(|error| {
10657 invalid(Self::sqlite_failure(&self.storage_failure, error))
10658 })?;
10659 let mut mirror = if mirror_visible {
10660 Some(self.conn.prepare_cached(&visible_insert).map_err(|error| {
10661 invalid(Self::sqlite_failure(&self.storage_failure, error))
10662 })?)
10663 } else {
10664 None
10665 };
10666 for _ in 0..chunk {
10667 let Some(row) = rows.next().map_err(|error| {
10668 invalid(Self::sqlite_failure(&self.storage_failure, error))
10669 })?
10670 else {
10671 break;
10672 };
10673 for (i, column) in table.columns.iter().enumerate() {
10674 let cell = row.get_ref(i).map_err(|error| {
10675 invalid(Self::sqlite_failure(&self.storage_failure, error))
10676 })?;
10677 let param = image_cell_param(column, cell).map_err(invalid)?;
10678 ins.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
10679 invalid(Self::sqlite_failure(&self.storage_failure, error))
10680 })?;
10681 if let Some(mirror) = &mut mirror {
10682 mirror.raw_bind_parameter(i + 1, ¶m).map_err(|error| {
10683 invalid(Self::sqlite_failure(&self.storage_failure, error))
10684 })?;
10685 }
10686 }
10687 let version_index = table.columns.len();
10688 let version: i64 = row.get(version_index).map_err(|error| {
10689 invalid(Self::sqlite_failure(&self.storage_failure, error))
10690 })?;
10691 if version < 1 {
10692 return Err(invalid("image row version must be positive".into()));
10693 }
10694 ins.raw_bind_parameter(version_index + 1, version)
10695 .map_err(|error| {
10696 invalid(Self::sqlite_failure(&self.storage_failure, error))
10697 })?;
10698 let landed = ins.raw_execute().map_err(|error| {
10700 invalid(Self::sqlite_failure(&self.storage_failure, error))
10701 })? > 0;
10702 processed += 1;
10703 if processed.is_multiple_of(1024) && processed < chunk {
10704 self.progress
10705 .update(|p| p.rows_processed = u64::from(applied + processed));
10706 }
10707 if !landed {
10708 continue;
10709 }
10710 if let Some(mirror) = &mut mirror {
10711 mirror
10712 .raw_bind_parameter(version_index + 1, version)
10713 .map_err(|error| {
10714 invalid(Self::sqlite_failure(&self.storage_failure, error))
10715 })?;
10716 mirror.raw_execute().map_err(|error| {
10717 invalid(Self::sqlite_failure(&self.storage_failure, error))
10718 })?;
10719 }
10720 if reconcile_pending {
10721 changed_keys.push(row.get::<_, SqlValue>(table.pk_index).map_err(
10722 |error| invalid(Self::sqlite_failure(&self.storage_failure, error)),
10723 )?);
10724 }
10725 }
10726 }
10727 if processed > 0 {
10728 batch.table(&table.name);
10729 self.overlay_dirty.table(&table.name);
10730 }
10731 self.stored(record_applied_rows(
10732 &self.conn,
10733 unit_id,
10734 segment_id,
10735 i64::from(applied + processed),
10736 ))
10737 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10738 self.reconcile_import(table, &mut changed_keys, protected, reconcile_pending)
10739 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10740 let written = std::time::Instant::now();
10741 self.finish_observation("syncular_image_chunk", batch)
10742 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10743 Ok((processed, written))
10744 })();
10745 let (processed, written) = match result {
10746 Ok(done) => done,
10747 Err(error) => {
10748 self.rollback_observation("syncular_image_chunk");
10749 self.restore(rollback);
10750 return Err(error);
10751 }
10752 };
10753 let coalesced = self.coalesced.is_some();
10755 if coalesced {
10756 self.commit_coalesced_apply()
10757 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10758 }
10759 write += written - started;
10760 commit += written.elapsed();
10761 applied += processed;
10762 let last = processed < chunk || i64::from(applied) == actual_count;
10763 if last && self.conn.is_autocommit() {
10765 self.conn
10766 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10767 .map_err(|error| invalid(Self::sqlite_failure(&self.storage_failure, error)))?;
10768 }
10769 if coalesced {
10770 self.begin_coalesced_apply()
10771 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
10772 }
10773 self.progress.update(|p| {
10774 p.rows_processed = u64::from(applied);
10775 p.import_write_ms = u64::try_from(write.as_millis()).ok();
10776 p.import_commit_ms = u64::try_from(commit.as_millis()).ok();
10777 });
10778 let duration = started.elapsed().as_secs_f64() * 1000.0;
10779 #[cfg(test)]
10780 let duration = self.image_chunk_ms.get().unwrap_or(duration);
10781 std::thread::yield_now();
10782 if last {
10783 break;
10784 }
10785 limit = (processed * 2)
10787 .min(65_536)
10788 .min((f64::from(processed) * 250.0 / duration) as u32)
10789 .max(1024);
10790 self.image_chunk_rows.insert(table.name.clone(), limit);
10791 }
10792 if i64::from(applied) != actual_count {
10793 return Err(invalid(format!(
10794 "image yielded {applied} rows, expected {actual_count}"
10795 )));
10796 }
10797 Ok(applied - resumed)
10798 }
10799
10800 fn reconcile_rows(&self, drained: &[Arc<OutboxCommit>]) -> Result<(), String> {
10805 let names: BTreeSet<&str> = drained
10806 .iter()
10807 .flat_map(|commit| &commit.ops)
10808 .map(|op| op.table.as_str())
10809 .collect();
10810 for name in names {
10811 let Some(table) = self.schema.table(name) else {
10812 continue;
10813 };
10814 let ops = drained.iter().flat_map(|commit| &commit.ops);
10815 let tables = BTreeSet::from([name]);
10816 let rows: HashSet<_> = ops
10817 .clone()
10818 .map(|op| (op.table.as_str(), op.row_id.as_str()))
10819 .collect();
10820 let unique = table.indexes.iter().any(|index| index.unique);
10821 let acknowledged = if unique {
10822 self.acknowledged_operations(None, Some(&tables))?
10823 } else {
10824 Vec::new()
10825 };
10826 let peers = acknowledged
10827 .iter()
10828 .chain(
10829 self.failed_commits
10830 .iter()
10831 .flat_map(|(commit, _)| &commit.ops),
10832 )
10833 .chain(self.outbox.iter().flat_map(|commit| &commit.ops))
10834 .filter(|_| unique);
10835 self.restore_base_rows(table, &mut Vec::new(), peers.chain(ops.clone()))?;
10836 self.replay_overlay(&tables, (!unique).then_some(&rows))?;
10837 }
10838 Ok(())
10839 }
10840
10841 fn table_intent(&self, table: &str) -> Result<(bool, bool), String> {
10844 let touches = |op: &OutboxOp| op.table == table;
10845 Ok((
10846 self.outbox
10847 .iter()
10848 .flat_map(|commit| &commit.ops)
10849 .any(touches),
10850 self.failed_commits
10851 .iter()
10852 .flat_map(|(commit, _)| &commit.ops)
10853 .any(touches)
10854 || self
10855 .conn
10856 .query_row(
10857 "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows WHERE tbl=?1)",
10858 [table],
10859 |row| row.get(0),
10860 )
10861 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10862 ))
10863 }
10864
10865 fn reconcile_import(
10869 &mut self,
10870 table: &TableSchema,
10871 keys: &mut Vec<SqlValue>,
10872 protected: bool,
10873 restore: bool,
10874 ) -> Result<(), String> {
10875 if protected {
10876 return self.rebuild_overlay_if_dirty();
10877 }
10878 let pending = self.outbox.iter().flat_map(|commit| &commit.ops);
10879 if restore {
10880 self.restore_base_rows(table, keys, pending.clone())?;
10881 }
10882 self.apply_outbox_ops(pending.filter(|op| op.table == table.name))?;
10883 self.overlay_dirty.set(false);
10884 Ok(())
10885 }
10886
10887 fn restore_base_rows<'a>(
10891 &self,
10892 table: &TableSchema,
10893 keys: &mut Vec<SqlValue>,
10894 ops: impl IntoIterator<Item = &'a OutboxOp>,
10895 ) -> Result<(), String> {
10896 let visible = visible_table(&table.name);
10897 let base = base_table(&table.name);
10898 let pk = quote_ident(&table.primary_key);
10899 let predicate = row_id_predicate(table);
10900 let mut lookup = self.conn.prepare_cached(&format!(
10901 "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
10902 )).map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10903 let mut deleted = HashSet::new();
10904 for op in ops.into_iter().filter(|op| op.table == table.name) {
10905 if op.upsert {
10906 if let Some(values) = &op.values {
10907 let key = json_to_column_value(
10908 &table.columns[table.pk_index],
10909 values.get(&table.primary_key),
10910 )?;
10911 keys.push(owned_sql_value(RowParam::Cell(&key))?);
10912 }
10913 } else if deleted.insert(&op.row_id) {
10914 keys.extend(
10915 lookup
10916 .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
10917 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
10918 .collect::<Result<Vec<_>, _>>()
10919 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
10920 );
10921 }
10922 }
10923 for chunk in keys.chunks(400) {
10926 let holes = vec!["?"; chunk.len()].join(",");
10927 self.conn
10928 .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
10929 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10930 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10931 }
10932 let updates = table
10933 .columns
10934 .iter()
10935 .map(|column| column.name.as_str())
10936 .chain(std::iter::once("_syncular_version"))
10937 .chain(table.sealed_columns().iter().copied())
10938 .filter(|name| *name != table.primary_key)
10939 .map(|name| {
10940 let name = quote_ident(name);
10941 format!("{name}=excluded.{name}")
10942 })
10943 .collect::<Vec<_>>()
10944 .join(",");
10945 for chunk in keys.chunks(400) {
10946 let holes = vec!["?"; chunk.len()].join(",");
10947 self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
10948 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
10949 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
10950 }
10951 Ok(())
10952 }
10953
10954 fn compile_local_data_purge(
10957 &self,
10958 input: &LocalDataPurgeInput,
10959 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
10960 let invalid = |message: String| format!("sync.invalid_request: {message}");
10961 if input.purge_id.is_empty()
10962 || input.purge_id.len() > 128
10963 || !is_local_operation_code_like(&input.purge_id)
10964 {
10965 return Err(invalid(
10966 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
10967 ));
10968 }
10969 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
10970 return Err(invalid(format!(
10971 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
10972 )));
10973 }
10974
10975 let mut deduplicated: BTreeMap<
10976 String,
10977 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
10978 > = BTreeMap::new();
10979 for target in &input.targets {
10980 let table = self.schema.table(&target.table).ok_or_else(|| {
10981 invalid(format!(
10982 "local purge names unknown table {:?}",
10983 target.table
10984 ))
10985 })?;
10986 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
10987 return Err(invalid(format!(
10988 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
10989 target.table
10990 )));
10991 }
10992 let mut selectors = Vec::with_capacity(target.selectors.len());
10993 let mut canonical_selectors = BTreeMap::new();
10994 for (column_name, raw_values) in &target.selectors {
10995 let Some((column_index, column)) = table
10996 .columns
10997 .iter()
10998 .enumerate()
10999 .find(|(_, column)| column.name == *column_name)
11000 else {
11001 return Err(invalid(format!(
11002 "local purge target {:?} names unknown column {:?}",
11003 target.table, column_name
11004 )));
11005 };
11006 let encrypted = table
11007 .encrypted_columns
11008 .iter()
11009 .any(|candidate| candidate.index == column_index);
11010 if column.ty != ColumnType::String || encrypted {
11011 return Err(invalid(format!(
11012 "local purge selector {:?}.{:?} must be a plaintext string column",
11013 target.table, column_name
11014 )));
11015 }
11016 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
11017 return Err(invalid(format!(
11018 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
11019 target.table, column_name
11020 )));
11021 }
11022 let mut values = raw_values.clone();
11023 values.sort();
11024 values.dedup();
11025 if values.iter().any(|value| {
11026 value.is_empty()
11027 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
11028 || !is_local_operation_code_like(value)
11029 }) {
11030 return Err(invalid(format!(
11031 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
11032 )));
11033 }
11034 selectors.push((column_name.clone(), values.clone()));
11035 canonical_selectors.insert(column_name.clone(), values);
11036 }
11037 let canonical = LocalDataPurgeTarget {
11038 table: target.table.clone(),
11039 selectors: canonical_selectors,
11040 };
11041 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11042 deduplicated.insert(
11043 key,
11044 (
11045 CompiledLocalDataPurgeTarget {
11046 table: target.table.clone(),
11047 selectors,
11048 },
11049 canonical,
11050 ),
11051 );
11052 }
11053 let targets = deduplicated
11054 .values()
11055 .map(|(compiled, _)| compiled.clone())
11056 .collect::<Vec<_>>();
11057 let canonical = deduplicated
11058 .values()
11059 .map(|(_, target)| target.clone())
11060 .collect::<Vec<_>>();
11061 let canonical_plan =
11062 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
11063 Ok((targets, canonical_plan))
11064 }
11065
11066 fn local_purge_base_row_ids(
11067 &self,
11068 targets: &[CompiledLocalDataPurgeTarget],
11069 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
11070 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
11071 for target in targets {
11072 let table = self
11073 .schema
11074 .table(&target.table)
11075 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
11076 let mut clauses = Vec::with_capacity(target.selectors.len());
11077 let mut params = Vec::new();
11078 for (column, values) in &target.selectors {
11079 clauses.push(format!(
11080 "{} IN ({})",
11081 quote_ident(column),
11082 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
11083 ));
11084 params.extend(values.iter().cloned().map(SqlValue::Text));
11085 }
11086 let sql = format!(
11087 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
11088 quote_ident(&table.primary_key),
11089 base_table(&target.table),
11090 clauses.join(" AND ")
11091 );
11092 let mut statement = self
11093 .conn
11094 .prepare(&sql)
11095 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11096 let rows = statement
11097 .query_map(rusqlite::params_from_iter(params), |row| {
11098 let value = sql_ref_to_json(&table.columns[table.pk_index], row.get_ref(0)?);
11099 render_row_id_json(Some(&value))
11100 .map_err(|message| rusqlite::Error::ToSqlConversionFailure(message.into()))
11101 })
11102 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11103 let ids = by_table.entry(target.table.clone()).or_default();
11104 for row in rows {
11105 ids.insert(
11106 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?,
11107 );
11108 }
11109 }
11110 Ok(by_table)
11111 }
11112
11113 fn local_purge_values_match(
11114 target: &CompiledLocalDataPurgeTarget,
11115 values: &Map<String, Value>,
11116 ) -> bool {
11117 target.selectors.iter().all(|(column, allowed)| {
11118 values
11119 .get(column)
11120 .and_then(Value::as_str)
11121 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
11122 })
11123 }
11124
11125 pub fn purge_local_data(
11129 &mut self,
11130 input: &LocalDataPurgeInput,
11131 ) -> Result<LocalDataPurgeResult, String> {
11132 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
11133 let meta_key = format!("localPurge:{}", input.purge_id);
11134 let applied_plan = self
11135 .conn
11136 .query_row(
11137 "SELECT value FROM _syncular_meta WHERE key = ?1",
11138 rusqlite::params![meta_key],
11139 |row| row.get::<_, String>(0),
11140 )
11141 .optional()
11142 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11143 if let Some(applied_plan) = applied_plan {
11144 if applied_plan != canonical_plan {
11145 return Err(format!(
11146 "sync.invalid_request: local purge id {:?} was already used with a different plan",
11147 input.purge_id
11148 ));
11149 }
11150 self.erase_purged_pages()?;
11152 return Ok(LocalDataPurgeResult {
11153 already_applied: true,
11154 purged_rows: 0,
11155 dropped_commits: 0,
11156 });
11157 }
11158
11159 let _ = self.drop_previous_version();
11163
11164 let rollback = self.capture();
11165 self.begin_observation("syncular_local_purge")?;
11166 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
11167 let row_ids = self.local_purge_base_row_ids(&targets)?;
11168 let retained_dropped = self.drop_failed_commits(|table, values| {
11169 targets.iter().any(|target| {
11170 target.table == table && Self::local_purge_values_match(target, values)
11171 })
11172 })?;
11173 let doomed = self
11174 .outbox
11175 .iter()
11176 .filter(|commit| {
11177 commit.ops.iter().any(|operation| {
11178 let matching_targets = targets
11179 .iter()
11180 .filter(|target| target.table == operation.table)
11181 .collect::<Vec<_>>();
11182 if matching_targets.is_empty() {
11183 return false;
11184 }
11185 if row_ids
11186 .get(&operation.table)
11187 .is_some_and(|ids| ids.contains(&operation.row_id))
11188 {
11189 return true;
11190 }
11191 operation.values.as_ref().is_some_and(|values| {
11192 matching_targets
11193 .iter()
11194 .any(|target| Self::local_purge_values_match(target, values))
11195 })
11196 })
11197 })
11198 .cloned()
11199 .collect::<Vec<_>>();
11200 let doomed_ids = doomed
11201 .iter()
11202 .map(|commit| commit.client_commit_id.clone())
11203 .collect::<BTreeSet<_>>();
11204 let mut batch = ChangeAccumulator::default();
11205 for commit in &retained_dropped {
11206 batch.conflicts = true;
11207 batch.rejections = true;
11208 batch.outcomes = true;
11209 for operation in &commit.ops {
11210 batch.table(&operation.table);
11211 }
11212 }
11213 let mut rejections = Vec::new();
11214 for commit in &doomed {
11217 for operation in &commit.ops {
11218 batch.table(&operation.table);
11219 self.overlay_dirty.table(&operation.table);
11220 }
11221 let results = (0..commit.ops.len())
11222 .map(|op_index| {
11223 let rejection = RejectionRecord {
11224 client_commit_id: commit.client_commit_id.clone(),
11225 op_index: op_index as i32,
11226 code: "client.local_data_purged".to_owned(),
11227 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
11228 retryable: false,
11229 details: None,
11230 operation: None,
11231 };
11232 rejections.push(rejection.clone());
11233 CommitOperationOutcome::Error { rejection }
11234 })
11235 .collect::<Vec<_>>();
11236 self.persist_commit_outcome(
11237 &commit.client_commit_id,
11238 CommitOutcomeStatus::Rejected,
11239 &results,
11240 None,
11241 )?;
11242 self.delete_outbox_persisted(&commit.client_commit_id)?;
11243 }
11244 if !doomed.is_empty() {
11245 self.outbox
11246 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
11247 self.rejections.extend(rejections);
11248 self.prune_commit_outcomes()?;
11249 batch.status = true;
11250 batch.rejections = true;
11251 batch.outcomes = true;
11252 }
11253
11254 let mut purged_rows = 0usize;
11255 for (table, ids) in &row_ids {
11256 if ids.is_empty() {
11257 continue;
11258 }
11259 batch.table(table);
11260 self.overlay_dirty.table(table);
11261 for row_id in ids {
11262 self.delete_base_row(table, row_id, i64::MAX)?;
11263 purged_rows += 1;
11264 }
11265 }
11266 let touches = |table: &str, row_id: &str, values: Option<&Map<String, Value>>| {
11270 targets.iter().any(|target| {
11271 target.table == table
11272 && (row_ids.get(table).is_some_and(|ids| ids.contains(row_id))
11273 || values.is_some_and(|values| {
11274 Self::local_purge_values_match(target, values)
11275 }))
11276 })
11277 };
11278 let entries = self
11279 .conn
11280 .prepare("SELECT client_commit_id, results_json, operations_json FROM _syncular_commit_outcomes")
11281 .and_then(|mut statement| {
11282 statement
11283 .query_map([], |row| {
11284 Ok((
11285 row.get::<_, String>(0)?,
11286 row.get::<_, String>(1)?,
11287 row.get::<_, Option<String>>(2)?,
11288 ))
11289 })?
11290 .collect::<Result<Vec<_>, _>>()
11291 })
11292 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11293 for (client_commit_id, results_json, operations_json) in entries {
11294 let corrupt = |_| "sync.local_corrupt: persisted commit outcome is invalid".to_owned();
11295 let mut results: Vec<CommitOperationOutcome> =
11296 serde_json::from_str(&results_json).map_err(corrupt)?;
11297 let operations: Vec<CommitOperation> = operations_json
11298 .map(|json| serde_json::from_str(&json))
11299 .transpose()
11300 .map_err(corrupt)?
11301 .unwrap_or_default();
11302 let named = operations
11303 .iter()
11304 .chain(results.iter().filter_map(|result| match result {
11305 CommitOperationOutcome::Conflict { conflict } => conflict.operation.as_ref(),
11306 CommitOperationOutcome::Error { rejection } => rejection.operation.as_ref(),
11307 CommitOperationOutcome::Applied { .. } => None,
11308 }))
11309 .any(|operation| touches(&operation.table, &operation.row_id, operation.values.as_ref()))
11310 || results.iter().any(|result| {
11311 matches!(result, CommitOperationOutcome::Conflict { conflict }
11312 if touches(&conflict.table, &conflict.row_id, Some(&conflict.server_row)))
11313 });
11314 if !named {
11315 continue;
11316 }
11317 for result in &mut results {
11318 match result {
11319 CommitOperationOutcome::Conflict { conflict } => {
11320 conflict.server_row = Map::new();
11321 conflict.conflict_columns.clear();
11322 conflict.operation = None;
11323 }
11324 CommitOperationOutcome::Error { rejection } => {
11325 rejection.details = None;
11326 rejection.operation = None;
11327 }
11328 CommitOperationOutcome::Applied { .. } => {}
11329 }
11330 }
11331 let results_json = serde_json::to_string(&results).map_err(|error| error.to_string())?;
11332 self.conn
11333 .execute(
11334 "UPDATE _syncular_commit_outcomes SET results_json = ?1, operations_json = NULL WHERE client_commit_id = ?2",
11335 rusqlite::params![results_json, client_commit_id],
11336 )
11337 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11338 batch.outcomes = true;
11339 }
11340 self.rebuild_overlay_if_dirty()?;
11341 self.delete_unreferenced_cached_blobs()?;
11342 self.conn
11343 .execute(
11344 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11345 rusqlite::params![meta_key, canonical_plan],
11346 )
11347 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11348 Ok((
11349 batch,
11350 LocalDataPurgeResult {
11351 already_applied: false,
11352 purged_rows,
11353 dropped_commits: doomed.len(),
11354 },
11355 ))
11356 })()
11357 .and_then(|(batch, result)| {
11358 self.finish_observation("syncular_local_purge", batch)
11359 .map(|()| result)
11360 });
11361 let result = match applied {
11362 Ok(result) => result,
11363 Err(error) => {
11364 self.rollback_observation("syncular_local_purge");
11365 self.restore(rollback);
11366 return Err(error);
11367 }
11368 };
11369 self.cancel_sync_round();
11370 self.erase_purged_pages()?;
11371 Ok(result)
11372 }
11373
11374 fn erase_purged_pages(&self) -> Result<(), String> {
11379 let erased = self
11380 .conn
11381 .execute_batch("PRAGMA secure_delete = ON; VACUUM;")
11382 .and_then(|()| {
11383 self.conn
11384 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11385 });
11386 let reset = self.conn.execute_batch("PRAGMA secure_delete = OFF");
11387 erased
11388 .and(reset)
11389 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11390 }
11391
11392 pub fn rebootstrap_local_data(
11398 &mut self,
11399 input: &LocalDataRebootstrapInput,
11400 ) -> Result<LocalDataRebootstrapResult, String> {
11401 if input.rebootstrap_id.is_empty()
11402 || input.rebootstrap_id.len() > 128
11403 || !is_local_operation_code_like(&input.rebootstrap_id)
11404 {
11405 return Err(
11406 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
11407 .to_owned(),
11408 );
11409 }
11410 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
11411 if let Some(persisted) = meta_get(&self.conn, &meta_key).map_err(|_| {
11412 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
11413 })? {
11414 let (retained_commits, reset_subscriptions) =
11415 decode_local_rebootstrap_receipt(&persisted)?;
11416 return Ok(LocalDataRebootstrapResult {
11417 already_applied: true,
11418 retained_commits,
11419 reset_subscriptions,
11420 });
11421 }
11422 if self.stopped || self.schema_floor.is_some() {
11423 return Err(
11424 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
11425 .to_owned(),
11426 );
11427 }
11428
11429 let retained_commits = self.outbox.len();
11430 let reset_subscriptions = self.stored(load_units(&self.conn))?.len();
11431 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
11432 let rollback = self.capture();
11433 self.begin_observation("syncular_local_rebootstrap")?;
11434 let mut batch = ChangeAccumulator::default();
11435 let applied = (|| -> Result<(), String> {
11436 self.run_schema_reset_observed(&mut batch, false)?;
11437 self.conn
11438 .execute(
11439 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
11440 rusqlite::params![meta_key, receipt],
11441 )
11442 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11443 self.sync_needed = true;
11444 self.sync_intent_queue.push_back(SyncIntent::Interactive);
11445 batch.status = true;
11446 Ok(())
11447 })()
11448 .and_then(|()| self.finish_observation("syncular_local_rebootstrap", batch));
11449 if let Err(error) = applied {
11450 self.rollback_observation("syncular_local_rebootstrap");
11451 self.restore(rollback);
11452 return Err(error);
11453 }
11454
11455 Ok(LocalDataRebootstrapResult {
11456 already_applied: false,
11457 retained_commits,
11458 reset_subscriptions,
11459 })
11460 }
11461
11462 fn purge_scope_rows(
11468 &mut self,
11469 table_name: &str,
11470 effective: &[(String, Vec<String>)],
11471 owners: &[Vec<(String, Vec<String>)>],
11472 ) -> Result<bool, String> {
11473 let Some(table) = self.schema.table(table_name) else {
11474 return Ok(false);
11475 };
11476 let Ok(predicate) = local_scope_predicate(table, effective, owners) else {
11477 return Ok(false);
11478 };
11479 let Some((predicate, params)) = predicate else {
11480 return Ok(true);
11481 };
11482 self.overlay_dirty.table(table_name);
11483 for source in [base_table(table_name), visible_table(table_name)] {
11484 self.conn
11485 .execute(
11486 &format!("DELETE FROM {source} WHERE {predicate}"),
11487 rusqlite::params_from_iter(¶ms),
11488 )
11489 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11490 }
11491 Ok(true)
11492 }
11493
11494 fn drop_doomed_outbox(
11498 &mut self,
11499 table: &TableSchema,
11500 effective: &[(String, Vec<String>)],
11501 owners: &[Vec<(String, Vec<String>)>],
11502 ) -> Result<bool, String> {
11503 let doomed: Vec<_> =
11504 self.outbox
11505 .iter()
11506 .filter(|commit| {
11507 commit.ops.iter().any(|op| {
11508 op.upsert
11509 && op.table == table.name
11510 && op.values.as_ref().is_some_and(|values| {
11511 owns_local_row(table, effective, owners, values)
11512 })
11513 })
11514 })
11515 .cloned()
11516 .collect();
11517 if doomed.is_empty() {
11518 return Ok(false);
11519 }
11520 let mut rejections = Vec::new();
11521 for commit in &doomed {
11522 let results = commit
11523 .ops
11524 .iter()
11525 .enumerate()
11526 .map(|(op_index, operation)| {
11527 self.overlay_dirty.table(&operation.table);
11528 let rejection = RejectionRecord {
11529 client_commit_id: commit.client_commit_id.clone(),
11530 op_index: op_index as i32,
11531 code: "sync.scope_revoked".to_owned(),
11532 message: "the commit was dropped because its effective scope was revoked"
11533 .to_owned(),
11534 retryable: false,
11535 details: None,
11536 operation: Some(CommitOperation::from(operation)),
11537 };
11538 rejections.push(rejection.clone());
11539 CommitOperationOutcome::Error { rejection }
11540 })
11541 .collect::<Vec<_>>();
11542 self.persist_commit_outcome(
11543 &commit.client_commit_id,
11544 CommitOutcomeStatus::Rejected,
11545 &results,
11546 Some(&commit.ops),
11547 )?;
11548 self.delete_outbox_persisted(&commit.client_commit_id)?;
11549 }
11550 self.prune_commit_outcomes()?;
11551 let doomed_ids = doomed
11552 .iter()
11553 .map(|commit| commit.client_commit_id.as_str())
11554 .collect::<BTreeSet<_>>();
11555 self.outbox
11556 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
11557 self.rejections.extend(rejections);
11558 Ok(true)
11559 }
11560
11561 pub fn upload_blob(
11568 &mut self,
11569 bytes: &[u8],
11570 media_type: Option<String>,
11571 name: Option<String>,
11572 ) -> Result<Value, String> {
11573 let blob_id = blob_id_for(bytes);
11574 let now = self.clock_now_ms();
11575 let tx = self
11576 .conn
11577 .transaction()
11578 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11579 tx.execute(
11580 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
11581 ON CONFLICT(blob_id) DO NOTHING",
11582 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
11583 )
11584 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11585 tx.execute(
11586 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
11587 rusqlite::params![blob_id, media_type, now],
11588 )
11589 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11590 tx.commit()
11591 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11592 self.enforce_blob_cache_cap();
11593 let mut obj = Map::new();
11594 obj.insert("blobId".to_owned(), Value::from(blob_id));
11595 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
11596 if let Some(mt) = media_type {
11597 obj.insert("mediaType".to_owned(), Value::from(mt));
11598 }
11599 if let Some(n) = name {
11600 obj.insert("name".to_owned(), Value::from(n));
11601 }
11602 Ok(Value::Object(obj))
11603 }
11604
11605 pub fn fetch_blob_bytes(
11609 &mut self,
11610 transport: &mut dyn Transport,
11611 blob_id_or_ref: &str,
11612 ) -> Result<FetchedBlob, (String, String)> {
11613 let simple = |m: String| ("client.failed".to_owned(), m);
11614 let (blob_id, byte_length) = if blob_id_or_ref.starts_with("sha256:") {
11617 (blob_id_or_ref.to_owned(), None)
11618 } else {
11619 let value: Value = serde_json::from_str(blob_id_or_ref)
11620 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
11621 (
11622 value
11623 .get("blobId")
11624 .and_then(Value::as_str)
11625 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
11626 .to_owned(),
11627 value.get("byteLength").and_then(Value::as_u64),
11628 )
11629 };
11630 if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
11631 return Ok(cached);
11632 }
11633 if !self.transport_enabled {
11634 return Err(("sync.offline".into(), "transport gate is closed".into()));
11635 }
11636 #[cfg(feature = "bench-internals")]
11642 let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
11643 let bytes = match transport
11644 .blob_download(&blob_id, byte_length)
11645 .map_err(|e| (e.code, e.message))?
11646 {
11647 BlobDownload::Bytes(bytes) => bytes,
11648 BlobDownload::Url {
11649 url,
11650 url_expires_at_ms,
11651 } => {
11652 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
11654 return Err((
11655 "sync.segment_expired".to_owned(),
11656 format!(
11657 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
11658 ),
11659 ));
11660 }
11661 transport
11662 .fetch_blob_url(&url, byte_length)
11663 .map_err(|e| (e.code, e.message))?
11664 }
11665 };
11666 #[cfg(feature = "bench-internals")]
11667 drop(download_phase);
11668 #[cfg(feature = "bench-internals")]
11669 let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
11670 if byte_length.is_some_and(|cap| bytes.len() as u64 > cap) {
11673 return Err((
11674 "sync.invalid_request".to_owned(),
11675 "blob exceeds its byteLength".to_owned(),
11676 ));
11677 }
11678 if blob_id_for(&bytes) != blob_id {
11679 return Err(simple(format!(
11680 "blob content address mismatch for {blob_id}"
11681 )));
11682 }
11683 #[cfg(feature = "bench-internals")]
11684 drop(validate_phase);
11685 #[cfg(feature = "bench-internals")]
11686 let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
11687 let now = self.clock_now_ms();
11688 self.conn
11689 .execute(
11690 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
11691 rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
11692 )
11693 .map_err(|e| simple(e.to_string()))?;
11694 self.conn
11695 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
11696 .map_err(|e| simple(e.to_string()))?;
11697 self.enforce_blob_cache_cap();
11698 #[cfg(feature = "bench-internals")]
11699 drop(insert_phase);
11700 self.get_cached_blob_bytes(&blob_id)
11701 .map_err(simple)?
11702 .ok_or_else(|| simple("blob cache write failed".to_owned()))
11703 }
11704
11705 fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
11706 #[cfg(feature = "bench-internals")]
11707 let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
11708 let mut stmt = self
11709 .conn
11710 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
11711 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11712 let mut rows = stmt
11713 .query(rusqlite::params![blob_id])
11714 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11715 if let Some(row) = rows
11716 .next()
11717 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
11718 {
11719 let bytes: Vec<u8> = row
11720 .get(0)
11721 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11722 let byte_length: i64 = row
11723 .get(1)
11724 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11725 let media_type: Option<String> = row
11726 .get(2)
11727 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11728 return Ok(Some(FetchedBlob {
11729 blob_id: blob_id.to_owned(),
11730 byte_length,
11731 bytes,
11732 media_type,
11733 }));
11734 }
11735 Ok(None)
11736 }
11737
11738 fn pending_blob_uploads(&self) -> Result<Vec<crate::round::PendingUpload>, TransportError> {
11740 let pending: Vec<(String, Option<String>)> = {
11741 let mut stmt = self
11742 .conn
11743 .prepare(
11744 "SELECT blob_id, media_type FROM (
11745 SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
11746 UNION ALL
11747 SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
11748 FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
11749 WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
11750 ) ORDER BY created_at_ms, blob_id",
11751 )
11752 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11753 let rows = stmt
11754 .query_map([], |row| {
11755 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
11756 })
11757 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11758 rows.collect::<Result<_, _>>()
11759 .map_err(|error| match error {
11760 rusqlite::Error::InvalidColumnType(..)
11761 | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
11762 "sync.local_corrupt",
11763 "Pending blob upload metadata is invalid",
11764 ),
11765 _ => TransportError::new("client.failed", error.to_string()),
11766 })?
11767 };
11768 let mut uploads = Vec::new();
11769 for (blob_id, media_type) in pending {
11770 let bytes = (|| {
11771 let stored: Option<(SqlValue, SqlValue)> = self
11772 .conn
11773 .query_row(
11774 "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
11775 rusqlite::params![blob_id],
11776 |row| Ok((row.get(0)?, row.get(1)?)),
11777 )
11778 .optional()
11779 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
11780 let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
11781 return Err(TransportError::new(
11782 "sync.local_corrupt",
11783 "Pending blob upload body is missing or corrupt",
11784 ));
11785 };
11786 if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
11787 return Err(TransportError::new(
11788 "sync.local_corrupt",
11789 "Pending blob upload body is missing or corrupt",
11790 ));
11791 }
11792 Ok(bytes)
11793 })();
11794 let failed = bytes.is_err();
11795 uploads.push(crate::round::PendingUpload {
11796 id: blob_id,
11797 bytes,
11798 media_type,
11799 });
11800 if failed {
11801 break;
11802 }
11803 }
11804 Ok(uploads)
11805 }
11806
11807 fn visible_blob_ids_sql(&self) -> String {
11808 let mut selects = Vec::new();
11809 for table in &self.schema.tables {
11810 for column in table
11811 .columns
11812 .iter()
11813 .filter(|column| column.ty == ColumnType::BlobRef)
11814 {
11815 let identifier = quote_ident(&column.name);
11816 selects.push(format!(
11817 "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
11818 WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
11819 AND json_type({identifier}, '$.blobId') = 'text'",
11820 quote_ident(&table.name),
11821 ));
11822 }
11823 }
11824 if selects.is_empty() {
11825 "SELECT NULL AS blob_id WHERE 0".to_owned()
11826 } else {
11827 selects.join(" UNION ")
11828 }
11829 }
11830
11831 fn enforce_blob_cache_cap(&self) {
11834 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
11835 return;
11836 };
11837 let mut total: i64 = self
11838 .conn
11839 .query_row(
11840 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
11841 [],
11842 |row| row.get(0),
11843 )
11844 .unwrap_or(0);
11845 if total <= max_bytes {
11846 return;
11847 }
11848 let query = format!(
11849 "SELECT blob_id, byte_length FROM _syncular_blobs
11850 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11851 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11852 AND blob_id NOT IN ({})
11853 ORDER BY created_at_ms ASC, blob_id ASC",
11854 self.visible_blob_ids_sql(),
11855 );
11856 let candidates: Vec<(String, i64)> = {
11857 let Ok(mut stmt) = self.conn.prepare(&query) else {
11858 return;
11859 };
11860 let Ok(rows) = stmt.query_map([], |row| {
11861 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
11862 }) else {
11863 return;
11864 };
11865 rows.filter_map(Result::ok).collect()
11866 };
11867 for (blob_id, byte_length) in candidates {
11868 if total <= max_bytes {
11869 break;
11870 }
11871 let _ = self.conn.execute(
11872 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
11873 rusqlite::params![blob_id],
11874 );
11875 total -= byte_length;
11876 }
11877 }
11878
11879 fn delete_unreferenced_cached_blobs(&mut self) -> Result<(), String> {
11880 #[cfg(feature = "bench-internals")]
11881 let _phase = self.benchmark_phases.start(Phase::BlobRetention);
11882 if !self.schema_has_blobs() {
11883 return Ok(());
11884 }
11885 self.rebuild_overlay_if_dirty()?;
11886 self.conn
11887 .execute(
11888 &format!(
11889 "DELETE FROM _syncular_blobs
11890 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
11891 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
11892 AND blob_id NOT IN ({})",
11893 self.visible_blob_ids_sql(),
11894 ),
11895 [],
11896 )
11897 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11898 Ok(())
11899 }
11900
11901 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
11906 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
11907 return sql.clone();
11908 }
11909 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
11910 columns.push(quote_ident("_syncular_version"));
11911 columns.extend(table.sealed_columns().iter().map(|name| quote_ident(name)));
11912 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
11913 let primary_key = quote_ident(&table.primary_key);
11914 let updates = columns
11915 .iter()
11916 .filter(|column| **column != primary_key)
11917 .map(|column| format!("{column}=excluded.{column}"))
11918 .collect::<Vec<_>>()
11919 .join(", ");
11920 let guard = if full_table == base_table(&table.name) {
11923 format!(" WHERE excluded._syncular_version > {full_table}._syncular_version")
11924 } else {
11925 String::new()
11926 };
11927 let sql = format!(
11928 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}{guard}",
11929 columns.join(", "),
11930 placeholders.join(", ")
11931 );
11932 self.insert_sql
11933 .borrow_mut()
11934 .insert(full_table.to_owned(), sql.clone());
11935 sql
11936 }
11937
11938 fn write_base_row(
11941 &self,
11942 table_name: &str,
11943 row: &Row,
11944 version: i64,
11945 sealed: Option<&SealedRow>,
11946 ) -> Result<bool, String> {
11947 let landed = self.write_row(&base_table(table_name), table_name, row, version, sealed)?;
11948 if landed {
11949 self.overlay_dirty.table(table_name);
11950 }
11951 Ok(landed)
11952 }
11953
11954 fn write_row(
11955 &self,
11956 full_table: &str,
11957 table_name: &str,
11958 row: &Row,
11959 version: i64,
11960 sealed: Option<&SealedRow>,
11961 ) -> Result<bool, String> {
11962 #[cfg(feature = "bench-internals")]
11963 let _phase = self.benchmark_phases.start(Phase::RowWrite);
11964 let table = self
11965 .schema
11966 .table(table_name)
11967 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
11968 let sql = self.insert_row_sql(full_table, table);
11969 let mut stmt = self
11970 .conn
11971 .prepare_cached(&sql)
11972 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
11973 let params = row
11974 .iter()
11975 .map(RowParam::Cell)
11976 .chain(std::iter::once(RowParam::Version(version)))
11977 .chain(
11978 [
11979 sealed.map(|sealed| sealed.key_id.as_str()),
11980 sealed.map(|sealed| sealed.envelopes.as_str()),
11981 ]
11982 .into_iter()
11983 .take(table.sealed_columns().len())
11984 .map(RowParam::Text),
11985 );
11986 stmt.execute(rusqlite::params_from_iter(params))
11987 .map(|changed| changed > 0)
11988 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
11989 }
11990
11991 fn delete_base_row(
11994 &self,
11995 table_name: &str,
11996 row_id: &str,
11997 version: i64,
11998 ) -> Result<bool, String> {
11999 let table = self
12000 .schema
12001 .table(table_name)
12002 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
12003 let sql = format!(
12004 "DELETE FROM {} WHERE {} AND _syncular_version < ?2",
12005 base_table(table_name),
12006 row_id_predicate(table)
12007 );
12008 let mut stmt = self
12009 .conn
12010 .prepare_cached(&sql)
12011 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12012 let landed = stmt
12013 .execute(rusqlite::params![row_id, version])
12014 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12015 > 0;
12016 if landed {
12017 self.overlay_dirty.table(table_name);
12018 }
12019 Ok(landed)
12020 }
12021
12022 fn has_acknowledged_rows(&self) -> Result<bool, String> {
12023 self.conn
12024 .query_row(
12025 "SELECT EXISTS(SELECT 1 FROM _syncular_acked_rows)",
12026 [],
12027 |row| row.get(0),
12028 )
12029 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12030 }
12031
12032 fn acknowledged_operations(
12033 &self,
12034 rows: Option<&HashSet<(&str, &str)>>,
12035 tables: Option<&BTreeSet<&str>>,
12036 ) -> Result<Vec<OutboxOp>, String> {
12037 let keys = rows.map(|rows| serde_json::to_string(&rows).expect("row keys"));
12038 let mut statement = self
12039 .conn
12040 .prepare_cached(if keys.is_some() {
12041 ACKED_ROWS_BY_ROW
12042 } else {
12043 "SELECT op_json,intent_json FROM _syncular_acked_rows WHERE ?1 IS NULL AND (?2 IS NULL OR tbl IN(SELECT value FROM json_each(?2))) ORDER BY commit_seq,idx"
12044 })
12045 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12046 let rows = statement
12047 .query_map(
12048 rusqlite::params![
12049 keys,
12050 tables.map(|tables| serde_json::to_string(tables).expect("table keys"))
12051 ],
12052 |row| {
12053 #[cfg(test)]
12054 self.acknowledged_replay_count
12055 .set(self.acknowledged_replay_count.get() + 1);
12056 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
12057 },
12058 )
12059 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12060 rows.map(|row| {
12061 let (op, intent) =
12062 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12063 let op: CommitOperation =
12064 serde_json::from_str(&op).map_err(|_| "sync.local_corrupt".to_owned())?;
12065 let table = self
12066 .schema
12067 .table(&op.table)
12068 .ok_or_else(|| "sync.unknown_table".to_owned())?;
12069 let values = if op.op == "upsert" && self.visible_row(table, &op.row_id)?.is_none() {
12070 Some(serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?)
12071 } else {
12072 op.values.clone()
12073 };
12074 Ok(OutboxOp {
12075 values,
12076 ..op.into()
12077 })
12078 })
12079 .collect()
12080 }
12081
12082 fn drop_acknowledged_rows(
12083 &self,
12084 matches: impl Fn(&str, &Map<String, Value>) -> bool,
12085 ) -> Result<Vec<OutboxCommit>, String> {
12086 let mut statement = self
12087 .conn
12088 .prepare("SELECT commit_id,tbl,intent_json,id FROM _syncular_acked_rows")
12089 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12090 let rows = statement
12091 .query_map([], |row| {
12092 Ok((
12093 row.get::<_, String>(0)?,
12094 row.get::<_, String>(1)?,
12095 row.get::<_, String>(2)?,
12096 row.get::<_, String>(3)?,
12097 ))
12098 })
12099 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12100 let mut doomed = HashSet::new();
12101 for row in rows {
12102 let (id, table, intent, row_id) =
12103 row.map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12104 let values =
12105 serde_json::from_str(&intent).map_err(|_| "sync.local_corrupt".to_owned())?;
12106 let base_matches = self
12107 .schema
12108 .table(&table)
12109 .map(|schema| self.stored_row(schema, &row_id, true))
12110 .transpose()?
12111 .flatten()
12112 .is_some_and(|(base, _)| {
12113 let schema = self.schema.table(&table).expect("acknowledged table");
12114 let values = schema
12115 .columns
12116 .iter()
12117 .zip(base.iter())
12118 .map(|(column, value)| (column.name.clone(), column_value_to_json(value)))
12119 .collect();
12120 matches(&table, &values)
12121 });
12122 if matches(&table, &values) || base_matches {
12123 doomed.insert(id);
12124 }
12125 }
12126 let mut dropped = Vec::new();
12127 for id in doomed {
12128 let mut statement = self
12129 .conn
12130 .prepare("SELECT op_json FROM _syncular_acked_rows WHERE commit_id=?1 ORDER BY idx")
12131 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12132 let operations = statement
12133 .query_map([&id], |row| row.get::<_, String>(0))
12134 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12135 let mut ops = Vec::new();
12136 for operation in operations {
12137 let json = operation
12138 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12139 let op: CommitOperation =
12140 serde_json::from_str(&json).map_err(|_| "sync.local_corrupt".to_owned())?;
12141 self.overlay_dirty.table(&op.table);
12142 ops.push(OutboxOp {
12143 table: op.table,
12144 row_id: op.row_id,
12145 upsert: op.op == "upsert",
12146 base_version: op.base_version,
12147 values: op.values,
12148 insert: op.insert,
12149 });
12150 }
12151 dropped.push(OutboxCommit {
12152 client_commit_id: id.clone(),
12153 ops,
12154 });
12155 self.conn
12156 .execute("DELETE FROM _syncular_acked_rows WHERE commit_id=?1", [id])
12157 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12158 }
12159 Ok(dropped)
12160 }
12161
12162 fn rebuild_overlay_if_dirty(&mut self) -> Result<(), String> {
12165 if self.overlay_dirty.get() {
12166 self.rebuild_overlay()?;
12167 }
12168 Ok(())
12169 }
12170
12171 fn rebuild_overlay(&mut self) -> Result<(), String> {
12175 #[cfg(feature = "bench-internals")]
12176 let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
12177 #[cfg(test)]
12178 self.overlay_rebuild_count
12179 .set(self.overlay_rebuild_count.get() + 1);
12180 let dirty = self.overlay_dirty.snapshot();
12181 let tables: Vec<_> = self
12182 .schema
12183 .tables
12184 .iter()
12185 .filter(|table| {
12186 dirty
12187 .as_ref()
12188 .is_none_or(|names| names.is_empty() || names.contains(&table.name))
12189 })
12190 .cloned()
12191 .collect();
12192 let names: BTreeSet<_> = tables.iter().map(|table| table.name.as_str()).collect();
12193 self.begin_observation("syncular_overlay")?;
12194 let rebuilt = (|| {
12195 for table in &tables {
12196 for index in &table.fts_indexes {
12197 self.drop_fts_triggers(index)?;
12198 }
12199 let visible = visible_table(&table.name);
12200 let base = base_table(&table.name);
12201 self.conn
12202 .execute_batch(&format!(
12203 "DELETE FROM {visible}; INSERT INTO {visible} SELECT * FROM {base}"
12204 ))
12205 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12206 }
12207 self.replay_overlay(&names, None)?;
12208 for table in &tables {
12209 for index in &table.fts_indexes {
12210 self.rebuild_fts_projection(table, index)?;
12211 self.create_fts_triggers(table, index)?;
12212 }
12213 }
12214 self.conn
12215 .execute_batch("RELEASE syncular_overlay")
12216 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12217 Ok(())
12218 })();
12219 if rebuilt.is_err() {
12220 self.conn
12221 .execute_batch("ROLLBACK TO syncular_overlay; RELEASE syncular_overlay")
12222 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12223 } else {
12224 self.overlay_dirty.set(false);
12225 }
12226 rebuilt
12227 }
12228
12229 fn replay_overlay(
12233 &self,
12234 tables: &BTreeSet<&str>,
12235 rows: Option<&HashSet<(&str, &str)>>,
12236 ) -> Result<(), String> {
12237 let replays = |op: &&OutboxOp| {
12238 tables.contains(op.table.as_str())
12239 && rows.is_none_or(|rows| rows.contains(&(op.table.as_str(), op.row_id.as_str())))
12240 };
12241 self.apply_outbox_ops(&self.acknowledged_operations(rows, Some(tables))?)?;
12242 for (commit, initial) in self.failed_commits.iter() {
12243 for operation in commit.ops.iter().filter(replays) {
12244 let Some(table) = self.schema.table(&operation.table) else {
12245 continue;
12246 };
12247 if operation.upsert
12248 && self.visible_row(table, &operation.row_id)?.is_none()
12249 && operation.presented_values(table).is_some_and(|values| {
12250 table
12251 .columns
12252 .iter()
12253 .all(|column| values.contains_key(&column.name))
12254 })
12255 {
12256 let key = serde_json::to_string(&(&operation.table, &operation.row_id))
12257 .expect("row key serialization");
12258 if let Some(Value::Object(values)) = initial.get(&key) {
12259 self.apply_outbox_ops(&[OutboxOp {
12260 values: Some(values.clone()),
12261 ..operation.clone()
12262 }])?;
12263 }
12264 }
12265 }
12266 self.apply_outbox_ops(commit.ops.iter().filter(replays))?;
12267 }
12268 self.apply_outbox_ops(
12269 self.outbox
12270 .iter()
12271 .flat_map(|commit| &commit.ops)
12272 .filter(replays),
12273 )
12274 }
12275
12276 fn apply_outbox_ops<'a>(
12281 &self,
12282 ops: impl IntoIterator<Item = &'a OutboxOp>,
12283 ) -> Result<(), String> {
12284 #[cfg(feature = "bench-internals")]
12285 let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
12286 for op in ops {
12287 #[cfg(feature = "bench-internals")]
12288 if let Some(phase) = &mut phase {
12289 phase.unit();
12290 }
12291 if op.upsert {
12292 if let Some((table, values)) = self
12293 .schema
12294 .table(&op.table)
12295 .filter(|table| table.indexes.iter().any(|index| index.unique))
12296 .and_then(|table| Some((table, op.presented_values(table)?)))
12297 {
12298 let mut intended: Map<String, Value> = self
12299 .visible_row(table, &op.row_id)?
12300 .map(|(row, _)| {
12301 table
12302 .columns
12303 .iter()
12304 .zip(&row)
12305 .map(|(column, value)| {
12306 (column.name.clone(), column_value_to_json(value))
12307 })
12308 .collect()
12309 })
12310 .unwrap_or_default();
12311 intended.extend(values.into_owned());
12312 if !self.unique_conflicts(table, &intended, false)?.is_empty() {
12313 continue;
12314 }
12315 }
12316 }
12317 self.apply_outbox_op(op)?;
12318 }
12319 Ok(())
12320 }
12321
12322 fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
12323 let Some(table) = self.schema.table(&op.table) else {
12324 return Ok(());
12325 };
12326 let visible = visible_table(&table.name);
12327 if !op.upsert {
12328 let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
12329 self.conn
12330 .prepare_cached(&sql)
12331 .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
12332 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12333 return Ok(());
12334 }
12335 let Some(values) = op.presented_values(table) else {
12339 return Ok(());
12340 };
12341 let full = table
12347 .columns
12348 .iter()
12349 .all(|column| values.contains_key(&column.name));
12350 let local = self.visible_row(table, &op.row_id)?;
12351 if local.is_none() && !full {
12352 return Ok(());
12353 }
12354 let mut row: Row = Vec::with_capacity(table.columns.len());
12355 for (index, column) in table.columns.iter().enumerate() {
12356 let incoming = match values.get(&column.name) {
12357 Some(raw) => json_to_column_value(
12358 if raw.get("$bytes").is_some()
12359 && table
12360 .encrypted_columns
12361 .iter()
12362 .any(|encrypted| encrypted.index == index)
12363 {
12364 &table.wire_columns[index]
12365 } else {
12366 column
12367 },
12368 Some(raw),
12369 ),
12370 None => Ok(local
12371 .as_ref()
12372 .and_then(|(values, _)| values.get(index).cloned())
12373 .flatten()),
12374 };
12375 row.push(incoming.map_err(|_| {
12376 "sync.local_corrupt: persisted optimistic row value is invalid".to_owned()
12377 })?);
12378 }
12379 let version = local.as_ref().map_or(-1, |(_, version)| *version);
12380 self.write_row(&visible, &table.name, &row, version, None)
12381 .map(|_| ())
12382 .map_err(|error| {
12383 if error.starts_with("UNIQUE constraint failed:") {
12384 "sync.constraint_violation: local write violates a unique constraint".to_owned()
12385 } else {
12386 error
12387 }
12388 })
12389 }
12390
12391 fn visible_row_exists(&self, table: &TableSchema, row_id: &str) -> Result<bool, String> {
12395 self.conn
12396 .query_row(
12397 &format!(
12398 "SELECT EXISTS(SELECT 1 FROM {} WHERE {})",
12399 visible_table(&table.name),
12400 row_id_predicate(table)
12401 ),
12402 rusqlite::params![row_id],
12403 |row| row.get(0),
12404 )
12405 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))
12406 }
12407
12408 fn refuse_sealed_row(&self, table: &TableSchema, row_id: &str) -> Result<(), ClientError> {
12411 if !table.has_encrypted_columns() {
12412 return Ok(());
12413 }
12414 let key_id: Option<String> = self
12415 .conn
12416 .query_row(
12417 &format!(
12418 "SELECT \"_sync_sealed_key\" FROM {} WHERE {}",
12419 visible_table(&table.name),
12420 row_id_predicate(table)
12421 ),
12422 rusqlite::params![row_id],
12423 |row| row.get(0),
12424 )
12425 .optional()
12426 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12427 .flatten();
12428 match key_id {
12429 None => Ok(()),
12430 Some(key_id) => Err(ClientError {
12431 code: "client.encryption_key_missing".to_owned(),
12432 message: "the row is sealed under a key this device does not hold".to_owned(),
12433 details: Some(json!({ "table": table.name, "keyId": key_id })),
12434 retryable: false,
12435 }),
12436 }
12437 }
12438
12439 fn reveal_sealed_rows(&mut self) -> Result<(), TransportError> {
12443 let storage = |message: String| TransportError::new("storage.failed", message);
12444 let tables: Vec<TableSchema> = self
12445 .schema
12446 .tables
12447 .iter()
12448 .filter(|table| table.has_encrypted_columns())
12449 .cloned()
12450 .collect();
12451 for table in &tables {
12452 let base = base_table(&table.name);
12453 let pk = quote_ident(&table.primary_key);
12454 let sealed: Vec<(SqlValue, i64, String)> = self
12455 .conn
12456 .prepare_cached(&format!(
12457 "SELECT {pk}, \"_syncular_version\", \"_sync_sealed\" FROM {base} WHERE \"_sync_sealed_key\" IS NOT NULL"
12458 ))
12459 .and_then(|mut statement| {
12460 statement
12461 .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))?
12462 .collect()
12463 })
12464 .map_err(|error| storage(Self::sqlite_failure(&self.storage_failure, error)))?;
12465 let mut revealed = Vec::new();
12466 for (key, version, envelopes) in sealed {
12467 if let Some(row) =
12468 crate::values::reveal_sealed_row(table, &envelopes, &self.encryption)?
12469 {
12470 revealed.push((key, version, row));
12471 }
12472 }
12473 if revealed.is_empty() {
12474 continue;
12475 }
12476 self.begin_observation("syncular_reveal").map_err(storage)?;
12477 let outcome = (|| {
12478 let assignments = table
12479 .encrypted_columns
12480 .iter()
12481 .map(|enc| format!("{}=?", quote_ident(&table.columns[enc.index].name)))
12482 .collect::<Vec<_>>()
12483 .join(",");
12484 let mut statement = self
12485 .conn
12486 .prepare_cached(&format!(
12487 "UPDATE {base} SET {assignments},\"_sync_sealed_key\"=NULL,\"_sync_sealed\"=NULL WHERE {pk}=? AND \"_syncular_version\"=?"
12488 ))
12489 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12490 for (key, version, row) in &revealed {
12491 let mut params = table
12492 .encrypted_columns
12493 .iter()
12494 .map(|enc| owned_sql_value(RowParam::Cell(&row[enc.index])))
12495 .collect::<Result<Vec<_>, _>>()?;
12496 params.push(key.clone());
12497 params.push(SqlValue::Integer(*version));
12498 statement
12499 .execute(rusqlite::params_from_iter(params))
12500 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12501 }
12502 drop(statement);
12503 self.overlay_dirty.table(&table.name);
12504 self.rebuild_overlay()?;
12505 let mut batch = ChangeAccumulator::default();
12506 batch.table(&table.name);
12507 self.finish_observation("syncular_reveal", batch)
12508 })();
12509 if outcome.is_err() {
12510 self.rollback_observation("syncular_reveal");
12511 }
12512 outcome.map_err(storage)?;
12513 }
12514 Ok(())
12515 }
12516
12517 fn visible_row(&self, table: &TableSchema, row_id: &str) -> Result<Option<(Row, i64)>, String> {
12520 self.stored_row(table, row_id, false)
12521 }
12522
12523 fn stored_row(
12524 &self,
12525 table: &TableSchema,
12526 row_id: &str,
12527 base: bool,
12528 ) -> Result<Option<(Row, i64)>, String> {
12529 let sql = format!(
12530 "SELECT * FROM {} WHERE {}",
12531 if base {
12532 base_table(&table.name)
12533 } else {
12534 visible_table(&table.name)
12535 },
12536 row_id_predicate(table)
12537 );
12538 let mut stmt = self
12539 .conn
12540 .prepare_cached(&sql)
12541 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12542 let mut rows = stmt
12543 .query(rusqlite::params![row_id])
12544 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12545 let Some(row) = rows
12546 .next()
12547 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?
12548 else {
12549 return Ok(None);
12550 };
12551 let mut values = Vec::with_capacity(table.columns.len());
12552 for (index, column) in table.columns.iter().enumerate() {
12553 let raw = row
12554 .get_ref(index)
12555 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12556 if let rusqlite::types::ValueRef::Blob(bytes) = raw {
12557 if table
12558 .encrypted_columns
12559 .iter()
12560 .any(|encrypted| encrypted.index == index)
12561 {
12562 values.push(Some(ColumnValue::Bytes(bytes.to_vec())));
12563 continue;
12564 }
12565 }
12566 let json = sql_ref_to_json(column, raw);
12567 values.push(json_to_column_value(column, Some(&json)).map_err(|_| {
12568 "sync.local_corrupt: persisted local row value is invalid".to_owned()
12569 })?);
12570 }
12571 let version: i64 = row
12572 .get(table.columns.len())
12573 .map_err(|error| Self::sqlite_failure(&self.storage_failure, error))?;
12574 Ok(Some((values, version)))
12575 }
12576
12577 #[must_use]
12581 pub fn realtime_policy(&self) -> RealtimePolicy {
12582 self.realtime_policy
12583 }
12584
12585 #[must_use]
12587 pub fn realtime_state(&self) -> RealtimeState {
12588 self.realtime_state
12589 }
12590
12591 pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
12594 if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
12595 return Err(
12596 "sync.invalid_request: disconnect realtime before changing the realtime policy"
12597 .to_owned(),
12598 );
12599 }
12600 self.realtime_policy = policy;
12601 match policy {
12602 RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
12603 RealtimePolicy::Required | RealtimePolicy::Optional
12604 if self.realtime_state == RealtimeState::Disabled =>
12605 {
12606 self.set_realtime_state(RealtimeState::Disconnected, None);
12607 }
12608 RealtimePolicy::Required | RealtimePolicy::Optional => {}
12609 }
12610 Ok(())
12611 }
12612
12613 fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
12616 self.realtime_state = state;
12617 self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
12618 if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
12619 self.realtime_reason_code = None;
12620 self.realtime_retry_delay_ms = None;
12621 }
12622 }
12623
12624 #[must_use]
12625 fn realtime_connected(&self) -> bool {
12626 self.realtime_state == RealtimeState::Connected
12627 }
12628
12629 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
12630 if !self.transport_enabled {
12631 return Err("sync.offline: transport gate is closed".into());
12632 }
12633 if self.realtime_policy == RealtimePolicy::Off {
12634 return Err(
12635 "sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
12636 );
12637 }
12638 if self.realtime_connected() {
12639 return Ok(());
12640 }
12641 self.realtime_state = RealtimeState::Connecting;
12642 match transport.realtime_connect_for_client(&self.client_id) {
12643 Ok(()) => {
12644 self.set_realtime_state(RealtimeState::Connected, None);
12645 Ok(())
12646 }
12647 Err(error) => {
12648 self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
12649 Err(format!("{}: {}", error.code, error.message))
12650 }
12651 }
12652 }
12653
12654 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
12655 self.cancel_sync_round();
12656 if self.realtime_state == RealtimeState::Disconnected
12659 || self.realtime_state == RealtimeState::Disabled
12660 {
12661 return;
12662 }
12663 let _ = transport.realtime_close();
12664 self.set_realtime_state(
12665 if self.realtime_policy == RealtimePolicy::Off {
12666 RealtimeState::Disabled
12667 } else {
12668 RealtimeState::Disconnected
12669 },
12670 None,
12671 );
12672 self.presence.clear(); }
12674
12675 pub fn realtime_lost(&mut self, transport: &mut dyn Transport, error: &TransportError) {
12681 if self.realtime_state != RealtimeState::Connected {
12682 return;
12683 }
12684 self.cancel_sync_round();
12685 let _ = transport.realtime_close();
12686 self.set_realtime_state(RealtimeState::Lost, Some(&error.code));
12687 self.realtime_retry_delay_ms = Some(self.schedule_background_retry());
12688 self.presence.clear();
12689 }
12690
12691 pub fn set_presence(
12697 &mut self,
12698 transport: &mut dyn Transport,
12699 scope_key: &str,
12700 doc: Option<&Value>,
12701 ) -> Result<(), String> {
12702 if !self.transport_enabled {
12703 return Err("sync.offline: transport gate is closed".into());
12704 }
12705 if !self.realtime_connected() {
12706 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
12707 }
12708 let text = encode_presence_publish(scope_key, doc);
12709 transport
12710 .realtime_send(&text)
12711 .map_err(|e| format!("{}: {}", e.code, e.message))
12712 }
12713
12714 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
12716 self.presence
12717 .get(scope_key)
12718 .map(|peers| peers.values().cloned().collect())
12719 .unwrap_or_default()
12720 }
12721
12722 fn apply_presence(
12724 &mut self,
12725 scope_key: String,
12726 kind: Option<PresenceKind>,
12727 actor_id: Option<String>,
12728 client_id: Option<String>,
12729 doc: Option<Value>,
12730 error: Option<String>,
12731 ) {
12732 if error.is_some() {
12735 return;
12736 }
12737 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
12738 return;
12739 };
12740 let peer_key = format!("{actor_id} {client_id}");
12741 match kind {
12742 PresenceKind::Leave => {
12743 if let Some(peers) = self.presence.get_mut(&scope_key) {
12744 peers.remove(&peer_key);
12745 if peers.is_empty() {
12746 self.presence.remove(&scope_key);
12747 }
12748 }
12749 }
12750 _ => {
12751 let doc = match doc {
12752 Some(Value::Object(_)) => doc.unwrap(),
12753 _ => return,
12754 };
12755 self.presence.entry(scope_key).or_default().insert(
12756 peer_key,
12757 PresencePeer {
12758 actor_id,
12759 client_id,
12760 doc,
12761 },
12762 );
12763 }
12764 }
12765 }
12766
12767 pub fn on_realtime_text(&mut self, text: &str) {
12769 if self.stopped || !self.transport_enabled || self.security_preflight {
12770 return;
12771 }
12772 match parse_control(text) {
12773 Ok(ControlMessage::Hello { requires_sync, .. }) => {
12774 if requires_sync {
12775 self.set_sync_needed(true, true);
12777 }
12778 }
12779 Ok(ControlMessage::Presence {
12780 scope_key,
12781 kind,
12782 actor_id,
12783 client_id,
12784 doc,
12785 error,
12786 ..
12787 }) => {
12788 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
12789 }
12790 Ok(ControlMessage::Wake { .. }) => {
12791 self.set_sync_needed(true, true);
12793 }
12794 _ => {}
12795 }
12796 }
12797
12798 pub fn on_realtime_binary_queued(&mut self, bytes: &[u8]) -> Vec<String> {
12802 let mut transport = crate::round::DownloadResults::default();
12803 self.on_realtime_binary(&mut transport, bytes);
12804 transport.controls
12805 }
12806
12807 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
12813 if self.stopped || !self.transport_enabled || self.security_preflight {
12814 return;
12815 }
12816 #[cfg(feature = "bench-internals")]
12817 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
12818 let message = match decode_message(bytes) {
12819 Ok(m) if m.msg_kind == MsgKind::Response => m,
12820 _ => {
12821 self.set_sync_needed(true, true);
12822 return;
12823 }
12824 };
12825 #[cfg(feature = "bench-internals")]
12826 drop(decode_phase);
12827 if let Some(Frame::RespHeader {
12830 required_schema_version: Some(required),
12831 latest_schema_version,
12832 ..
12833 }) = message.frames.first()
12834 {
12835 let floor = SchemaFloor {
12836 required_schema_version: Some(*required),
12837 latest_schema_version: *latest_schema_version,
12838 };
12839 if self.set_schema_floor(Some(floor)).is_err() {
12840 self.set_sync_needed(true, true);
12841 }
12842 return;
12843 }
12844 let through = message.frames.iter().find_map(|frame| match frame {
12845 Frame::LiveEnd { through_seq, .. } => Some(*through_seq),
12846 _ => None,
12847 });
12848 let epoch = self.get_meta(LOG_EPOCH_KEY).ok().flatten();
12849 let mut dropped = !matches!(message.frames.first(), Some(Frame::RespHeader {
12850 reset_required: false,
12851 log_epoch,
12852 ..
12853 }) if Some(log_epoch) == epoch.as_ref());
12854 let mut applied: Option<i64> = None;
12855 dropped |= self.begin_coalesced_apply().is_err();
12856 let mut receivers: Option<Vec<UnitRecord>> = None;
12857 for frame in message.frames {
12858 if dropped {
12859 break;
12860 }
12861 match frame {
12862 Frame::LiveStart { status, .. } => {
12863 let Ok(cursor) = self.stored(replica_cursor(&self.conn)) else {
12864 dropped = true;
12865 break;
12866 };
12867 if status != LiveStatus::Active || cursor < 0 {
12868 dropped = true;
12869 } else if !through.is_some_and(|through| through <= cursor) {
12870 receivers = self.stored(load_units(&self.conn)).ok().map(|units| {
12872 units
12873 .into_iter()
12874 .filter(|unit| {
12875 unit.status == UnitStatus::Active
12876 && unit.echo.is_some()
12877 && unit.cursor.is_none()
12878 })
12879 .collect()
12880 });
12881 dropped = receivers.is_none();
12882 }
12883 }
12884 Frame::Commit {
12885 commit_seq,
12886 tables,
12887 changes,
12888 ..
12889 } => {
12890 let Some(receivers) = receivers.clone() else {
12891 continue;
12892 };
12893 dropped = self
12894 .apply_live_commit(
12895 &receivers,
12896 &tables,
12897 changes,
12898 commit_seq,
12899 AppliedServerCommitSource::Delta,
12900 )
12901 .is_err();
12902 }
12903 Frame::LiveEnd { through_seq, .. } if receivers.is_some() => {
12904 dropped = self
12905 .set_meta(REPLICA_CURSOR_KEY, &through_seq.to_string())
12906 .is_err();
12907 applied = Some(through_seq);
12908 }
12909 Frame::Error { .. } => dropped = true,
12910 _ => {}
12911 }
12912 }
12913 if self.finish_coalesced_apply().is_err() || dropped {
12916 applied = None;
12917 dropped = true;
12918 }
12919 if let Some(cursor) = applied {
12920 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12922 let _ = transport.realtime_send(&ack);
12923 }
12924 if dropped {
12925 self.set_sync_needed(true, true);
12926 }
12927 }
12928
12929 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
12932 if !self.realtime_connected() {
12933 return;
12934 }
12935 if let Ok(cursor @ 0..) = self.stored(replica_cursor(&self.conn)) {
12936 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
12937 let _ = transport.realtime_send(&ack);
12938 }
12939 }
12940}
12941
12942#[cfg(test)]
12943mod previous_version_wiring_tests;
12944
12945#[cfg(test)]
12946mod identity_tests;
12947
12948#[cfg(test)]
12949mod insert_intent_tests;
12950
12951#[cfg(test)]
12952mod robustness_tests;
12953
12954#[cfg(test)]
12955mod range_image_tests;