1use std::cell::{Cell, RefCell};
11use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
12
13use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
14use rusqlite::{Connection, OpenFlags, OptionalExtension};
15use serde::{Deserialize, Serialize};
16use serde_json::{Map, Value};
17use sha2::{Digest, Sha256};
18use ssp2::decode::WIRE_VERSION;
19use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
20use ssp2::primitives::RawJson;
21use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
22use ssp2::{
23 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
24 PresenceKind,
25};
26
27use crate::api::{
28 ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
29 ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
30 ClientDiagnosticsStorage, ClientLimits, CommandEffects, CommitOperation,
31 CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution,
32 CommitOutcomeStatus, ConflictRecord, CoverageSnapshot, DiagnosticLastChange,
33 DiagnosticLastRound, DiagnosticRoundCounters, DiagnosticSubscription, LeaseState,
34 LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget, LocalDataRebootstrapInput,
35 LocalDataRebootstrapResult, Mutation, PresencePeer, QueryRow, QuerySnapshot, QueryValue,
36 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
37 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
38 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
39 CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
40};
41use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
42use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
43use crate::values::{
44 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes, encode_row_json,
45 json_to_column_value, json_to_scope_map, normalize_values_casing, render_row_id_json,
46 scope_map_to_json, sort_scope_map,
47};
48
49const DEFAULT_ACCEPT: u8 = 0b0111;
54const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
55const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
56const ACCEPT_SQLITE: u8 = 1 << 2;
57const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
58const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
59
60const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
62const LOCAL_REVISION_KEY: &str = "localRevision";
63const CLIENT_ID_KEY: &str = "clientId";
64const LEASE_STATE_KEY: &str = "leaseState";
65const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
66const LOG_EPOCH_KEY: &str = "logEpoch";
67const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
72const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
73const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
74const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
77
78#[derive(Debug, Serialize, Deserialize)]
79#[serde(rename_all = "camelCase", deny_unknown_fields)]
80struct PersistedLocalDataRebootstrapReceipt {
81 version: u8,
82 retained_commits: u64,
83 reset_subscriptions: u64,
84}
85
86fn invalid_local_rebootstrap_receipt() -> String {
87 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
88}
89
90fn encode_local_rebootstrap_receipt(
91 retained_commits: usize,
92 reset_subscriptions: usize,
93) -> Result<String, String> {
94 let retained_commits =
95 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
96 let reset_subscriptions =
97 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
98 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
99 return Err(invalid_local_rebootstrap_receipt());
100 }
101 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
102 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
103 retained_commits,
104 reset_subscriptions,
105 })
106 .map_err(|_| invalid_local_rebootstrap_receipt())
107}
108
109fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
110 if value == "v1" {
112 return Ok((0, 0));
113 }
114 let receipt: PersistedLocalDataRebootstrapReceipt =
115 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
116 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
117 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
118 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
119 {
120 return Err(invalid_local_rebootstrap_receipt());
121 }
122 Ok((
123 usize::try_from(receipt.retained_commits)
124 .map_err(|_| invalid_local_rebootstrap_receipt())?,
125 usize::try_from(receipt.reset_subscriptions)
126 .map_err(|_| invalid_local_rebootstrap_receipt())?,
127 ))
128}
129
130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
131enum SubState {
132 Active,
133 Revoked,
134 Failed,
135}
136
137#[cfg(test)]
138mod observation_tests {
139 use super::*;
140 use crate::native_transport::HostTransport;
141 use serde_json::json;
142
143 fn client() -> SyncClient {
144 SyncClient::new(
145 "retry-test".to_owned(),
146 &json!({
147 "version": 1,
148 "tables": [{
149 "name": "tasks",
150 "primaryKey": "id",
151 "columns": [
152 { "name": "id", "type": "string", "nullable": false },
153 { "name": "project_id", "type": "string", "nullable": false }
154 ],
155 "scopes": [{ "pattern": "project:{project_id}" }]
156 }]
157 }),
158 ClientLimits::default(),
159 )
160 .expect("test client")
161 }
162
163 #[test]
164 fn background_retry_deadlines_back_off_and_reset() {
165 let mut client = client();
166 client.schedule_background_retry();
167 assert!(matches!(
168 client.drain_sync_intents().as_slice(),
169 [SyncIntent::Background { delay_ms: 250 }]
170 ));
171 client.schedule_background_retry();
172 assert!(matches!(
173 client.drain_sync_intents().as_slice(),
174 [SyncIntent::Background { delay_ms: 500 }]
175 ));
176 client.reset_background_retry();
177 client.schedule_background_retry();
178 assert!(matches!(
179 client.drain_sync_intents().as_slice(),
180 [SyncIntent::Background { delay_ms: 250 }]
181 ));
182 }
183
184 #[test]
185 fn log_epoch_reset_requests_an_immediate_follow_up_round() {
186 let mut client = client();
187 client
188 .subscribe(
189 "epoch-tasks".to_owned(),
190 "tasks".to_owned(),
191 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
192 None,
193 )
194 .expect("subscribe");
195 client.drain_sync_intents();
196
197 client
198 .run_log_epoch_reset("epoch-2")
199 .expect("reset partition log epoch");
200
201 assert!(matches!(
202 client.drain_sync_intents().as_slice(),
203 [SyncIntent::Interactive]
204 ));
205 }
206
207 struct CountingRealtimeTransport {
208 connects: usize,
209 closes: usize,
210 }
211
212 impl Transport for CountingRealtimeTransport {
213 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
214 Err(TransportError::new("sync.transport_failed", "offline"))
215 }
216
217 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
218 Err(TransportError::new("sync.transport_failed", "offline"))
219 }
220
221 fn download_segment(
222 &mut self,
223 _request: &SegmentRequest,
224 ) -> Result<Vec<u8>, TransportError> {
225 Err(TransportError::new("sync.transport_failed", "offline"))
226 }
227
228 fn realtime_connect(&mut self) -> Result<(), TransportError> {
229 self.connects += 1;
230 Ok(())
231 }
232
233 fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
234 Ok(())
235 }
236
237 fn realtime_close(&mut self) -> Result<(), TransportError> {
238 self.closes += 1;
239 Ok(())
240 }
241 }
242
243 #[test]
244 fn realtime_connection_ownership_is_idempotent() {
245 let mut client = client();
246 let mut transport = CountingRealtimeTransport {
247 connects: 0,
248 closes: 0,
249 };
250 client
251 .connect_realtime(&mut transport)
252 .expect("first connect");
253 client
254 .connect_realtime(&mut transport)
255 .expect("idempotent connect");
256 assert_eq!(transport.connects, 1);
257 client.disconnect_realtime(&mut transport);
258 client.disconnect_realtime(&mut transport);
259 assert_eq!(transport.closes, 1);
260 client
261 .connect_realtime(&mut transport)
262 .expect("deliberate reconnect");
263 assert_eq!(transport.connects, 2);
264 }
265
266 #[test]
267 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
268 let mut client = client();
269 client
270 .subscribe(
271 "repair-tasks".to_owned(),
272 "tasks".to_owned(),
273 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
274 None,
275 )
276 .expect("subscribe");
277 {
278 let sub = client
279 .subs
280 .iter_mut()
281 .find(|sub| sub.id == "repair-tasks")
282 .expect("subscription");
283 sub.cursor = 42;
284 sub.synced_once = true;
285 let persisted = sub.clone();
286 client.persist_sub(&persisted);
287 }
288 for table in ["_syncular_base_tasks", "tasks"] {
289 client
290 .conn
291 .execute(
292 &format!(
293 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
294 ),
295 rusqlite::params!["server-row", "p1"],
296 )
297 .expect("seed server row");
298 }
299 let pending = client
300 .mutate(vec![Mutation::Upsert {
301 table: "tasks".to_owned(),
302 values: Map::from_iter([
303 ("id".to_owned(), Value::from("offline-row")),
304 ("project_id".to_owned(), Value::from("p1")),
305 ]),
306 base_version: None,
307 }])
308 .expect("queue offline work");
309 client.drain_change_batches();
310 client.drain_sync_intents();
311
312 assert_eq!(
313 client
314 .rebootstrap_local_data(&LocalDataRebootstrapInput {
315 rebootstrap_id: "support-case-001".to_owned(),
316 })
317 .expect("rebootstrap"),
318 LocalDataRebootstrapResult {
319 already_applied: false,
320 retained_commits: 1,
321 reset_subscriptions: 1,
322 }
323 );
324 let visible_ids = client
325 .conn
326 .prepare("SELECT id FROM tasks ORDER BY id")
327 .expect("prepare visible ids")
328 .query_map([], |row| row.get::<_, String>(0))
329 .expect("query visible ids")
330 .collect::<Result<Vec<_>, _>>()
331 .expect("collect visible ids");
332 assert_eq!(visible_ids, vec!["offline-row"]);
333 assert_eq!(client.pending_commit_ids(), vec![pending]);
334 assert_eq!(
335 client
336 .subscription_state("repair-tasks")
337 .expect("subscription")
338 .cursor,
339 -1
340 );
341 assert!(client.upgrading());
342 assert!(client.sync_needed());
343 assert!(matches!(
344 client.drain_sync_intents().as_slice(),
345 [SyncIntent::Interactive]
346 ));
347 assert_eq!(client.drain_change_batches().len(), 1);
348
349 assert_eq!(
350 client
351 .rebootstrap_local_data(&LocalDataRebootstrapInput {
352 rebootstrap_id: "support-case-001".to_owned(),
353 })
354 .expect("idempotent retry"),
355 LocalDataRebootstrapResult {
356 already_applied: true,
357 retained_commits: 1,
358 reset_subscriptions: 1,
359 }
360 );
361 assert!(client.drain_change_batches().is_empty());
362 }
363
364 #[test]
365 fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
366 let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
367 assert_eq!(
368 decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
369 (3, 4)
370 );
371 assert_eq!(
372 decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
373 (0, 0)
374 );
375 for malformed in [
376 "",
377 "{}",
378 r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
379 r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
380 r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
381 ] {
382 assert_eq!(
383 decode_local_rebootstrap_receipt(malformed)
384 .expect_err("malformed receipt must fail"),
385 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
386 );
387 }
388 }
389
390 #[test]
391 fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
392 let path = std::env::temp_dir().join(format!(
393 "syncular-rebootstrap-receipt-{}.db",
394 uuid::Uuid::new_v4()
395 ));
396 let schema = json!({
397 "version": 1,
398 "tables": [{
399 "name": "tasks",
400 "primaryKey": "id",
401 "columns": [
402 { "name": "id", "type": "string", "nullable": false },
403 { "name": "project_id", "type": "string", "nullable": false }
404 ],
405 "scopes": [{ "pattern": "project:{project_id}" }]
406 }]
407 });
408 let path_string = path.to_str().expect("UTF-8 temp path");
409
410 {
411 let mut first = SyncClient::open_path(
412 "repair-restart-client".to_owned(),
413 &schema,
414 ClientLimits::default(),
415 path_string,
416 )
417 .expect("first open");
418 first
419 .subscribe(
420 "repair-tasks".to_owned(),
421 "tasks".to_owned(),
422 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
423 None,
424 )
425 .expect("subscribe");
426 first
427 .mutate(vec![Mutation::Upsert {
428 table: "tasks".to_owned(),
429 values: Map::from_iter([
430 ("id".to_owned(), Value::from("offline-row")),
431 ("project_id".to_owned(), Value::from("p1")),
432 ]),
433 base_version: None,
434 }])
435 .expect("queue offline work");
436 assert_eq!(
437 first
438 .rebootstrap_local_data(&LocalDataRebootstrapInput {
439 rebootstrap_id: "restart-receipt".to_owned(),
440 })
441 .expect("first rebootstrap"),
442 LocalDataRebootstrapResult {
443 already_applied: false,
444 retained_commits: 1,
445 reset_subscriptions: 1,
446 }
447 );
448 }
449
450 let mut reopened = SyncClient::open_path(
451 "repair-restart-client".to_owned(),
452 &schema,
453 ClientLimits::default(),
454 path_string,
455 )
456 .expect("reopen");
457 reopened.drain_change_batches();
458 reopened.drain_sync_intents();
459 assert_eq!(
460 reopened
461 .rebootstrap_local_data(&LocalDataRebootstrapInput {
462 rebootstrap_id: "restart-receipt".to_owned(),
463 })
464 .expect("receipt replay"),
465 LocalDataRebootstrapResult {
466 already_applied: true,
467 retained_commits: 1,
468 reset_subscriptions: 1,
469 }
470 );
471 assert!(reopened.drain_change_batches().is_empty());
472 assert!(reopened.drain_sync_intents().is_empty());
473 drop(reopened);
474 std::fs::remove_file(path).expect("remove temp database");
475 }
476
477 #[test]
478 fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
479 let mut malformed = client();
480 malformed
481 .subscribe(
482 "repair-tasks".to_owned(),
483 "tasks".to_owned(),
484 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
485 None,
486 )
487 .expect("subscribe");
488 malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
489 malformed.drain_change_batches();
490 malformed.drain_sync_intents();
491 let malformed_error = malformed
492 .rebootstrap_local_data(&LocalDataRebootstrapInput {
493 rebootstrap_id: "malformed".to_owned(),
494 })
495 .expect_err("malformed receipt must fail closed");
496 assert_eq!(
497 malformed_error,
498 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
499 );
500 assert!(!malformed.upgrading());
501 assert_eq!(
502 malformed
503 .subscription_state("repair-tasks")
504 .expect("unchanged subscription")
505 .cursor,
506 -1
507 );
508 assert!(malformed.drain_change_batches().is_empty());
509 assert!(malformed.drain_sync_intents().is_empty());
510
511 let mut unreadable = client();
512 unreadable
513 .conn
514 .execute(
515 "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
516 rusqlite::params!["server-row", "p1"],
517 )
518 .expect("seed visible row");
519 unreadable
520 .conn
521 .execute("DROP TABLE _syncular_meta", [])
522 .expect("break marker storage");
523 let unreadable_error = unreadable
524 .rebootstrap_local_data(&LocalDataRebootstrapInput {
525 rebootstrap_id: "unreadable".to_owned(),
526 })
527 .expect_err("unreadable marker storage must fail closed");
528 assert_eq!(
529 unreadable_error,
530 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
531 );
532 let visible_rows = unreadable
533 .conn
534 .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
535 .expect("visible projection remains");
536 assert_eq!(visible_rows, 1);
537 assert!(!unreadable.upgrading());
538 assert!(unreadable.drain_change_batches().is_empty());
539 assert!(unreadable.drain_sync_intents().is_empty());
540 }
541
542 #[test]
543 fn local_rebootstrap_cannot_bypass_schema_floor() {
544 let mut client = client();
545 client.set_schema_floor(Some(SchemaFloor {
546 required_schema_version: Some(2),
547 latest_schema_version: Some(2),
548 }));
549 let error = client
550 .rebootstrap_local_data(&LocalDataRebootstrapInput {
551 rebootstrap_id: "blocked-floor".to_owned(),
552 })
553 .expect_err("schema floor must block repair");
554 assert!(error.contains("cannot bypass an active schema-floor stop"));
555 }
556
557 #[test]
558 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
559 let mut client = client();
560 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
561 const COMMIT_COUNT: usize = 32;
562
563 for index in 0..COMMIT_COUNT {
564 client
565 .mutate(vec![Mutation::Upsert {
566 table: "tasks".to_owned(),
567 values: Map::from_iter([
568 ("id".to_owned(), Value::from(format!("task-{index}"))),
569 ("project_id".to_owned(), Value::from("project-1")),
570 ]),
571 base_version: None,
572 }])
573 .expect("queue commit");
574 }
575
576 let (_, request_meta) = client.build_request(false);
577 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
578 let mut frames = vec![Frame::RespHeader {
579 required_schema_version: None,
580 latest_schema_version: None,
581 log_epoch: Some("epoch-1".to_owned()),
582 reset_required: Some(false),
583 }];
584 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
585 |(index, client_commit_id)| Frame::PushResult {
586 client_commit_id: client_commit_id.clone(),
587 status: PushStatus::Applied,
588 commit_seq: Some(index as i64 + 1),
589 results: vec![OpResult::Applied { op_index: 0 }],
590 },
591 ));
592 let response = Message {
593 wire_version: WIRE_VERSION,
594 msg_kind: MsgKind::Response,
595 frames,
596 };
597 let mut transport =
598 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
599
600 client.overlay_rebuild_count.set(0);
601 client.outcome_prune_count.set(0);
602 let outcome = client.process_response(&mut transport, response, &request_meta);
603 assert!(matches!(outcome, SyncOutcome::Ok(_)));
604 assert!(client.pending_commit_ids().is_empty());
605 assert_eq!(
606 client.overlay_rebuild_count.get(),
607 1,
608 "one response must reconcile its acknowledged commits with one overlay rebuild"
609 );
610 assert_eq!(
611 client.outcome_prune_count.get(),
612 1,
613 "one response must enforce outcome retention once"
614 );
615 }
616
617 #[test]
618 fn mixed_push_results_reconcile_and_prune_once_per_response() {
619 let mut client = client();
620 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
621 for index in 0..4 {
622 client
623 .mutate(vec![Mutation::Upsert {
624 table: "tasks".to_owned(),
625 values: Map::from_iter([
626 ("id".to_owned(), Value::from(format!("task-{index}"))),
627 ("project_id".to_owned(), Value::from("project-1")),
628 ]),
629 base_version: None,
630 }])
631 .expect("queue commit");
632 }
633
634 let (_, request_meta) = client.build_request(false);
635 let ids = &request_meta.pushed_ids;
636 assert_eq!(ids.len(), 4);
637 let response = Message {
638 wire_version: WIRE_VERSION,
639 msg_kind: MsgKind::Response,
640 frames: vec![
641 Frame::RespHeader {
642 required_schema_version: None,
643 latest_schema_version: None,
644 log_epoch: Some("epoch-1".to_owned()),
645 reset_required: Some(false),
646 },
647 Frame::PushResult {
648 client_commit_id: ids[0].clone(),
649 status: PushStatus::Applied,
650 commit_seq: Some(1),
651 results: vec![OpResult::Applied { op_index: 0 }],
652 },
653 Frame::PushResult {
654 client_commit_id: ids[1].clone(),
655 status: PushStatus::Cached,
656 commit_seq: Some(2),
657 results: vec![OpResult::Applied { op_index: 0 }],
658 },
659 Frame::PushResult {
660 client_commit_id: ids[2].clone(),
661 status: PushStatus::Rejected,
662 commit_seq: None,
663 results: vec![OpResult::Error {
664 op_index: 0,
665 code: "sync.validation_failed".to_owned(),
666 message: "rejected".to_owned(),
667 retryable: false,
668 }],
669 },
670 Frame::PushResult {
671 client_commit_id: ids[3].clone(),
672 status: PushStatus::Rejected,
673 commit_seq: None,
674 results: vec![OpResult::Error {
675 op_index: 0,
676 code: "sync.idempotency_cache_miss".to_owned(),
677 message: "retry".to_owned(),
678 retryable: true,
679 }],
680 },
681 ],
682 };
683 let mut transport =
684 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
685
686 client.overlay_rebuild_count.set(0);
687 client.outcome_prune_count.set(0);
688 let outcome = client.process_response(&mut transport, response, &request_meta);
689 let SyncOutcome::Ok(report) = outcome else {
690 panic!("mixed push-result response failed");
691 };
692
693 assert_eq!(report.applied, ids[..2]);
694 assert_eq!(report.rejected, ids[2..3]);
695 assert_eq!(report.retryable, ids[3..4]);
696 assert_eq!(client.pending_commit_ids(), ids[3..4]);
697 assert_eq!(
698 client
699 .query("SELECT id FROM tasks ORDER BY id", &[])
700 .expect("query visible overlay"),
701 vec![Map::from_iter([("id".to_owned(), Value::from("task-3"))])]
702 );
703 assert_eq!(client.overlay_rebuild_count.get(), 1);
704 assert_eq!(client.outcome_prune_count.get(), 1);
705 }
706
707 #[test]
708 fn secondary_unique_collision_preserves_existing_synced_row() {
709 let client = SyncClient::new(
710 "unique-upsert-test".to_owned(),
711 &json!({
712 "version": 1,
713 "tables": [{
714 "name": "tasks",
715 "primaryKey": "id",
716 "columns": [
717 { "name": "id", "type": "string", "nullable": false },
718 { "name": "project_id", "type": "string", "nullable": false },
719 { "name": "title", "type": "string", "nullable": false }
720 ],
721 "scopes": [{ "pattern": "project:{project_id}" }],
722 "indexes": [{
723 "name": "tasks_by_project_title",
724 "columns": ["project_id", "title"],
725 "unique": true
726 }]
727 }]
728 }),
729 ClientLimits::default(),
730 )
731 .expect("test client");
732 let table = client.schema.table("tasks").expect("tasks table");
733 let sql = client.insert_row_sql(&base_table("tasks"), table);
734
735 client
736 .conn
737 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
738 .expect("insert first row");
739 client
740 .conn
741 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
742 .expect("update same primary key");
743 client
744 .conn
745 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
746 .expect("insert second row");
747 assert!(client
748 .conn
749 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
750 .is_err());
751
752 let rows = client
753 .conn
754 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
755 .expect("prepare rows")
756 .query_map([], |row| {
757 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
758 })
759 .expect("query rows")
760 .collect::<Result<Vec<_>, _>>()
761 .expect("collect rows");
762 assert_eq!(
763 rows,
764 vec![
765 ("t1".to_owned(), "updated".to_owned()),
766 ("t2".to_owned(), "original".to_owned())
767 ]
768 );
769 }
770
771 #[test]
772 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
773 let path = std::env::temp_dir().join(format!(
774 "syncular-startup-intent-{}.db",
775 uuid::Uuid::new_v4()
776 ));
777 let schema = json!({
778 "version": 1,
779 "tables": [{
780 "name": "tasks",
781 "primaryKey": "id",
782 "columns": [
783 { "name": "id", "type": "string", "nullable": false },
784 { "name": "project_id", "type": "string", "nullable": false }
785 ],
786 "scopes": [{ "pattern": "project:{project_id}" }]
787 }]
788 });
789
790 {
791 let mut first = SyncClient::open_path_with_identity(
792 None,
793 &schema,
794 ClientLimits::default(),
795 path.to_str().expect("UTF-8 temp path"),
796 )
797 .expect("first open");
798 first
799 .set_window(
800 &WindowBase {
801 table: "tasks".to_owned(),
802 variable: "project_id".to_owned(),
803 fixed_scopes: Vec::new(),
804 params: None,
805 },
806 &["persisted".to_owned()],
807 )
808 .expect("persist window");
809 }
810
811 let mut reopened = SyncClient::open_path_with_identity(
812 None,
813 &schema,
814 ClientLimits::default(),
815 path.to_str().expect("UTF-8 temp path"),
816 )
817 .expect("reopen");
818 assert!(reopened.sync_needed());
819 assert!(matches!(
820 reopened.drain_sync_intents().as_slice(),
821 [SyncIntent::Interactive]
822 ));
823 drop(reopened);
824 std::fs::remove_file(path).expect("remove temp database");
825 }
826
827 #[test]
828 fn reopening_preserves_immutable_subscription_identity_and_progress() {
829 let path = std::env::temp_dir().join(format!(
830 "syncular-subscription-identity-{}.db",
831 uuid::Uuid::new_v4()
832 ));
833 let schema = json!({
834 "version": 1,
835 "tables": [
836 {
837 "name": "tasks",
838 "primaryKey": "id",
839 "columns": [
840 { "name": "id", "type": "string", "nullable": false },
841 { "name": "project_id", "type": "string", "nullable": false }
842 ],
843 "scopes": [{ "pattern": "project:{project_id}" }]
844 },
845 {
846 "name": "docs",
847 "primaryKey": "id",
848 "columns": [
849 { "name": "id", "type": "string", "nullable": false },
850 { "name": "org_id", "type": "string", "nullable": false },
851 { "name": "project_id", "type": "string", "nullable": false }
852 ],
853 "scopes": [
854 { "pattern": "org:{org_id}" },
855 { "pattern": "project:{projectId}", "column": "project_id" }
856 ]
857 }
858 ]
859 });
860
861 {
862 let mut first = SyncClient::open_path_with_identity(
863 None,
864 &schema,
865 ClientLimits::default(),
866 path.to_str().expect("UTF-8 temp path"),
867 )
868 .expect("first open");
869 first
870 .subscribe(
871 "stable-subscription".to_owned(),
872 "tasks".to_owned(),
873 vec![(
874 "project_id".to_owned(),
875 vec!["p2".to_owned(), "p1".to_owned()],
876 )],
877 Some(r#"{"view":"v1"}"#.to_owned()),
878 )
879 .expect("persist subscription");
880 let persisted = {
881 let subscription = first
882 .subs
883 .iter_mut()
884 .find(|subscription| subscription.id == "stable-subscription")
885 .expect("subscription");
886 subscription.cursor = 41;
887 subscription.bootstrap_state = Some("resume-token".to_owned());
888 subscription.effective = Some(vec![(
889 "project_id".to_owned(),
890 vec!["p1".to_owned(), "p2".to_owned()],
891 )]);
892 subscription.synced_once = true;
893 subscription.clone()
894 };
895 first.persist_sub(&persisted);
896 }
897
898 let mut reopened = SyncClient::open_path_with_identity(
899 None,
900 &schema,
901 ClientLimits::default(),
902 path.to_str().expect("UTF-8 temp path"),
903 )
904 .expect("reopen");
905 let progress = reopened
906 .subscription_state("stable-subscription")
907 .expect("persisted state");
908 assert_eq!(progress.cursor, 41);
909 assert!(progress.has_resume_token);
910
911 reopened
912 .subscribe(
913 "stable-subscription".to_owned(),
914 "tasks".to_owned(),
915 vec![(
916 "project_id".to_owned(),
917 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
918 )],
919 Some(r#"{"view":"v1"}"#.to_owned()),
920 )
921 .expect("canonical intent is idempotent");
922 assert_eq!(
923 reopened
924 .subscription_state("stable-subscription")
925 .expect("unchanged state")
926 .cursor,
927 progress.cursor
928 );
929
930 for (table, scopes, params) in [
931 (
932 "tasks",
933 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
934 Some(r#"{"view":"v1"}"#.to_owned()),
935 ),
936 (
937 "tasks",
938 vec![(
939 "project_id".to_owned(),
940 vec!["p2".to_owned(), "p1".to_owned()],
941 )],
942 Some(r#"{"view":"v2"}"#.to_owned()),
943 ),
944 (
945 "docs",
946 vec![
947 ("org_id".to_owned(), vec!["o1".to_owned()]),
948 ("projectId".to_owned(), vec!["p1".to_owned()]),
949 ],
950 Some(r#"{"view":"v1"}"#.to_owned()),
951 ),
952 ] {
953 let error = reopened
954 .subscribe(
955 "stable-subscription".to_owned(),
956 table.to_owned(),
957 scopes,
958 params,
959 )
960 .expect_err("identity rebind must fail");
961 assert!(error.starts_with("client.subscription_intent_mismatch:"));
962 assert_eq!(
963 reopened
964 .subscription_state("stable-subscription")
965 .expect("unchanged state")
966 .cursor,
967 progress.cursor
968 );
969 }
970
971 drop(reopened);
972 std::fs::remove_file(path).expect("remove temp database");
973 }
974
975 #[test]
976 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
977 let path = std::env::temp_dir().join(format!(
978 "syncular-satisfied-schema-floor-{}.db",
979 uuid::Uuid::new_v4()
980 ));
981 let schema = json!({
982 "version": 23,
983 "tables": [{
984 "name": "tasks",
985 "primaryKey": "id",
986 "columns": [
987 { "name": "id", "type": "string", "nullable": false },
988 { "name": "project_id", "type": "string", "nullable": false }
989 ],
990 "scopes": [{ "pattern": "project:{project_id}" }]
991 }]
992 });
993
994 {
995 let mut first = SyncClient::open_path_with_identity(
996 None,
997 &schema,
998 ClientLimits::default(),
999 path.to_str().expect("UTF-8 temp path"),
1000 )
1001 .expect("first open");
1002 first
1003 .subscribe(
1004 "tasks".to_owned(),
1005 "tasks".to_owned(),
1006 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1007 None,
1008 )
1009 .expect("persist subscription");
1010 first.set_schema_floor(Some(SchemaFloor {
1011 required_schema_version: Some(22),
1012 latest_schema_version: Some(22),
1013 }));
1014 }
1015
1016 let mut reopened = SyncClient::open_path_with_identity(
1017 None,
1018 &schema,
1019 ClientLimits::default(),
1020 path.to_str().expect("UTF-8 temp path"),
1021 )
1022 .expect("reopen");
1023 assert!(reopened.schema_floor().is_none());
1024 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
1025 assert!(reopened.sync_needed());
1026 assert!(matches!(
1027 reopened.drain_sync_intents().as_slice(),
1028 [SyncIntent::Interactive]
1029 ));
1030 drop(reopened);
1031 std::fs::remove_file(path).expect("remove temp database");
1032 }
1033
1034 #[test]
1035 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
1036 let path = std::env::temp_dir().join(format!(
1037 "syncular-unsatisfied-schema-floor-{}.db",
1038 uuid::Uuid::new_v4()
1039 ));
1040 let schema = json!({
1041 "version": 1,
1042 "tables": [{
1043 "name": "tasks",
1044 "primaryKey": "id",
1045 "columns": [
1046 { "name": "id", "type": "string", "nullable": false },
1047 { "name": "project_id", "type": "string", "nullable": false }
1048 ],
1049 "scopes": [{ "pattern": "project:{project_id}" }]
1050 }]
1051 });
1052
1053 {
1054 let mut first = SyncClient::open_path_with_identity(
1055 None,
1056 &schema,
1057 ClientLimits::default(),
1058 path.to_str().expect("UTF-8 temp path"),
1059 )
1060 .expect("first open");
1061 first
1062 .subscribe(
1063 "tasks".to_owned(),
1064 "tasks".to_owned(),
1065 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1066 None,
1067 )
1068 .expect("persist subscription");
1069 first.set_schema_floor(Some(SchemaFloor {
1070 required_schema_version: Some(2),
1071 latest_schema_version: Some(2),
1072 }));
1073 }
1074
1075 let reopened = SyncClient::open_path_with_identity(
1076 None,
1077 &schema,
1078 ClientLimits::default(),
1079 path.to_str().expect("UTF-8 temp path"),
1080 )
1081 .expect("reopen");
1082 assert_eq!(
1083 reopened.schema_floor(),
1084 Some(&SchemaFloor {
1085 required_schema_version: Some(2),
1086 latest_schema_version: Some(2),
1087 })
1088 );
1089 assert!(!reopened.sync_needed());
1090 drop(reopened);
1091 std::fs::remove_file(path).expect("remove temp database");
1092 }
1093
1094 #[test]
1095 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
1096 let path = std::env::temp_dir().join(format!(
1097 "syncular-indexed-column-bump-{}.db",
1098 uuid::Uuid::new_v4()
1099 ));
1100 let old_schema = json!({
1101 "version": 1,
1102 "tables": [
1103 {
1104 "name": "tasks",
1105 "primaryKey": "id",
1106 "columns": [
1107 { "name": "id", "type": "string", "nullable": false },
1108 { "name": "project_id", "type": "string", "nullable": false }
1109 ],
1110 "scopes": [{ "pattern": "project:{project_id}" }]
1111 },
1112 {
1113 "name": "legacy",
1114 "primaryKey": "id",
1115 "columns": [
1116 { "name": "id", "type": "string", "nullable": false },
1117 { "name": "project_id", "type": "string", "nullable": false }
1118 ],
1119 "scopes": [{ "pattern": "project:{project_id}" }]
1120 }
1121 ]
1122 });
1123 {
1124 let mut first = SyncClient::open_path_with_identity(
1125 None,
1126 &old_schema,
1127 ClientLimits::default(),
1128 path.to_str().expect("UTF-8 temp path"),
1129 )
1130 .expect("open old schema");
1131 first
1132 .subscribe(
1133 "legacy-sub".to_owned(),
1134 "legacy".to_owned(),
1135 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1136 None,
1137 )
1138 .expect("persist legacy subscription");
1139 }
1140
1141 let new_schema = json!({
1142 "version": 2,
1143 "tables": [{
1144 "name": "tasks",
1145 "primaryKey": "id",
1146 "columns": [
1147 { "name": "id", "type": "string", "nullable": false },
1148 { "name": "project_id", "type": "string", "nullable": false },
1149 { "name": "facility_membership_id", "type": "string", "nullable": true }
1150 ],
1151 "scopes": [{ "pattern": "project:{project_id}" }],
1152 "indexes": [{
1153 "name": "tasks_by_membership",
1154 "columns": ["project_id", "facility_membership_id"],
1155 "unique": false
1156 }]
1157 }]
1158 });
1159 let upgraded = SyncClient::open_path_with_identity(
1160 None,
1161 &new_schema,
1162 ClientLimits::default(),
1163 path.to_str().expect("UTF-8 temp path"),
1164 )
1165 .expect("open upgraded schema");
1166 let columns = upgraded
1167 .conn
1168 .prepare("PRAGMA table_info(tasks)")
1169 .expect("prepare columns")
1170 .query_map([], |row| row.get::<_, String>(1))
1171 .expect("query columns")
1172 .collect::<Result<Vec<_>, _>>()
1173 .expect("collect columns");
1174 assert!(columns.contains(&"facility_membership_id".to_owned()));
1175 let index_count: i64 = upgraded
1176 .conn
1177 .query_row(
1178 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
1179 [],
1180 |row| row.get(0),
1181 )
1182 .expect("query index");
1183 assert_eq!(index_count, 1);
1184 assert!(upgraded.subscription_state("legacy-sub").is_none());
1185 drop(upgraded);
1186 std::fs::remove_file(path).expect("remove temp database");
1187 }
1188
1189 #[test]
1190 fn migrates_pre_envelope_outcome_journal_additively() {
1191 let path = std::env::temp_dir().join(format!(
1192 "syncular-outcome-migration-{}.db",
1193 uuid::Uuid::new_v4()
1194 ));
1195 let conn = Connection::open(&path).expect("open legacy database");
1196 conn.execute_batch(
1197 "CREATE TABLE _syncular_commit_outcomes (
1198 seq INTEGER PRIMARY KEY AUTOINCREMENT,
1199 client_commit_id TEXT NOT NULL UNIQUE,
1200 status TEXT NOT NULL,
1201 recorded_at_ms INTEGER NOT NULL,
1202 results_json TEXT NOT NULL,
1203 resolution TEXT NOT NULL DEFAULT 'active',
1204 resolved_at_ms INTEGER,
1205 replacement_client_commit_id TEXT);",
1206 )
1207 .expect("create legacy outcome journal");
1208 drop(conn);
1209 let schema = json!({
1210 "version": 1,
1211 "tables": [{
1212 "name": "tasks",
1213 "primaryKey": "id",
1214 "columns": [
1215 { "name": "id", "type": "string", "nullable": false },
1216 { "name": "project_id", "type": "string", "nullable": false }
1217 ],
1218 "scopes": [{ "pattern": "project:{project_id}" }]
1219 }]
1220 });
1221 let client = SyncClient::open_path(
1222 "migration-native".to_owned(),
1223 &schema,
1224 ClientLimits::default(),
1225 path.to_str().expect("UTF-8 temp path"),
1226 )
1227 .expect("migrate database");
1228 let has_operations = client
1229 .conn
1230 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
1231 .expect("prepare table info")
1232 .query_map([], |row| row.get::<_, String>(1))
1233 .expect("query table info")
1234 .filter_map(Result::ok)
1235 .any(|column| column == "operations_json");
1236 assert!(has_operations);
1237 drop(client);
1238 std::fs::remove_file(path).expect("remove temp database");
1239 }
1240
1241 #[test]
1242 fn durable_conflict_outcome_and_resolution_survive_reopen() {
1243 let path = std::env::temp_dir().join(format!(
1244 "syncular-durable-outcome-{}.db",
1245 uuid::Uuid::new_v4()
1246 ));
1247 let schema = json!({
1248 "version": 1,
1249 "tables": [{
1250 "name": "tasks",
1251 "primaryKey": "id",
1252 "columns": [
1253 { "name": "id", "type": "string", "nullable": false },
1254 { "name": "project_id", "type": "string", "nullable": false }
1255 ],
1256 "scopes": [{ "pattern": "project:{project_id}" }]
1257 }]
1258 });
1259 let conflict = ConflictRecord {
1260 client_commit_id: "losing-commit".to_owned(),
1261 op_index: 0,
1262 table: "tasks".to_owned(),
1263 row_id: "t1".to_owned(),
1264 code: "sync.version_conflict".to_owned(),
1265 message: "stale base version".to_owned(),
1266 server_version: 2,
1267 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
1268 operation: Some(CommitOperation {
1269 table: "tasks".to_owned(),
1270 row_id: "t1".to_owned(),
1271 op: "upsert".to_owned(),
1272 base_version: Some(1),
1273 values: None,
1274 changed_fields: None,
1275 }),
1276 };
1277 let failed_operations = vec![
1278 OutboxOp {
1279 upsert: true,
1280 table: "tasks".to_owned(),
1281 row_id: "t1".to_owned(),
1282 base_version: Some(1),
1283 values: None,
1284 changed_fields: None,
1285 },
1286 OutboxOp {
1287 upsert: true,
1288 table: "tasks".to_owned(),
1289 row_id: "status-event-1".to_owned(),
1290 base_version: Some(0),
1291 values: None,
1292 changed_fields: None,
1293 },
1294 ];
1295
1296 {
1297 let mut first = SyncClient::open_path(
1298 "durable-native".to_owned(),
1299 &schema,
1300 ClientLimits::default(),
1301 path.to_str().expect("UTF-8 temp path"),
1302 )
1303 .expect("first open");
1304 first
1305 .begin_observation("test_outcome")
1306 .expect("begin outcome");
1307 first
1308 .persist_commit_outcome(
1309 "losing-commit",
1310 CommitOutcomeStatus::Conflict,
1311 &[CommitOperationOutcome::Conflict {
1312 conflict: conflict.clone(),
1313 }],
1314 Some(&failed_operations),
1315 )
1316 .expect("persist outcome");
1317 first.conflicts.push(conflict);
1318 first
1319 .finish_observation(
1320 "test_outcome",
1321 ChangeAccumulator {
1322 conflicts: true,
1323 outcomes: true,
1324 ..ChangeAccumulator::default()
1325 },
1326 )
1327 .expect("commit outcome");
1328 }
1329
1330 {
1331 let mut reopened = SyncClient::open_path(
1332 "durable-native".to_owned(),
1333 &schema,
1334 ClientLimits::default(),
1335 path.to_str().expect("UTF-8 temp path"),
1336 )
1337 .expect("reopen");
1338 assert_eq!(reopened.conflicts().len(), 1);
1339 let outcome = reopened
1340 .commit_outcome("losing-commit")
1341 .expect("read outcome")
1342 .expect("outcome");
1343 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
1344 let operations = outcome.operations.expect("aggregate envelope");
1345 assert_eq!(operations.len(), 2);
1346 assert_eq!(operations[1].row_id, "status-event-1");
1347 let resolved = reopened
1348 .resolve_commit_outcome(ResolveCommitOutcomeInput {
1349 client_commit_id: "losing-commit".to_owned(),
1350 resolution: CommitOutcomeResolution::ResolvedKeepServer,
1351 replacement_client_commit_id: None,
1352 })
1353 .expect("resolve");
1354 assert_eq!(
1355 resolved.resolution,
1356 CommitOutcomeResolution::ResolvedKeepServer
1357 );
1358 assert!(reopened.conflicts().is_empty());
1359 }
1360
1361 let reopened = SyncClient::open_path(
1362 "durable-native".to_owned(),
1363 &schema,
1364 ClientLimits::default(),
1365 path.to_str().expect("UTF-8 temp path"),
1366 )
1367 .expect("second reopen");
1368 assert!(reopened.conflicts().is_empty());
1369 assert_eq!(
1370 reopened
1371 .commit_outcome("losing-commit")
1372 .expect("read outcome")
1373 .expect("outcome")
1374 .resolution,
1375 CommitOutcomeResolution::ResolvedKeepServer
1376 );
1377 drop(reopened);
1378 std::fs::remove_file(path).expect("remove temp database");
1379 }
1380
1381 #[test]
1382 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
1383 let path =
1384 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
1385 let schema = json!({
1386 "version": 1,
1387 "tables": [{
1388 "name": "tasks",
1389 "primaryKey": "id",
1390 "columns": [
1391 { "name": "id", "type": "string", "nullable": false },
1392 { "name": "project_id", "type": "string", "nullable": false }
1393 ],
1394 "scopes": [{ "pattern": "project:{project_id}" }]
1395 }]
1396 });
1397 let mut client = SyncClient::open_path_with_identity(
1398 Some("sidecar-client".to_owned()),
1399 &schema,
1400 ClientLimits::default(),
1401 path.to_str().expect("UTF-8 temp path"),
1402 )
1403 .expect("open owner");
1404 let base = WindowBase {
1405 table: "tasks".to_owned(),
1406 variable: "project_id".to_owned(),
1407 fixed_scopes: Vec::new(),
1408 params: None,
1409 };
1410 client
1411 .set_window(&base, &["one".to_owned()])
1412 .expect("set window");
1413 client
1414 .mutate(vec![Mutation::Upsert {
1415 table: "tasks".to_owned(),
1416 values: Map::from_iter([
1417 ("id".to_owned(), Value::from("t1")),
1418 ("project_id".to_owned(), Value::from("one")),
1419 ]),
1420 base_version: None,
1421 }])
1422 .expect("local mutate");
1423
1424 let coverage = [WindowCoverage {
1425 base,
1426 units: vec!["one".to_owned(), "missing".to_owned()],
1427 }];
1428 let owner = client
1429 .query_snapshot(
1430 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
1431 &[],
1432 &coverage,
1433 )
1434 .expect("owner snapshot");
1435 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
1436 let sidecar = reader
1437 .query_snapshot(
1438 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
1439 &[],
1440 &coverage,
1441 )
1442 .expect("sidecar snapshot");
1443
1444 assert_eq!(sidecar.revision, owner.revision);
1445 assert_eq!(sidecar.rows, owner.rows);
1446 assert_eq!(
1447 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
1448 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
1449 );
1450 assert_eq!(sidecar.revision, "2");
1451 assert_eq!(sidecar.rows[0]["id"], "t1");
1452 assert_eq!(sidecar.rows[0]["server_version"], -1);
1453 assert!(!sidecar.coverage.complete);
1454 assert_eq!(sidecar.coverage.pending.len(), 1);
1455 assert_eq!(sidecar.coverage.missing.len(), 1);
1456
1457 drop(reader);
1458 drop(client);
1459 std::fs::remove_file(path).expect("remove temp database");
1460 }
1461
1462 #[test]
1463 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
1464 let schema = json!({
1465 "version": 1,
1466 "tables": [{
1467 "name": "catalogue_codes",
1468 "primaryKey": "id",
1469 "columns": [
1470 { "name": "id", "type": "string", "nullable": false },
1471 { "name": "release_id", "type": "string", "nullable": false },
1472 { "name": "code", "type": "string", "nullable": false },
1473 { "name": "title", "type": "string", "nullable": false }
1474 ],
1475 "scopes": [{ "pattern": "release:{release_id}" }],
1476 "ftsIndexes": [{
1477 "name": "catalogue_codes_fts",
1478 "columns": ["code", "title"],
1479 "tokenize": "unicode61 remove_diacritics 2"
1480 }]
1481 }]
1482 });
1483 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
1484 .expect("FTS5 client");
1485 let insert_trigger: String = client
1486 .conn
1487 .query_row(
1488 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
1489 [],
1490 |row| row.get(0),
1491 )
1492 .expect("insert trigger");
1493 let replace_guard: String = client
1494 .conn
1495 .query_row(
1496 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
1497 [],
1498 |row| row.get(0),
1499 )
1500 .expect("replace guard");
1501 assert!(!insert_trigger.contains("DELETE FROM"));
1502 assert!(replace_guard.contains("BEFORE INSERT"));
1503 assert!(replace_guard.contains("WHEN EXISTS"));
1504 let search = |client: &SyncClient, query: &str| {
1505 client
1506 .query(
1507 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
1508 &[Value::from(query)],
1509 )
1510 .expect("FTS query")
1511 };
1512
1513 client
1514 .mutate(vec![Mutation::Upsert {
1515 table: "catalogue_codes".to_owned(),
1516 values: Map::from_iter([
1517 ("id".to_owned(), Value::from("c1")),
1518 ("release_id".to_owned(), Value::from("r1")),
1519 ("code".to_owned(), Value::from("A01")),
1520 ("title".to_owned(), Value::from("Cholera")),
1521 ]),
1522 base_version: None,
1523 }])
1524 .expect("insert code");
1525 assert_eq!(search(&client, "cholera").len(), 1);
1526
1527 client
1528 .mutate(vec![Mutation::Upsert {
1529 table: "catalogue_codes".to_owned(),
1530 values: Map::from_iter([
1531 ("id".to_owned(), Value::from("c1")),
1532 ("release_id".to_owned(), Value::from("r1")),
1533 ("code".to_owned(), Value::from("A01")),
1534 ("title".to_owned(), Value::from("Enteric infection")),
1535 ]),
1536 base_version: None,
1537 }])
1538 .expect("update code");
1539 assert!(search(&client, "cholera").is_empty());
1540 assert_eq!(search(&client, "enteric").len(), 1);
1541
1542 client
1543 .mutate(vec![Mutation::Delete {
1544 table: "catalogue_codes".to_owned(),
1545 row_id: "c1".to_owned(),
1546 base_version: None,
1547 }])
1548 .expect("delete code");
1549 assert!(search(&client, "enteric").is_empty());
1550 }
1551
1552 #[test]
1553 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
1554 let schema = json!({
1555 "version": 1,
1556 "tables": [{
1557 "name": "patient_notes",
1558 "primaryKey": "id",
1559 "columns": [
1560 { "name": "id", "type": "string", "nullable": false },
1561 { "name": "practice_id", "type": "string", "nullable": false },
1562 { "name": "encryption_key_id", "type": "string", "nullable": false },
1563 { "name": "title", "type": "string", "nullable": false }
1564 ],
1565 "scopes": [{ "pattern": "practice:{practice_id}" }],
1566 "ftsIndexes": [{
1567 "name": "patient_notes_fts",
1568 "columns": ["title"],
1569 "tokenize": "unicode61 remove_diacritics 2"
1570 }]
1571 }]
1572 });
1573 let mut client = SyncClient::new(
1574 "local-purge-test".to_owned(),
1575 &schema,
1576 ClientLimits::default(),
1577 )
1578 .expect("local purge client");
1579 client
1580 .conn
1581 .execute(
1582 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
1583 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
1584 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
1585 [],
1586 )
1587 .expect("seed base rows");
1588 client.overlay_dirty.set(true);
1589 client.rebuild_overlay_if_dirty();
1590
1591 let note = |id: &str, key_id: &str, title: &str| {
1592 Map::from_iter([
1593 ("id".to_owned(), Value::from(id)),
1594 ("practice_id".to_owned(), Value::from("practice-1")),
1595 ("encryption_key_id".to_owned(), Value::from(key_id)),
1596 ("title".to_owned(), Value::from(title)),
1597 ])
1598 };
1599 let doomed = client
1600 .mutate(vec![
1601 Mutation::Upsert {
1602 table: "patient_notes".to_owned(),
1603 values: note("target", "key-revoked", "Target changed"),
1604 base_version: None,
1605 },
1606 Mutation::Upsert {
1607 table: "patient_notes".to_owned(),
1608 values: note("unrelated", "key-held", "Unrelated changed"),
1609 base_version: None,
1610 },
1611 ])
1612 .expect("doomed commit");
1613 let kept = client
1614 .mutate(vec![Mutation::Upsert {
1615 table: "patient_notes".to_owned(),
1616 values: note("kept", "key-held", "Kept optimistic"),
1617 base_version: None,
1618 }])
1619 .expect("kept commit");
1620 client.drain_change_batches();
1621
1622 let input = LocalDataPurgeInput {
1623 purge_id: "purge-001".to_owned(),
1624 targets: vec![LocalDataPurgeTarget {
1625 table: "patient_notes".to_owned(),
1626 selectors: BTreeMap::from([(
1627 "encryption_key_id".to_owned(),
1628 vec!["key-revoked".to_owned()],
1629 )]),
1630 }],
1631 };
1632 assert_eq!(
1633 client.purge_local_data(&input).expect("apply purge"),
1634 LocalDataPurgeResult {
1635 already_applied: false,
1636 purged_rows: 1,
1637 dropped_commits: 1,
1638 }
1639 );
1640 let rows = client
1641 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
1642 .expect("visible rows");
1643 assert_eq!(rows.len(), 2);
1644 assert_eq!(rows[0]["id"], "kept");
1645 assert_eq!(rows[1]["id"], "unrelated");
1646 assert_eq!(rows[1]["title"], "Unrelated original");
1647 let fts = client
1648 .query(
1649 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
1650 &[],
1651 )
1652 .expect("fts query");
1653 assert!(fts.is_empty());
1654 assert_eq!(client.outbox.len(), 1);
1655 assert_eq!(client.outbox[0].client_commit_id, kept);
1656 let outcome = client
1657 .commit_outcome(&doomed)
1658 .expect("read doomed outcome")
1659 .expect("doomed outcome");
1660 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
1661 match &outcome.results[0] {
1662 CommitOperationOutcome::Error { rejection } => {
1663 assert_eq!(rejection.code, "client.local_data_purged");
1664 assert_eq!(rejection.client_commit_id, doomed);
1665 }
1666 other => panic!("expected local purge rejection, got {other:?}"),
1667 }
1668 assert_eq!(client.drain_change_batches().len(), 1);
1669 assert_eq!(
1670 client.purge_local_data(&input).expect("retry purge"),
1671 LocalDataPurgeResult {
1672 already_applied: true,
1673 purged_rows: 0,
1674 dropped_commits: 0,
1675 }
1676 );
1677 let conflicting = LocalDataPurgeInput {
1678 purge_id: input.purge_id.clone(),
1679 targets: vec![LocalDataPurgeTarget {
1680 table: "patient_notes".to_owned(),
1681 selectors: BTreeMap::from([(
1682 "encryption_key_id".to_owned(),
1683 vec!["key-held".to_owned()],
1684 )]),
1685 }],
1686 };
1687 assert!(client
1688 .purge_local_data(&conflicting)
1689 .expect_err("id collision must fail")
1690 .contains("already used with a different plan"));
1691 }
1692}
1693
1694impl SubState {
1695 fn name(self) -> &'static str {
1696 match self {
1697 SubState::Active => "active",
1698 SubState::Revoked => "revoked",
1699 SubState::Failed => "failed",
1700 }
1701 }
1702
1703 fn parse(value: &str) -> Self {
1704 match value {
1705 "revoked" => Self::Revoked,
1706 "failed" => Self::Failed,
1707 _ => Self::Active,
1708 }
1709 }
1710}
1711
1712#[derive(Debug, Clone)]
1713struct Subscription {
1714 id: String,
1715 table: String,
1716 requested: Vec<(String, Vec<String>)>,
1717 params: Option<String>,
1718 cursor: i64,
1719 bootstrap_state: Option<String>,
1721 state: SubState,
1722 reason_code: Option<String>,
1723 effective: Option<Vec<(String, Vec<String>)>>,
1726 synced_once: bool,
1727}
1728
1729#[derive(Debug, Clone)]
1730struct OutboxOp {
1731 upsert: bool,
1732 table: String,
1733 row_id: String,
1734 base_version: Option<i64>,
1735 values: Option<Map<String, Value>>,
1738 changed_fields: Option<Vec<String>>,
1740}
1741
1742impl From<&OutboxOp> for CommitOperation {
1743 fn from(operation: &OutboxOp) -> Self {
1744 Self {
1745 table: operation.table.clone(),
1746 row_id: operation.row_id.clone(),
1747 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
1748 base_version: operation.base_version,
1749 values: operation.values.clone(),
1750 changed_fields: operation.changed_fields.clone(),
1751 }
1752 }
1753}
1754
1755#[derive(Debug, Clone)]
1756struct OutboxCommit {
1757 client_commit_id: String,
1758 ops: Vec<OutboxOp>,
1759}
1760
1761#[derive(Debug, Clone)]
1762struct CompiledLocalDataPurgeTarget {
1763 table: String,
1764 selectors: Vec<(String, Vec<String>)>,
1765}
1766
1767struct StoredCommitOutcomeRow {
1768 sequence: i64,
1769 client_commit_id: String,
1770 status: String,
1771 recorded_at_ms: i64,
1772 results_json: String,
1773 operations_json: Option<String>,
1774 resolution: String,
1775 resolved_at_ms: Option<i64>,
1776 replacement_client_commit_id: Option<String>,
1777}
1778
1779enum SectionError {
1782 FailClosed,
1783 Abort(String, String),
1784}
1785
1786struct RequestMeta {
1787 pushed_ids: Vec<String>,
1788 fresh: Vec<(String, bool)>,
1791 accept: u8,
1792 deferred_commits: usize,
1796}
1797
1798#[derive(Default)]
1799struct ChangeAccumulator {
1800 tables: BTreeMap<String, Option<BTreeSet<String>>>,
1802 windows: BTreeMap<(String, String), BTreeSet<String>>,
1803 status: bool,
1804 conflicts: bool,
1805 rejections: bool,
1806 outcomes: bool,
1807}
1808
1809impl ChangeAccumulator {
1810 fn table(&mut self, table: &str) {
1811 self.tables.insert(table.to_owned(), None);
1812 }
1813
1814 fn scope(&mut self, table: &str, key: String) {
1815 match self.tables.get_mut(table) {
1816 Some(None) => {}
1817 Some(Some(keys)) => {
1818 keys.insert(key);
1819 }
1820 None => {
1821 self.tables
1822 .insert(table.to_owned(), Some(BTreeSet::from([key])));
1823 }
1824 }
1825 }
1826
1827 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
1828 self.windows
1829 .entry((base_key.to_owned(), table.to_owned()))
1830 .or_default()
1831 .insert(unit.to_owned());
1832 }
1833
1834 fn touched(&self) -> bool {
1835 !self.tables.is_empty()
1836 || !self.windows.is_empty()
1837 || self.status
1838 || self.conflicts
1839 || self.rejections
1840 || self.outcomes
1841 }
1842}
1843
1844const PUSH_OPS_PER_REQUEST: usize = 500;
1850const MAX_LOCAL_PURGE_TARGETS: usize = 64;
1851const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
1852const MAX_LOCAL_PURGE_VALUES: usize = 128;
1853const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
1854pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
1855
1856pub struct SyncClient {
1857 conn: Connection,
1858 schema: ClientSchema,
1859 client_id: String,
1860 limits: ClientLimits,
1861 subs: Vec<Subscription>,
1862 outbox: Vec<OutboxCommit>,
1863 conflicts: Vec<ConflictRecord>,
1864 rejections: Vec<RejectionRecord>,
1865 schema_floor: Option<SchemaFloor>,
1866 lease_state: Option<LeaseState>,
1868 stopped: bool,
1870 upgrading: bool,
1872 sync_needed: bool,
1874 realtime_connected: bool,
1875 presence: HashMap<String, HashMap<String, PresencePeer>>,
1877 now_ms: Option<i64>,
1880 encryption: crate::values::EncryptionConfig,
1884 security_preflight: bool,
1887 insert_sql: RefCell<HashMap<String, String>>,
1893 overlay_dirty: Cell<bool>,
1897 #[cfg(test)]
1900 overlay_rebuild_count: Cell<usize>,
1901 #[cfg(test)]
1904 outcome_prune_count: Cell<usize>,
1905 change_queue: VecDeque<ClientChangeBatch>,
1907 sync_intent_queue: VecDeque<SyncIntent>,
1908 retry_delay_ms: u64,
1910 last_round: Option<DiagnosticLastRound>,
1911 last_change: Option<DiagnosticLastChange>,
1912}
1913
1914fn quote_ident(name: &str) -> String {
1915 format!("\"{}\"", name.replace('"', "\"\""))
1916}
1917
1918fn is_local_operation_code_like(value: &str) -> bool {
1919 let bytes = value.as_bytes();
1920 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
1921 && bytes
1922 .iter()
1923 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
1924}
1925
1926fn base_table(name: &str) -> String {
1927 quote_ident(&format!("_syncular_base_{name}"))
1928}
1929
1930type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
1935
1936fn window_base_key(base: &WindowBase) -> String {
1937 format!(
1938 "{}\0{}\0{}",
1939 base.table,
1940 base.variable,
1941 canonical_scope_json(&base.fixed_scopes)
1942 )
1943}
1944
1945fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
1947 let mut scopes = base.fixed_scopes.clone();
1948 scopes.retain(|(k, _)| k != &base.variable);
1949 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
1950 scopes
1951}
1952
1953fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
1957 let canonical = canonical_scope_json(&unit_scopes(base, unit));
1958 let digest = Sha256::digest(canonical.as_bytes());
1959 let hex = bytes_to_hex(&digest);
1960 format!("w:{}:{}", base.table, &hex[..16])
1961}
1962
1963fn visible_table(name: &str) -> String {
1964 quote_ident(name)
1965}
1966
1967const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
1968
1969fn is_synced_table_name(name: &str) -> bool {
1972 if name.starts_with("sqlite_") {
1973 return false;
1974 }
1975 if name.starts_with("_syncular_base_") {
1976 return true; }
1978 !name.starts_with("_syncular_")
1980}
1981
1982fn blob_id_for(bytes: &[u8]) -> String {
1984 let digest = Sha256::digest(bytes);
1985 format!("sha256:{}", bytes_to_hex(&digest))
1986}
1987
1988enum RowParam<'a> {
1992 Cell(&'a Option<ColumnValue>),
1993 Version(i64),
1994}
1995
1996impl rusqlite::ToSql for RowParam<'_> {
1997 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
1998 Ok(match self {
1999 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
2000 RowParam::Cell(cell) => match cell {
2001 None => ToSqlOutput::Owned(SqlValue::Null),
2002 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
2003 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
2004 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
2005 Some(ColumnValue::Boolean(b)) => {
2006 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
2007 }
2008 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
2009 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
2010 }
2011 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
2013 ToSqlOutput::Borrowed(ValueRef::Blob(b))
2014 }
2015 },
2016 })
2017 }
2018}
2019
2020fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
2028 use ssp2::segment::ColumnType;
2029 let mismatch = || {
2030 Err(format!(
2031 "image column {:?} holds a value of the wrong type",
2032 column.name
2033 ))
2034 };
2035 match value {
2036 ValueRef::Null => {
2037 if !column.nullable {
2038 return Err(format!(
2039 "image column {:?} is NULL but not nullable",
2040 column.name
2041 ));
2042 }
2043 Ok(ToSqlOutput::Owned(SqlValue::Null))
2044 }
2045 ValueRef::Integer(i) => match column.ty {
2046 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
2047 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
2048 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
2049 _ => mismatch(),
2050 },
2051 ValueRef::Real(_) => match column.ty {
2052 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
2053 _ => mismatch(),
2054 },
2055 ValueRef::Text(t) => {
2056 std::str::from_utf8(t)
2057 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
2058 match column.ty {
2059 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
2060 Ok(ToSqlOutput::Borrowed(value))
2061 }
2062 _ => mismatch(),
2063 }
2064 }
2065 ValueRef::Blob(_) => match column.ty {
2066 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
2068 _ => mismatch(),
2069 },
2070 }
2071}
2072
2073fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
2074 use rusqlite::types::ValueRef;
2075 match value {
2076 ValueRef::Null => Value::Null,
2077 ValueRef::Integer(i) => match column.ty {
2078 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
2079 ssp2::segment::ColumnType::Float => {
2080 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
2081 }
2082 _ => Value::from(i),
2083 },
2084 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
2085 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
2086 ValueRef::Blob(b) => {
2087 let mut map = Map::new();
2088 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
2089 Value::Object(map)
2090 }
2091 }
2092}
2093
2094fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
2098 Ok(match value {
2099 Value::Null => SqlValue::Null,
2100 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
2101 Value::Number(n) => {
2102 if let Some(i) = n.as_i64() {
2103 SqlValue::Integer(i)
2104 } else if let Some(f) = n.as_f64() {
2105 SqlValue::Real(f)
2106 } else {
2107 return Err(format!("query param number {n} is out of range"));
2108 }
2109 }
2110 Value::String(s) => SqlValue::Text(s.clone()),
2111 Value::Object(_) => {
2112 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
2113 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
2114 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
2115 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
2116 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
2117 })?)
2118 } else {
2119 return Err(
2120 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
2121 .to_owned(),
2122 );
2123 }
2124 }
2125 Value::Array(_) => return Err("query array params are not supported".to_owned()),
2126 })
2127}
2128
2129fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
2134 use rusqlite::types::ValueRef;
2135 match value {
2136 ValueRef::Null => Value::Null,
2137 ValueRef::Integer(i) => {
2138 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
2142 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
2143 Value::from(i)
2144 } else {
2145 let mut map = Map::new();
2146 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
2147 Value::Object(map)
2148 }
2149 }
2150 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
2151 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
2152 ValueRef::Blob(b) => {
2153 let mut map = Map::new();
2154 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
2155 Value::Object(map)
2156 }
2157 }
2158}
2159
2160fn query_connection(
2161 conn: &Connection,
2162 sql: &str,
2163 params: &[QueryValue],
2164) -> Result<Vec<QueryRow>, String> {
2165 crate::query_guard::assert_read_only_query(sql)?;
2166 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
2167 let bound: Vec<SqlValue> = params
2168 .iter()
2169 .map(json_param_to_sql)
2170 .collect::<Result<_, _>>()?;
2171 let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
2172 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
2173 let bound_refs: Vec<&dyn rusqlite::ToSql> =
2174 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
2175 let mut sql_rows = stmt
2176 .query(bound_refs.as_slice())
2177 .map_err(|e| e.to_string())?;
2178 let mut out = Vec::new();
2179 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
2180 let mut record = Map::new();
2181 for (i, name) in column_names.iter().enumerate() {
2182 let value = row.get_ref(i).map_err(|e| e.to_string())?;
2183 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
2184 }
2185 out.push(record);
2186 }
2187 Ok(out)
2188}
2189
2190fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
2191 let mut stmt = conn
2192 .prepare(
2193 "SELECT windows.unit, subscriptions.state_json
2194 FROM _syncular_windows AS windows
2195 JOIN _syncular_subscriptions AS subscriptions
2196 ON subscriptions.id = windows.sub_id
2197 WHERE windows.base = ?1
2198 ORDER BY windows.unit ASC",
2199 )
2200 .map_err(|error| error.to_string())?;
2201 let rows = stmt
2202 .query_map(rusqlite::params![window_base_key(base)], |row| {
2203 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2204 })
2205 .map_err(|error| error.to_string())?;
2206 let mut units = Vec::new();
2207 let mut pending = Vec::new();
2208 for row in rows {
2209 let (unit, raw) = row.map_err(|error| error.to_string())?;
2210 let state: Value = serde_json::from_str(&raw)
2211 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
2212 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
2213 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
2214 || state
2215 .get("bootstrapState")
2216 .is_some_and(|value| !value.is_null());
2217 if is_pending {
2218 pending.push(unit.clone());
2219 }
2220 units.push(unit);
2221 }
2222 Ok(WindowState { units, pending })
2223}
2224
2225fn snapshot_connection(
2226 conn: &Connection,
2227 sql: &str,
2228 params: &[Value],
2229 coverage: &[WindowCoverage],
2230) -> Result<QuerySnapshot, String> {
2231 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
2232 .map_err(|error| error.to_string())?;
2233 let result = (|| {
2234 let revision = conn
2235 .query_row(
2236 "SELECT value FROM _syncular_meta WHERE key = ?1",
2237 rusqlite::params![LOCAL_REVISION_KEY],
2238 |row| row.get::<_, String>(0),
2239 )
2240 .ok()
2241 .and_then(|value| value.parse::<u64>().ok())
2242 .unwrap_or(0);
2243 let rows = query_connection(conn, sql, params)?;
2244 let mut pending = Vec::new();
2245 let mut missing = Vec::new();
2246 for requested in coverage {
2247 let base_key = window_base_key(&requested.base);
2248 let state = persisted_window_state(conn, &requested.base)?;
2249 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
2250 let reference = WindowUnitRef {
2251 base_key: base_key.clone(),
2252 unit: unit.clone(),
2253 };
2254 if !state.units.iter().any(|held| held == &unit) {
2255 missing.push(reference);
2256 } else if state.pending.iter().any(|held| held == &unit) {
2257 pending.push(reference);
2258 }
2259 }
2260 }
2261 Ok(QuerySnapshot {
2262 revision: revision.to_string(),
2263 rows,
2264 coverage: CoverageSnapshot {
2265 complete: pending.is_empty() && missing.is_empty(),
2266 pending,
2267 missing,
2268 },
2269 })
2270 })();
2271 match result {
2272 Ok(snapshot) => {
2273 conn.execute_batch("RELEASE syncular_snapshot_read")
2274 .map_err(|error| error.to_string())?;
2275 Ok(snapshot)
2276 }
2277 Err(error) => {
2278 let _ = conn.execute_batch(
2279 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
2280 );
2281 Err(error)
2282 }
2283 }
2284}
2285
2286pub struct FileQuerySnapshotReader {
2291 path: String,
2292 conn: Option<Connection>,
2293}
2294
2295impl FileQuerySnapshotReader {
2296 #[must_use]
2297 pub fn new(path: impl Into<String>) -> Self {
2298 Self {
2299 path: path.into(),
2300 conn: None,
2301 }
2302 }
2303
2304 fn connection(&mut self) -> Result<&Connection, String> {
2305 if self.conn.is_none() {
2306 let conn = Connection::open_with_flags(
2307 &self.path,
2308 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
2309 )
2310 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
2311 conn.busy_timeout(std::time::Duration::from_millis(250))
2312 .map_err(|error| error.to_string())?;
2313 self.conn = Some(conn);
2314 }
2315 self.conn
2316 .as_ref()
2317 .ok_or_else(|| "read sidecar connection missing".to_owned())
2318 }
2319
2320 pub fn query_snapshot(
2321 &mut self,
2322 sql: &str,
2323 params: &[Value],
2324 coverage: &[WindowCoverage],
2325 ) -> Result<QuerySnapshot, String> {
2326 snapshot_connection(self.connection()?, sql, params, coverage)
2327 }
2328}
2329
2330impl SyncClient {
2331 pub fn new_with_identity(
2332 client_id: Option<String>,
2333 schema_json: &Value,
2334 limits: ClientLimits,
2335 ) -> Result<Self, String> {
2336 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
2337 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
2338 Self::with_connection(resolved, schema_json, limits, conn)
2339 }
2340
2341 pub fn new(
2342 client_id: String,
2343 schema_json: &Value,
2344 limits: ClientLimits,
2345 ) -> Result<Self, String> {
2346 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
2347 Self::with_connection(client_id, schema_json, limits, conn)
2348 }
2349
2350 pub fn open_path(
2356 client_id: String,
2357 schema_json: &Value,
2358 limits: ClientLimits,
2359 path: &str,
2360 ) -> Result<Self, String> {
2361 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
2362 Self::with_connection(client_id, schema_json, limits, conn)
2363 }
2364
2365 pub fn open_path_with_identity(
2366 client_id: Option<String>,
2367 schema_json: &Value,
2368 limits: ClientLimits,
2369 path: &str,
2370 ) -> Result<Self, String> {
2371 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
2372 conn.busy_timeout(std::time::Duration::from_millis(250))
2378 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
2379 conn.pragma_update(None, "journal_mode", "WAL")
2380 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
2381 let persisted = conn
2382 .query_row(
2383 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
2384 [],
2385 |row| row.get::<_, String>(0),
2386 )
2387 .ok();
2388 let resolved = persisted
2389 .clone()
2390 .or(client_id.clone())
2391 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
2392 if let (Some(existing), Some(requested)) = (persisted, client_id) {
2393 if existing != requested {
2394 return Err(format!(
2395 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
2396 ));
2397 }
2398 }
2399 Self::with_connection(resolved, schema_json, limits, conn)
2400 }
2401
2402 #[must_use]
2403 pub fn client_id(&self) -> &str {
2404 &self.client_id
2405 }
2406
2407 pub fn with_connection(
2414 client_id: String,
2415 schema_json: &Value,
2416 limits: ClientLimits,
2417 conn: Connection,
2418 ) -> Result<Self, String> {
2419 if limits.outcome_retention_max_entries == Some(0) {
2420 return Err(
2421 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
2422 );
2423 }
2424 let schema = parse_schema_json(schema_json)?;
2425 let mut client = SyncClient {
2426 conn,
2427 schema,
2428 client_id,
2429 limits,
2430 subs: Vec::new(),
2431 outbox: Vec::new(),
2432 conflicts: Vec::new(),
2433 rejections: Vec::new(),
2434 schema_floor: None,
2435 lease_state: None,
2436 stopped: false,
2437 upgrading: false,
2438 sync_needed: false,
2439 realtime_connected: false,
2440 presence: HashMap::new(),
2441 now_ms: None,
2442 encryption: crate::values::EncryptionConfig::default(),
2443 security_preflight: false,
2444 insert_sql: RefCell::new(HashMap::new()),
2445 overlay_dirty: Cell::new(false),
2446 #[cfg(test)]
2447 overlay_rebuild_count: Cell::new(0),
2448 #[cfg(test)]
2449 outcome_prune_count: Cell::new(0),
2450 change_queue: VecDeque::new(),
2451 sync_intent_queue: VecDeque::new(),
2452 retry_delay_ms: 250,
2453 last_round: None,
2454 last_change: None,
2455 };
2456 client
2460 .conn
2461 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
2462 client.create_bookkeeping_tables()?;
2466 match client.get_meta(CLIENT_ID_KEY) {
2467 Some(existing) if existing != client.client_id => {
2468 return Err(format!(
2469 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
2470 client.client_id
2471 ));
2472 }
2473 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
2474 _ => {}
2475 }
2476 client.restore_persisted_state()?;
2477 let marker = client
2478 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
2479 .and_then(|value| value.parse::<i32>().ok());
2480 match marker {
2481 None => {
2482 client.create_synced_tables()?;
2483 client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
2484 }
2485 Some(version) if version == client.schema.version => {
2486 client.create_synced_tables()?;
2487 }
2488 Some(_) => client.run_schema_reset()?,
2489 }
2490 client.clear_satisfied_persisted_schema_floor();
2491 client.prune_unknown_subscriptions()?;
2492 if marker == Some(client.schema.version) && !client.outbox.is_empty() {
2493 client.overlay_dirty.set(true);
2496 client.rebuild_overlay();
2497 }
2498 client.enqueue_startup_sync_if_needed();
2504 Ok(client)
2505 }
2506
2507 pub fn set_now_ms(&mut self, now_ms: i64) {
2510 self.now_ms = Some(now_ms);
2511 }
2512
2513 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
2517 self.encryption = encryption;
2518 }
2519
2520 #[must_use]
2521 pub fn security_lifecycle(&self) -> &'static str {
2522 if self.security_preflight {
2523 "preflight"
2524 } else {
2525 "active"
2526 }
2527 }
2528
2529 #[must_use]
2530 pub fn security_preflight(&self) -> bool {
2531 self.security_preflight
2532 }
2533
2534 pub fn begin_security_preflight(&mut self) {
2541 self.seal_security_on_teardown();
2542 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
2545 }
2546
2547 pub fn seal_security_on_teardown(&mut self) {
2557 self.security_preflight = true;
2558 self.encryption = crate::values::EncryptionConfig::default();
2559 self.sync_intent_queue.clear();
2560 }
2561
2562 pub fn activate_security(
2564 &mut self,
2565 encryption: crate::values::EncryptionConfig,
2566 ) -> Result<(), String> {
2567 if !self.security_preflight {
2568 return Err(
2569 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
2570 );
2571 }
2572 self.encryption = encryption;
2573 self.security_preflight = false;
2574 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
2575 self.enqueue_startup_sync_if_needed();
2576 Ok(())
2577 }
2578
2579 fn clock_now_ms(&self) -> i64 {
2580 self.now_ms.unwrap_or_else(|| {
2581 std::time::SystemTime::now()
2582 .duration_since(std::time::UNIX_EPOCH)
2583 .map(|d| d.as_millis() as i64)
2584 .unwrap_or(0)
2585 })
2586 }
2587
2588 fn create_bookkeeping_tables(&self) -> Result<(), String> {
2589 self.conn
2591 .execute_batch(
2592 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
2593 seq INTEGER PRIMARY KEY AUTOINCREMENT,
2594 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
2595 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
2596 seq INTEGER PRIMARY KEY AUTOINCREMENT,
2597 client_commit_id TEXT NOT NULL UNIQUE,
2598 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
2599 recorded_at_ms INTEGER NOT NULL,
2600 results_json TEXT NOT NULL,
2601 operations_json TEXT,
2602 resolution TEXT NOT NULL DEFAULT 'active'
2603 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
2604 resolved_at_ms INTEGER,
2605 replacement_client_commit_id TEXT);
2606 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
2607 ON _syncular_commit_outcomes(resolution, seq);
2608 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
2609 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
2610 CREATE TABLE IF NOT EXISTS _syncular_meta (
2611 key TEXT PRIMARY KEY, value TEXT NOT NULL);
2612 CREATE TABLE IF NOT EXISTS _syncular_windows (
2613 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
2614 PRIMARY KEY (base, unit));
2615 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
2616 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
2617 effective_scopes TEXT NOT NULL);",
2618 )
2619 .map_err(|e| e.to_string())?;
2620 let _ = self
2623 .conn
2624 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
2625 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
2626 self.set_meta(LOCAL_REVISION_KEY, "0");
2627 }
2628 if self.schema_has_blobs() {
2633 self.conn
2634 .execute_batch(
2635 "CREATE TABLE IF NOT EXISTS _syncular_blobs (blob_id TEXT PRIMARY KEY,
2636 bytes BLOB NOT NULL, byte_length INTEGER NOT NULL,
2637 media_type TEXT, refcount INTEGER NOT NULL DEFAULT 0,
2638 created_at_ms INTEGER NOT NULL,
2639 last_used_ms INTEGER NOT NULL DEFAULT 0);
2640 CREATE TABLE IF NOT EXISTS _syncular_blob_uploads (blob_id TEXT PRIMARY KEY,
2641 media_type TEXT, created_at_ms INTEGER NOT NULL);",
2642 )
2643 .map_err(|e| e.to_string())?;
2644 let _ = self.conn.execute_batch(
2648 "ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0",
2649 );
2650 }
2651 Ok(())
2652 }
2653
2654 fn schema_has_blobs(&self) -> bool {
2656 self.schema
2657 .tables
2658 .iter()
2659 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
2660 }
2661
2662 fn get_meta(&self, key: &str) -> Option<String> {
2665 self.conn
2666 .query_row(
2667 "SELECT value FROM _syncular_meta WHERE key = ?1",
2668 rusqlite::params![key],
2669 |row| row.get::<_, String>(0),
2670 )
2671 .ok()
2672 }
2673
2674 fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
2675 self.conn
2676 .query_row(
2677 "SELECT value FROM _syncular_meta WHERE key = ?1",
2678 rusqlite::params![key],
2679 |row| row.get::<_, String>(0),
2680 )
2681 .optional()
2682 .map_err(|_| {
2683 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
2684 })
2685 }
2686
2687 fn set_meta(&self, key: &str, value: &str) {
2688 let _ = self.conn.execute(
2689 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
2690 rusqlite::params![key, value],
2691 );
2692 }
2693
2694 fn delete_meta(&self, key: &str) {
2695 let _ = self.conn.execute(
2696 "DELETE FROM _syncular_meta WHERE key = ?1",
2697 rusqlite::params![key],
2698 );
2699 }
2700
2701 fn restore_persisted_state(&mut self) -> Result<(), String> {
2702 self.subs = {
2703 let mut stmt = self
2704 .conn
2705 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
2706 .map_err(|error| error.to_string())?;
2707 let rows = stmt
2708 .query_map([], |row| {
2709 Ok((
2710 row.get::<_, String>(0)?,
2711 row.get::<_, String>(1)?,
2712 row.get::<_, String>(2)?,
2713 ))
2714 })
2715 .map_err(|error| error.to_string())?;
2716 let mut subscriptions = Vec::new();
2717 for row in rows {
2718 let (id, table, raw) = row.map_err(|error| error.to_string())?;
2719 let state: Value = serde_json::from_str(&raw)
2720 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
2721 let requested = json_to_scope_map(
2722 state.get("requested").unwrap_or(&Value::Object(Map::new())),
2723 )?;
2724 let effective = state
2725 .get("effectiveScopes")
2726 .filter(|value| !value.is_null())
2727 .map(json_to_scope_map)
2728 .transpose()?;
2729 subscriptions.push(Subscription {
2730 id,
2731 table,
2732 requested,
2733 params: state
2734 .get("params")
2735 .and_then(Value::as_str)
2736 .map(str::to_owned),
2737 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
2738 bootstrap_state: state
2739 .get("bootstrapState")
2740 .and_then(Value::as_str)
2741 .map(str::to_owned),
2742 state: SubState::parse(
2743 state
2744 .get("status")
2745 .and_then(Value::as_str)
2746 .unwrap_or("active"),
2747 ),
2748 reason_code: state
2749 .get("reasonCode")
2750 .and_then(Value::as_str)
2751 .map(str::to_owned),
2752 effective,
2753 synced_once: state
2754 .get("syncedOnce")
2755 .and_then(Value::as_bool)
2756 .unwrap_or(false),
2757 });
2758 }
2759 subscriptions
2760 };
2761
2762 self.outbox = {
2763 let mut stmt = self
2764 .conn
2765 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
2766 .map_err(|error| error.to_string())?;
2767 let rows = stmt
2768 .query_map([], |row| {
2769 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2770 })
2771 .map_err(|error| error.to_string())?;
2772 let mut commits = Vec::new();
2773 for row in rows {
2774 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
2775 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
2776 format!("invalid persisted outbox {client_commit_id:?}: {error}")
2777 })?;
2778 let mut ops = Vec::with_capacity(entries.len());
2779 for entry in entries {
2780 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
2781 ops.push(OutboxOp {
2782 upsert: op == "upsert",
2783 table: entry
2784 .get("table")
2785 .and_then(Value::as_str)
2786 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
2787 .to_owned(),
2788 row_id: entry
2789 .get("rowId")
2790 .and_then(Value::as_str)
2791 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
2792 .to_owned(),
2793 base_version: entry.get("baseVersion").and_then(Value::as_i64),
2794 values: entry.get("values").and_then(Value::as_object).cloned(),
2795 changed_fields: entry.get("changedFields").and_then(Value::as_array).map(
2796 |values| {
2797 values
2798 .iter()
2799 .filter_map(Value::as_str)
2800 .map(str::to_owned)
2801 .collect()
2802 },
2803 ),
2804 });
2805 }
2806 commits.push(OutboxCommit {
2807 client_commit_id,
2808 ops,
2809 });
2810 }
2811 commits
2812 };
2813
2814 self.prune_commit_outcomes()?;
2815 let active = self.commit_outcomes(CommitOutcomeQuery {
2816 active_only: true,
2817 ..CommitOutcomeQuery::default()
2818 })?;
2819 self.conflicts = active
2820 .iter()
2821 .flat_map(|outcome| outcome.results.iter())
2822 .filter_map(|result| match result {
2823 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
2824 _ => None,
2825 })
2826 .collect();
2827 self.rejections = active
2828 .iter()
2829 .flat_map(|outcome| outcome.results.iter())
2830 .filter_map(|result| match result {
2831 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
2832 _ => None,
2833 })
2834 .collect();
2835
2836 self.lease_state = self
2837 .get_meta(LEASE_STATE_KEY)
2838 .map(|raw| serde_json::from_str(&raw))
2839 .transpose()
2840 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
2841 self.schema_floor = self
2842 .get_meta(SCHEMA_FLOOR_KEY)
2843 .map(|raw| serde_json::from_str(&raw))
2844 .transpose()
2845 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
2846 self.stopped = self.schema_floor.is_some();
2847 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
2850 self.security_preflight = true;
2851 }
2852 Ok(())
2853 }
2854
2855 fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
2858 let valid_tables: BTreeSet<String> = self
2859 .schema
2860 .tables
2861 .iter()
2862 .map(|table| table.name.clone())
2863 .collect();
2864 let stale_ids: Vec<String> = self
2865 .subs
2866 .iter()
2867 .filter(|sub| !valid_tables.contains(&sub.table))
2868 .map(|sub| sub.id.clone())
2869 .collect();
2870 for id in &stale_ids {
2871 self.conn
2872 .execute(
2873 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
2874 rusqlite::params![id],
2875 )
2876 .map_err(|error| error.to_string())?;
2877 self.conn
2878 .execute(
2879 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
2880 rusqlite::params![id],
2881 )
2882 .map_err(|error| error.to_string())?;
2883 self.conn
2884 .execute(
2885 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
2886 rusqlite::params![id],
2887 )
2888 .map_err(|error| error.to_string())?;
2889 }
2890 self.subs.retain(|sub| valid_tables.contains(&sub.table));
2891 Ok(())
2892 }
2893
2894 #[must_use]
2895 pub fn local_revision(&self) -> u64 {
2896 self.get_meta(LOCAL_REVISION_KEY)
2897 .and_then(|value| value.parse().ok())
2898 .unwrap_or(0)
2899 }
2900
2901 #[must_use]
2902 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
2903 SyncStatusSnapshot {
2904 current_schema_version: self.schema.version,
2905 outbox: self.outbox.len(),
2906 upgrading: self.upgrading,
2907 lease_state: self.lease_state.clone(),
2908 schema_floor: self.schema_floor.clone(),
2909 sync_needed: self.sync_needed,
2910 }
2911 }
2912
2913 pub fn diagnostics_snapshot(
2914 &self,
2915 request: &ClientDiagnosticsRequest,
2916 ) -> Result<ClientDiagnosticsSnapshot, String> {
2917 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
2918 return Err(format!(
2919 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
2920 ));
2921 }
2922 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
2923 for sub in &self.subs {
2924 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
2925 let complete =
2926 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
2927 let state = match sub.state {
2928 SubState::Revoked => "revoked",
2929 SubState::Failed => "failed",
2930 SubState::Active if reset => "reset",
2931 SubState::Active if complete => "complete",
2932 SubState::Active => "bootstrapping",
2933 };
2934 subscriptions.insert(
2935 sub.id.clone(),
2936 DiagnosticSubscription {
2937 id: sub.id.clone(),
2938 table: sub.table.clone(),
2939 state: state.to_owned(),
2940 complete,
2941 cursor: Some(sub.cursor),
2942 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
2943 },
2944 );
2945 }
2946 for expected in &request.expected_subscriptions {
2947 if expected.id.is_empty() || expected.table.is_empty() {
2948 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
2949 }
2950 if subscriptions
2951 .get(&expected.id)
2952 .is_some_and(|registered| registered.table != expected.table)
2953 {
2954 subscriptions.insert(
2955 expected.id.clone(),
2956 DiagnosticSubscription {
2957 id: expected.id.clone(),
2958 table: expected.table.clone(),
2959 state: "failed".to_owned(),
2960 complete: false,
2961 cursor: None,
2962 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
2963 },
2964 );
2965 } else {
2966 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
2967 DiagnosticSubscription {
2968 id: expected.id.clone(),
2969 table: expected.table.clone(),
2970 state: "unregistered".to_owned(),
2971 complete: false,
2972 cursor: None,
2973 reason_code: None,
2974 }
2975 });
2976 }
2977 }
2978 let mut ordered_subscriptions = Vec::new();
2979 let mut included = BTreeSet::new();
2980 for expected in &request.expected_subscriptions {
2981 if included.insert(expected.id.clone()) {
2982 if let Some(subscription) = subscriptions.get(&expected.id) {
2983 ordered_subscriptions.push(subscription.clone());
2984 }
2985 }
2986 }
2987 for (id, subscription) in &subscriptions {
2988 if included.insert(id.clone()) {
2989 ordered_subscriptions.push(subscription.clone());
2990 }
2991 }
2992 let subscriptions_truncated =
2993 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
2994 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
2995 let captured_at_ms = self.clock_now_ms();
2996 let lease = if let Some(error_code) = self
2997 .lease_state
2998 .as_ref()
2999 .and_then(|state| state.error_code.clone())
3000 {
3001 ClientDiagnosticsLease {
3002 state: "stopped".to_owned(),
3003 expires_at_ms: self
3004 .lease_state
3005 .as_ref()
3006 .and_then(|state| state.expires_at_ms),
3007 error_code: Some(Self::diagnostic_code(&error_code)),
3008 }
3009 } else if let Some(expires_at_ms) = self
3010 .lease_state
3011 .as_ref()
3012 .and_then(|state| state.expires_at_ms)
3013 {
3014 ClientDiagnosticsLease {
3015 state: if expires_at_ms <= captured_at_ms {
3016 "expired".to_owned()
3017 } else {
3018 "active".to_owned()
3019 },
3020 expires_at_ms: Some(expires_at_ms),
3021 error_code: None,
3022 }
3023 } else {
3024 ClientDiagnosticsLease {
3025 state: "none".to_owned(),
3026 expires_at_ms: None,
3027 error_code: None,
3028 }
3029 };
3030 let connectivity = match self.last_round.as_ref() {
3031 Some(round) if round.status == "succeeded" => "online",
3032 Some(round)
3033 if round.status == "failed"
3034 && round
3035 .error_code
3036 .as_deref()
3037 .is_some_and(Self::retryable_transport_code) =>
3038 {
3039 "offline"
3040 }
3041 _ => "unknown",
3042 };
3043 Ok(ClientDiagnosticsSnapshot {
3044 version: CLIENT_DIAGNOSTICS_VERSION,
3045 captured_at_ms,
3046 host: ClientDiagnosticsHost {
3047 kind: "direct".to_owned(),
3048 role: "single".to_owned(),
3049 connectivity: connectivity.to_owned(),
3050 realtime: if self.realtime_connected {
3051 "connected".to_owned()
3052 } else {
3053 "disconnected".to_owned()
3054 },
3055 },
3056 security_lifecycle: self.security_lifecycle().to_owned(),
3057 schema: ClientDiagnosticsSchema {
3058 current_version: self.schema.version,
3059 upgrading: self.upgrading,
3060 required_version: self
3061 .schema_floor
3062 .as_ref()
3063 .and_then(|floor| floor.required_schema_version),
3064 latest_version: self
3065 .schema_floor
3066 .as_ref()
3067 .and_then(|floor| floor.latest_schema_version),
3068 },
3069 replica: ClientDiagnosticsReplica {
3070 local_revision: self.local_revision().to_string(),
3071 sync_needed: self.sync_needed,
3072 pending_outbox: self.outbox.len(),
3073 },
3074 lease,
3075 subscriptions: ordered_subscriptions,
3076 subscriptions_truncated,
3077 last_round: self.last_round.clone(),
3078 last_change: self.last_change.clone(),
3079 storage: self.diagnostics_storage(),
3080 })
3081 }
3082
3083 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
3084 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
3085 let page_count: i64 = self
3086 .conn
3087 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
3088 let page_size: i64 = self
3089 .conn
3090 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
3091 let outbox_bytes: i64 = self.conn.query_row(
3092 "SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox",
3093 [],
3094 |row| row.get(0),
3095 )?;
3096 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.query_row(
3097 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
3098 [],
3099 |row| Ok((row.get(0)?, row.get(1)?)),
3100 )?;
3101 let blob_bytes = if self.schema_has_blobs() {
3102 self.conn.query_row(
3103 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
3104 [],
3105 |row| row.get(0),
3106 )?
3107 } else {
3108 0
3109 };
3110 let pressure = self
3111 .limits
3112 .blob_cache_max_bytes
3113 .is_some_and(|limit| blob_bytes > limit);
3114 Ok(ClientDiagnosticsStorage {
3115 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
3116 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
3117 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
3118 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
3119 retained_outcome_entries: Some(outcome_entries.max(0)),
3120 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
3121 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
3122 })
3123 };
3124 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
3125 status: "unreadable".to_owned(),
3126 database_bytes_approx: None,
3127 pending_outbox_bytes_approx: None,
3128 retained_outcome_bytes_approx: None,
3129 retained_outcome_entries: None,
3130 blob_cache_bytes_approx: None,
3131 pressure_reason_code: None,
3132 })
3133 }
3134
3135 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
3136 self.change_queue.drain(..).collect()
3137 }
3138
3139 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
3140 self.sync_intent_queue.drain(..).collect()
3141 }
3142
3143 fn schedule_background_retry(&mut self) {
3144 self.sync_intent_queue.push_back(SyncIntent::Background {
3145 delay_ms: self.retry_delay_ms,
3146 });
3147 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
3148 }
3149
3150 fn reset_background_retry(&mut self) {
3151 self.retry_delay_ms = 250;
3152 }
3153
3154 fn retryable_transport_code(code: &str) -> bool {
3155 code == "transport.failed"
3156 || code == "transport.unavailable"
3157 || code == "sync.transport_failed"
3158 }
3159
3160 fn diagnostic_code(code: &str) -> String {
3161 let valid = !code.is_empty()
3162 && code.len() <= 96
3163 && code.contains('.')
3164 && code
3165 .bytes()
3166 .next()
3167 .is_some_and(|byte| byte.is_ascii_lowercase())
3168 && code.bytes().all(|byte| {
3169 byte.is_ascii_lowercase()
3170 || byte.is_ascii_digit()
3171 || matches!(byte, b'.' | b'_' | b'-')
3172 });
3173 if valid {
3174 code.to_owned()
3175 } else {
3176 "client.unknown_failure".to_owned()
3177 }
3178 }
3179
3180 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
3181 if self.sync_needed != value {
3182 if self.begin_observation("syncular_status").is_ok() {
3183 self.sync_needed = value;
3184 let batch = ChangeAccumulator {
3185 status: true,
3186 ..ChangeAccumulator::default()
3187 };
3188 if self.finish_observation("syncular_status", batch).is_err() {
3189 self.rollback_observation("syncular_status");
3190 }
3191 } else {
3192 self.sync_needed = value;
3193 }
3194 }
3195 if value && interactive {
3196 self.sync_intent_queue.push_back(SyncIntent::Interactive);
3197 }
3198 }
3199
3200 fn begin_observation(&self, name: &str) -> Result<(), String> {
3201 self.conn
3202 .execute_batch(&format!("SAVEPOINT {name}"))
3203 .map_err(|error| error.to_string())
3204 }
3205
3206 fn rollback_observation(&self, name: &str) {
3207 let _ = self
3208 .conn
3209 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
3210 }
3211
3212 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
3213 if !batch.touched() {
3214 self.conn
3215 .execute_batch(&format!("RELEASE {name}"))
3216 .map_err(|error| error.to_string())?;
3217 return Ok(());
3218 }
3219 let revision = self
3220 .local_revision()
3221 .checked_add(1)
3222 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
3223 self.conn
3224 .execute(
3225 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
3226 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
3227 )
3228 .map_err(|error| error.to_string())?;
3229 let status = batch.status.then(|| self.status_snapshot());
3230 let event = ClientChangeBatch {
3231 revision: revision.to_string(),
3232 tables: batch
3233 .tables
3234 .into_iter()
3235 .map(|(table, scope_keys)| TableChange {
3236 table,
3237 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
3238 })
3239 .collect(),
3240 windows: batch
3241 .windows
3242 .into_iter()
3243 .map(|((base_key, table), units)| WindowChange {
3244 base_key,
3245 table,
3246 units: units.into_iter().collect(),
3247 })
3248 .collect(),
3249 status,
3250 conflicts_changed: batch.conflicts,
3251 rejections_changed: batch.rejections,
3252 outcomes_changed: batch.outcomes,
3253 };
3254 self.conn
3255 .execute_batch(&format!("RELEASE {name}"))
3256 .map_err(|error| error.to_string())?;
3257 let mut diagnostic_tables = event
3258 .tables
3259 .iter()
3260 .map(|entry| entry.table.clone())
3261 .collect::<BTreeSet<_>>()
3262 .into_iter()
3263 .collect::<Vec<_>>();
3264 let mut diagnostic_windows = event
3265 .windows
3266 .iter()
3267 .map(|entry| entry.table.clone())
3268 .collect::<BTreeSet<_>>()
3269 .into_iter()
3270 .collect::<Vec<_>>();
3271 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
3272 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
3273 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
3274 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
3275 self.last_change = Some(DiagnosticLastChange {
3276 revision: event.revision.clone(),
3277 recorded_at_ms: self.clock_now_ms(),
3278 tables: diagnostic_tables,
3279 windows: diagnostic_windows,
3280 domains_truncated,
3281 status_changed: event.status.is_some(),
3282 conflicts_changed: event.conflicts_changed,
3283 rejections_changed: event.rejections_changed,
3284 outcomes_changed: event.outcomes_changed,
3285 });
3286 self.change_queue.push_back(event);
3287 Ok(())
3288 }
3289
3290 fn record_scope_map(
3291 &self,
3292 batch: &mut ChangeAccumulator,
3293 table_name: &str,
3294 scopes: &[(String, Vec<String>)],
3295 ) {
3296 let Some(table) = self.schema.table(table_name) else {
3297 return;
3298 };
3299 for (variable, values) in scopes {
3300 let Some(scope) = table
3301 .scope_variables
3302 .iter()
3303 .find(|scope| &scope.variable == variable)
3304 else {
3305 continue;
3306 };
3307 for value in values {
3308 batch.scope(table_name, format!("{}:{value}", scope.prefix));
3309 }
3310 }
3311 }
3312
3313 fn record_row_scopes(
3315 &self,
3316 batch: &mut ChangeAccumulator,
3317 table_name: &str,
3318 row_id: &str,
3319 base: bool,
3320 ) -> bool {
3321 let Some(table) = self.schema.table(table_name) else {
3322 return false;
3323 };
3324 if table.scope_variables.is_empty() {
3325 return false;
3326 }
3327 let columns = table
3328 .scope_variables
3329 .iter()
3330 .map(|scope| quote_ident(&scope.column))
3331 .collect::<Vec<_>>()
3332 .join(", ");
3333 let full_table = if base {
3334 base_table(table_name)
3335 } else {
3336 visible_table(table_name)
3337 };
3338 let sql = format!(
3339 "SELECT {columns} FROM {full_table} WHERE CAST({} AS TEXT) = ?1 LIMIT 1",
3340 quote_ident(&table.primary_key)
3341 );
3342 let Ok(mut stmt) = self.conn.prepare(&sql) else {
3343 return false;
3344 };
3345 let values = stmt.query_row(rusqlite::params![row_id], |row| {
3346 let mut values = Vec::with_capacity(table.scope_variables.len());
3347 for index in 0..table.scope_variables.len() {
3348 values.push(row.get::<_, Option<String>>(index)?);
3349 }
3350 Ok(values)
3351 });
3352 let Ok(values) = values else {
3353 return false;
3354 };
3355 let mut recorded = false;
3356 for (scope, value) in table.scope_variables.iter().zip(values) {
3357 if let Some(value) = value {
3358 batch.scope(table_name, format!("{}:{value}", scope.prefix));
3359 recorded = true;
3360 }
3361 }
3362 recorded
3363 }
3364
3365 fn record_commit_changes(
3366 &self,
3367 batch: &mut ChangeAccumulator,
3368 tables: &[String],
3369 changes: &[ssp2::model::Change],
3370 ) {
3371 for change in changes {
3372 let Some(table_name) = tables.get(change.table_index as usize) else {
3373 continue;
3374 };
3375 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
3376 if let Some(table) = self.schema.table(table_name) {
3377 for (variable, value) in &change.scopes {
3378 if let Some(scope) = table
3379 .scope_variables
3380 .iter()
3381 .find(|scope| &scope.variable == variable)
3382 {
3383 batch.scope(table_name, format!("{}:{value}", scope.prefix));
3384 precise = true;
3385 }
3386 }
3387 }
3388 if !precise {
3389 batch.table(table_name);
3390 }
3391 }
3392 }
3393
3394 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
3395 if effective.is_empty() {
3396 return false;
3397 }
3398 let Some(table) = self.schema.table(table_name) else {
3399 return false;
3400 };
3401 let mut clauses = Vec::new();
3402 let mut params = Vec::new();
3403 for (variable, values) in effective {
3404 let Some(column) = table.scope_column(variable) else {
3405 return false;
3406 };
3407 if values.is_empty() {
3408 return false;
3409 }
3410 let placeholders = values
3411 .iter()
3412 .map(|value| {
3413 params.push(SqlValue::Text(value.clone()));
3414 "?"
3415 })
3416 .collect::<Vec<_>>()
3417 .join(", ");
3418 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
3419 }
3420 let sql = format!(
3421 "SELECT 1 FROM {} WHERE {} LIMIT 1",
3422 base_table(table_name),
3423 clauses.join(" AND ")
3424 );
3425 self.conn
3426 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
3427 .is_ok()
3428 }
3429
3430 pub fn upgrading(&self) -> bool {
3432 self.upgrading
3433 }
3434
3435 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
3441 let new_schema = parse_schema_json(schema_json)?;
3442 let marker: Option<i32> = self
3443 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
3444 .and_then(|v| v.parse().ok());
3445 self.schema = new_schema;
3446 if marker != Some(self.schema.version) {
3447 self.run_schema_reset()?;
3448 }
3449 self.prune_unknown_subscriptions()?;
3450 self.enqueue_startup_sync_if_needed();
3454 Ok(())
3455 }
3456
3457 fn enqueue_startup_sync_if_needed(&mut self) {
3458 let startup_work = !self.stopped
3459 && (!self.outbox.is_empty()
3460 || self.subs.iter().any(|sub| sub.state == SubState::Active));
3461 if startup_work {
3462 self.sync_needed = true;
3463 self.sync_intent_queue.push_back(SyncIntent::Interactive);
3464 }
3465 }
3466
3467 fn clear_satisfied_persisted_schema_floor(&mut self) {
3474 let satisfied = self
3475 .schema_floor
3476 .as_ref()
3477 .and_then(|floor| floor.required_schema_version)
3478 .is_some_and(|required| self.schema.version >= required);
3479 if !satisfied {
3480 return;
3481 }
3482 self.schema_floor = None;
3483 self.stopped = false;
3484 self.delete_meta(SCHEMA_FLOOR_KEY);
3485 }
3486
3487 fn run_schema_reset(&mut self) -> Result<(), String> {
3493 self.begin_observation("syncular_schema_reset")?;
3494 let mut batch = ChangeAccumulator::default();
3495 let result = self.run_schema_reset_observed(&mut batch, true);
3496 if let Err(error) = result {
3497 self.rollback_observation("syncular_schema_reset");
3498 return Err(error);
3499 }
3500 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
3501 self.rollback_observation("syncular_schema_reset");
3502 return Err(error);
3503 }
3504 Ok(())
3505 }
3506
3507 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
3508 let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
3509 self.begin_observation("syncular_log_epoch_reset")?;
3510 let mut batch = ChangeAccumulator::default();
3511 let result = self.run_schema_reset_observed(&mut batch, false);
3512 if let Err(error) = result {
3513 self.rollback_observation("syncular_log_epoch_reset");
3514 return Err(error);
3515 }
3516 self.set_meta(LOG_EPOCH_KEY, log_epoch);
3517 self.sync_needed = true;
3518 batch.status = true;
3519 if let Err(error) = self.finish_observation("syncular_log_epoch_reset", batch) {
3520 self.rollback_observation("syncular_log_epoch_reset");
3521 return Err(error);
3522 }
3523 self.sync_intent_queue.push_back(SyncIntent::Interactive);
3524 Ok(resets)
3525 }
3526
3527 fn run_schema_reset_observed(
3528 &mut self,
3529 batch: &mut ChangeAccumulator,
3530 drop_incompatible: bool,
3531 ) -> Result<(), String> {
3532 self.upgrading = true;
3533 batch.status = true;
3534 for table in &self.schema.tables {
3535 batch.table(&table.name);
3536 }
3537 for (base_key, unit, table) in self.load_registered_window_units() {
3538 batch.window(&base_key, &table, &unit);
3539 }
3540 self.insert_sql.borrow_mut().clear();
3542 self.overlay_dirty.set(true);
3543 let virtual_tables: Vec<String> = {
3551 let mut stmt = self
3552 .conn
3553 .prepare(
3554 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
3555 )
3556 .map_err(|e| e.to_string())?;
3557 let rows = stmt
3558 .query_map([], |row| row.get::<_, String>(0))
3559 .map_err(|e| e.to_string())?;
3560 rows.filter_map(Result::ok)
3561 .filter(|name| is_synced_table_name(name))
3562 .collect()
3563 };
3564 for name in virtual_tables {
3565 self.conn
3566 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
3567 .map_err(|e| e.to_string())?;
3568 }
3569 let existing: Vec<String> = {
3570 let mut stmt = self
3571 .conn
3572 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
3573 .map_err(|e| e.to_string())?;
3574 let rows = stmt
3575 .query_map([], |row| row.get::<_, String>(0))
3576 .map_err(|e| e.to_string())?;
3577 rows.filter_map(Result::ok)
3578 .filter(|name| is_synced_table_name(name))
3579 .collect()
3580 };
3581 for name in &existing {
3582 if let Some(table) = name.strip_prefix("_syncular_base_") {
3583 batch.table(table);
3584 } else if !name.starts_with("_syncular_") {
3585 batch.table(name);
3586 }
3587 }
3588 for name in existing {
3589 self.conn
3590 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
3591 .map_err(|e| e.to_string())?;
3592 }
3593 self.create_synced_tables()?;
3595 for sub in &mut self.subs {
3597 sub.cursor = -1;
3598 sub.bootstrap_state = None;
3599 sub.effective = None;
3600 sub.state = SubState::Active;
3601 sub.reason_code = None;
3602 sub.synced_once = false;
3603 }
3604 let subs = self.subs.clone();
3605 for sub in &subs {
3606 self.persist_sub(sub);
3607 }
3608 self.stopped = false;
3610 self.schema_floor = None;
3611 self.delete_meta(SCHEMA_FLOOR_KEY);
3612 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
3614 if drop_incompatible && self.drop_incompatible_outbox()? {
3618 batch.rejections = true;
3619 batch.status = true;
3620 batch.outcomes = true;
3621 }
3622 self.rebuild_overlay();
3624 Ok(())
3625 }
3626
3627 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
3631 let schema = &self.schema;
3632 let incompatible = self
3633 .outbox
3634 .iter()
3635 .filter(|commit| {
3636 commit.ops.iter().any(|op| {
3637 if !op.upsert {
3638 return false;
3639 }
3640 match schema.table(&op.table) {
3641 None => true,
3642 Some(table) => op.values.as_ref().is_some_and(|values| {
3643 values
3644 .keys()
3645 .any(|key| !table.columns.iter().any(|c| &c.name == key))
3646 }),
3647 }
3648 })
3649 })
3650 .cloned()
3651 .collect::<Vec<_>>();
3652 if incompatible.is_empty() {
3653 return Ok(false);
3654 }
3655 let mut rejections = Vec::new();
3656 for commit in &incompatible {
3657 let results = commit
3658 .ops
3659 .iter()
3660 .enumerate()
3661 .map(|(op_index, operation)| {
3662 let rejection = RejectionRecord {
3663 client_commit_id: commit.client_commit_id.clone(),
3664 op_index: op_index as i32,
3665 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
3666 message: "the persisted commit cannot encode under the current schema"
3667 .to_owned(),
3668 retryable: false,
3669 details: None,
3670 operation: Some(CommitOperation::from(operation)),
3671 };
3672 rejections.push(rejection.clone());
3673 CommitOperationOutcome::Error { rejection }
3674 })
3675 .collect::<Vec<_>>();
3676 self.persist_commit_outcome(
3677 &commit.client_commit_id,
3678 CommitOutcomeStatus::Rejected,
3679 &results,
3680 Some(&commit.ops),
3681 )?;
3682 self.delete_outbox_persisted(&commit.client_commit_id)?;
3683 }
3684 self.prune_commit_outcomes()?;
3685 let incompatible_ids = incompatible
3686 .iter()
3687 .map(|commit| commit.client_commit_id.as_str())
3688 .collect::<BTreeSet<_>>();
3689 self.outbox
3690 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
3691 self.rejections.extend(rejections);
3692 Ok(true)
3693 }
3694
3695 fn create_synced_tables(&self) -> Result<(), String> {
3698 for table in &self.schema.tables {
3699 for (full, index_prefix) in [
3703 (base_table(&table.name), "_syncular_base_"),
3704 (visible_table(&table.name), ""),
3705 ] {
3706 let mut cols: Vec<String> =
3707 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
3708 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
3709 let sql = format!(
3710 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
3711 cols.join(", "),
3712 quote_ident(&table.primary_key)
3713 );
3714 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
3715 for index in &table.indexes {
3719 let unique = if index.unique { "UNIQUE " } else { "" };
3720 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
3721 let cols_sql = index
3722 .columns
3723 .iter()
3724 .map(|c| quote_ident(c))
3725 .collect::<Vec<_>>()
3726 .join(", ");
3727 let index_sql = format!(
3728 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
3729 );
3730 self.conn
3731 .execute(&index_sql, [])
3732 .map_err(|e| e.to_string())?;
3733 }
3734 }
3735 }
3736 for table in &self.schema.tables {
3739 for index in &table.fts_indexes {
3740 self.create_fts_projection(table, index)?;
3741 }
3742 }
3743 Ok(())
3744 }
3745
3746 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
3747 let count: i64 = self
3748 .conn
3749 .query_row(
3750 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
3751 rusqlite::params![index.name],
3752 |row| row.get(0),
3753 )
3754 .map_err(|e| e.to_string())?;
3755 Ok(count > 0)
3756 }
3757
3758 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
3759 for suffix in ["ai", "ad", "au"] {
3760 self.conn
3761 .execute(
3762 &format!(
3763 "DROP TRIGGER IF EXISTS {}",
3764 quote_ident(&format!("{}_{suffix}", index.name))
3765 ),
3766 [],
3767 )
3768 .map_err(|e| e.to_string())?;
3769 }
3770 Ok(())
3771 }
3772
3773 fn create_fts_triggers(
3774 &self,
3775 table: &TableSchema,
3776 index: &FtsIndexSchema,
3777 ) -> Result<(), String> {
3778 let fts = quote_ident(&index.name);
3779 let source = visible_table(&table.name);
3780 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
3781 let pk = quote_ident(&table.primary_key);
3782 let projection_columns = std::iter::once(source_id.clone())
3783 .chain(index.columns.iter().map(|column| quote_ident(column)))
3784 .collect::<Vec<_>>()
3785 .join(", ");
3786 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
3787 .chain(
3788 index
3789 .columns
3790 .iter()
3791 .map(|column| format!("new.{}", quote_ident(column))),
3792 )
3793 .collect::<Vec<_>>()
3794 .join(", ");
3795 let delete_new = format!("DELETE FROM {fts} WHERE {source_id} = CAST(new.{pk} AS TEXT)");
3796 let delete_old = format!("DELETE FROM {fts} WHERE {source_id} = CAST(old.{pk} AS TEXT)");
3797 let insert_new = format!("INSERT INTO {fts} ({projection_columns}) VALUES ({new_values})");
3798 let bi = quote_ident(&format!("{}_bi", index.name));
3799 let ai = quote_ident(&format!("{}_ai", index.name));
3800 let ad = quote_ident(&format!("{}_ad", index.name));
3801 let au = quote_ident(&format!("{}_au", index.name));
3802 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
3803 let sql = format!(
3804 "DROP TRIGGER IF EXISTS {bi};
3805 DROP TRIGGER IF EXISTS {ai};
3806 DROP TRIGGER IF EXISTS {ad};
3807 DROP TRIGGER IF EXISTS {au};
3808 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
3809 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
3810 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
3811 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
3812 );
3813 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
3814 }
3815
3816 fn rebuild_fts_projection(
3817 &self,
3818 table: &TableSchema,
3819 index: &FtsIndexSchema,
3820 ) -> Result<(), String> {
3821 let fts = quote_ident(&index.name);
3822 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
3823 let pk = quote_ident(&table.primary_key);
3824 let indexed_columns = index
3825 .columns
3826 .iter()
3827 .map(|column| quote_ident(column))
3828 .collect::<Vec<_>>();
3829 let projection_columns = std::iter::once(source_id)
3830 .chain(indexed_columns.iter().cloned())
3831 .collect::<Vec<_>>()
3832 .join(", ");
3833 let sql = format!(
3834 "DELETE FROM {fts}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source};",
3835 columns = indexed_columns.join(", "),
3836 source = visible_table(&table.name),
3837 );
3838 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
3839 }
3840
3841 fn create_fts_projection(
3842 &self,
3843 table: &TableSchema,
3844 index: &FtsIndexSchema,
3845 ) -> Result<(), String> {
3846 let existed = self.fts_projection_exists(index)?;
3847 let tokenizer = index.tokenize.replace('\'', "''");
3848 let columns = index
3849 .columns
3850 .iter()
3851 .map(|column| quote_ident(column))
3852 .collect::<Vec<_>>()
3853 .join(", ");
3854 let sql = format!(
3855 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
3856 fts = quote_ident(&index.name),
3857 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
3858 );
3859 self.conn.execute(&sql, []).map_err(|error| {
3860 format!(
3861 "cannot create local FTS5 projection {:?}: {error}",
3862 index.name
3863 )
3864 })?;
3865 self.create_fts_triggers(table, index)?;
3866 if !existed {
3867 self.rebuild_fts_projection(table, index)?;
3868 }
3869 Ok(())
3870 }
3871
3872 fn persist_sub(&self, sub: &Subscription) {
3875 let state = serde_json::json!({
3876 "requested": scope_map_to_json(&sub.requested),
3877 "params": sub.params,
3878 "cursor": sub.cursor,
3879 "bootstrapState": sub.bootstrap_state,
3880 "status": sub.state.name(),
3881 "reasonCode": sub.reason_code,
3882 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
3883 "syncedOnce": sub.synced_once,
3884 });
3885 let _ = self.conn.execute(
3886 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
3887 rusqlite::params![sub.id, sub.table, state.to_string()],
3888 );
3889 }
3890
3891 fn persist_outbox_insert(&self, commit: &OutboxCommit) {
3892 let ops: Vec<Value> = commit
3893 .ops
3894 .iter()
3895 .map(|op| {
3896 serde_json::json!({
3897 "op": if op.upsert { "upsert" } else { "delete" },
3898 "table": op.table,
3899 "rowId": op.row_id,
3900 "baseVersion": op.base_version,
3901 "values": op.values.clone().map(Value::Object),
3902 "changedFields": op.changed_fields,
3903 })
3904 })
3905 .collect();
3906 let _ = self.conn.execute(
3907 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
3908 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
3909 );
3910 }
3911
3912 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
3913 self.conn
3914 .execute(
3915 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
3916 rusqlite::params![client_commit_id],
3917 )
3918 .map(|_| ())
3919 .map_err(|error| error.to_string())
3920 }
3921
3922 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
3923 match status {
3924 CommitOutcomeStatus::Applied => "applied",
3925 CommitOutcomeStatus::Cached => "cached",
3926 CommitOutcomeStatus::Conflict => "conflict",
3927 CommitOutcomeStatus::Rejected => "rejected",
3928 }
3929 }
3930
3931 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
3932 match resolution {
3933 CommitOutcomeResolution::Active => "active",
3934 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
3935 CommitOutcomeResolution::Superseded => "superseded",
3936 CommitOutcomeResolution::Dismissed => "dismissed",
3937 }
3938 }
3939
3940 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
3941 match value {
3942 "applied" => Ok(CommitOutcomeStatus::Applied),
3943 "cached" => Ok(CommitOutcomeStatus::Cached),
3944 "conflict" => Ok(CommitOutcomeStatus::Conflict),
3945 "rejected" => Ok(CommitOutcomeStatus::Rejected),
3946 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
3947 }
3948 }
3949
3950 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
3951 match value {
3952 "active" => Ok(CommitOutcomeResolution::Active),
3953 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
3954 "superseded" => Ok(CommitOutcomeResolution::Superseded),
3955 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
3956 _ => Err(format!(
3957 "invalid persisted commit outcome resolution {value:?}"
3958 )),
3959 }
3960 }
3961
3962 fn persist_commit_outcome(
3963 &self,
3964 client_commit_id: &str,
3965 status: CommitOutcomeStatus,
3966 results: &[CommitOperationOutcome],
3967 operations: Option<&[OutboxOp]>,
3968 ) -> Result<(), String> {
3969 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
3970 let operations_json = operations
3971 .map(|items| {
3972 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
3973 })
3974 .transpose()
3975 .map_err(|error| error.to_string())?;
3976 self.conn
3977 .execute(
3978 "INSERT INTO _syncular_commit_outcomes (
3979 client_commit_id, status, recorded_at_ms, results_json,
3980 operations_json, resolution
3981 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
3982 rusqlite::params![
3983 client_commit_id,
3984 Self::outcome_status_name(status),
3985 self.clock_now_ms(),
3986 results_json,
3987 operations_json
3988 ],
3989 )
3990 .map(|_| ())
3991 .map_err(|error| error.to_string())
3992 }
3993
3994 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
3995 let StoredCommitOutcomeRow {
3996 sequence,
3997 client_commit_id,
3998 status,
3999 recorded_at_ms,
4000 results_json,
4001 operations_json,
4002 resolution,
4003 resolved_at_ms,
4004 replacement_client_commit_id,
4005 } = row;
4006 Ok(CommitOutcome {
4007 sequence,
4008 client_commit_id,
4009 status: Self::parse_outcome_status(&status)?,
4010 recorded_at_ms,
4011 results: serde_json::from_str(&results_json)
4012 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
4013 operations: operations_json
4014 .map(|value| {
4015 serde_json::from_str(&value).map_err(|error| {
4016 format!("invalid persisted commit outcome operations: {error}")
4017 })
4018 })
4019 .transpose()?,
4020 resolution: Self::parse_outcome_resolution(&resolution)?,
4021 resolved_at_ms,
4022 replacement_client_commit_id,
4023 })
4024 }
4025
4026 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
4027 let row = self
4028 .conn
4029 .query_row(
4030 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
4031 resolution, resolved_at_ms, replacement_client_commit_id
4032 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
4033 rusqlite::params![client_commit_id],
4034 |row| {
4035 Ok(StoredCommitOutcomeRow {
4036 sequence: row.get(0)?,
4037 client_commit_id: row.get(1)?,
4038 status: row.get(2)?,
4039 recorded_at_ms: row.get(3)?,
4040 results_json: row.get(4)?,
4041 operations_json: row.get(5)?,
4042 resolution: row.get(6)?,
4043 resolved_at_ms: row.get(7)?,
4044 replacement_client_commit_id: row.get(8)?,
4045 })
4046 },
4047 )
4048 .optional()
4049 .map_err(|error| error.to_string())?;
4050 row.map(Self::outcome_from_row).transpose()
4051 }
4052
4053 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
4054 if query.limit == Some(0) {
4055 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
4056 }
4057 let mut sql = String::from(
4058 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
4059 resolution, resolved_at_ms, replacement_client_commit_id
4060 FROM _syncular_commit_outcomes",
4061 );
4062 if query.active_only {
4063 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
4064 }
4065 sql.push_str(" ORDER BY seq DESC");
4066 if let Some(limit) = query.limit {
4067 sql.push_str(&format!(" LIMIT {limit}"));
4068 }
4069 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
4070 let rows = stmt
4071 .query_map([], |row| {
4072 Ok(StoredCommitOutcomeRow {
4073 sequence: row.get(0)?,
4074 client_commit_id: row.get(1)?,
4075 status: row.get(2)?,
4076 recorded_at_ms: row.get(3)?,
4077 results_json: row.get(4)?,
4078 operations_json: row.get(5)?,
4079 resolution: row.get(6)?,
4080 resolved_at_ms: row.get(7)?,
4081 replacement_client_commit_id: row.get(8)?,
4082 })
4083 })
4084 .map_err(|error| error.to_string())?;
4085 let mut outcomes = Vec::new();
4086 for row in rows {
4087 outcomes.push(Self::outcome_from_row(
4088 row.map_err(|error| error.to_string())?,
4089 )?);
4090 }
4091 Ok(outcomes)
4092 }
4093
4094 fn prune_commit_outcomes(&self) -> Result<(), String> {
4095 #[cfg(test)]
4096 self.outcome_prune_count
4097 .set(self.outcome_prune_count.get() + 1);
4098 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
4099 let count = self
4100 .conn
4101 .query_row(
4102 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
4103 [],
4104 |row| row.get::<_, i64>(0),
4105 )
4106 .map_err(|error| error.to_string())? as usize;
4107 let excess = count.saturating_sub(max_entries);
4108 if excess == 0 {
4109 return Ok(());
4110 }
4111 let mut stmt = self
4112 .conn
4113 .prepare(
4114 "SELECT seq FROM _syncular_commit_outcomes
4115 WHERE status IN ('applied', 'cached') OR resolution != 'active'
4116 ORDER BY seq ASC LIMIT ?1",
4117 )
4118 .map_err(|error| error.to_string())?;
4119 let rows = stmt
4120 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
4121 .map_err(|error| error.to_string())?;
4122 let sequences = rows
4123 .collect::<Result<Vec<_>, _>>()
4124 .map_err(|error| error.to_string())?;
4125 drop(stmt);
4126 for sequence in sequences {
4127 self.conn
4128 .execute(
4129 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
4130 rusqlite::params![sequence],
4131 )
4132 .map_err(|error| error.to_string())?;
4133 }
4134 Ok(())
4135 }
4136
4137 pub fn subscribe(
4140 &mut self,
4141 id: String,
4142 table: String,
4143 scopes: Vec<(String, Vec<String>)>,
4144 params: Option<String>,
4145 ) -> Result<(), String> {
4146 if self.schema.table(&table).is_none() {
4147 return Err(format!("unknown table {table:?}"));
4148 }
4149 if let Some(existing) = self.subs.iter().find(|subscription| subscription.id == id) {
4150 let same_intent = existing.table == table
4151 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
4152 && existing.params == params;
4153 if same_intent {
4154 return Ok(());
4155 }
4156 return Err(
4157 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
4158 .to_owned(),
4159 );
4160 }
4161 let sub = Subscription {
4162 id: id.clone(),
4163 table,
4164 requested: scopes,
4165 params,
4166 cursor: -1,
4167 bootstrap_state: None,
4168 state: SubState::Active,
4169 reason_code: None,
4170 effective: None,
4171 synced_once: false,
4172 };
4173 self.persist_sub(&sub);
4174 self.subs.push(sub);
4175 Ok(())
4176 }
4177
4178 pub fn unsubscribe(&mut self, id: &str) {
4179 self.subs.retain(|s| s.id != id);
4180 let _ = self.conn.execute(
4181 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
4182 rusqlite::params![id],
4183 );
4184 }
4185
4186 pub fn set_window(
4194 &mut self,
4195 base: &WindowBase,
4196 units: &[String],
4197 ) -> Result<CommandEffects, String> {
4198 let table = self
4199 .schema
4200 .table(&base.table)
4201 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
4202 if table.scope_column(&base.variable).is_none() {
4203 return Err(format!(
4204 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
4205 base.table, base.variable
4206 ));
4207 }
4208 let base_key = window_base_key(base);
4209 let wanted: std::collections::HashSet<&String> = units.iter().collect();
4210 let live = self.load_window_units(&base_key);
4211 self.begin_observation("syncular_window")?;
4212 let mut batch = ChangeAccumulator::default();
4213 let mut changed = false;
4214
4215 for unit in units {
4217 if live.iter().any(|(u, _)| u == unit) {
4218 continue;
4219 }
4220 let sub_id = derive_sub_id(base, unit);
4221 self.delete_pending_evict(&sub_id);
4222 self.insert_window_unit(&base_key, unit, &sub_id);
4223 self.subscribe(
4224 sub_id,
4225 base.table.clone(),
4226 unit_scopes(base, unit),
4227 base.params.clone(),
4228 )?;
4229 batch.window(&base_key, &base.table, unit);
4230 changed = true;
4231 }
4232
4233 for (unit, sub_id) in live {
4235 if wanted.contains(&unit) {
4236 continue;
4237 }
4238 let effective = self
4239 .subs
4240 .iter()
4241 .find(|sub| sub.id == sub_id)
4242 .and_then(|sub| sub.effective.clone())
4243 .unwrap_or_else(|| unit_scopes(base, &unit));
4244 self.record_scope_map(&mut batch, &base.table, &effective);
4245 batch.window(&base_key, &base.table, &unit);
4246 self.evict_unit(&base_key, base, &unit, &sub_id);
4247 changed = true;
4248 }
4249 if let Err(error) = self.finish_observation("syncular_window", batch) {
4250 self.rollback_observation("syncular_window");
4251 return Err(error);
4252 }
4253 Ok(if changed {
4254 CommandEffects::interactive()
4255 } else {
4256 CommandEffects::none()
4257 })
4258 }
4259
4260 pub fn window_state(&self, base: &WindowBase) -> WindowState {
4265 let mut units = Vec::new();
4266 let mut pending = Vec::new();
4267 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
4268 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
4269 Some(sub) => {
4270 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
4271 }
4272 None => true,
4273 };
4274 if is_pending {
4275 pending.push(unit.clone());
4276 }
4277 units.push(unit);
4278 }
4279 WindowState { units, pending }
4280 }
4281
4282 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
4283 let mut stmt = match self
4284 .conn
4285 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
4286 {
4287 Ok(stmt) => stmt,
4288 Err(_) => return Vec::new(),
4289 };
4290 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
4291 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
4292 });
4293 match rows {
4294 Ok(rows) => rows.filter_map(Result::ok).collect(),
4295 Err(_) => Vec::new(),
4296 }
4297 }
4298
4299 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
4300 let mut stmt = match self.conn.prepare(
4301 "SELECT windows.base, windows.unit, subscriptions.tbl
4302 FROM _syncular_windows AS windows
4303 JOIN _syncular_subscriptions AS subscriptions
4304 ON subscriptions.id = windows.sub_id
4305 ORDER BY windows.base, windows.unit",
4306 ) {
4307 Ok(stmt) => stmt,
4308 Err(_) => return Vec::new(),
4309 };
4310 let rows = stmt.query_map([], |row| {
4311 Ok((
4312 row.get::<_, String>(0)?,
4313 row.get::<_, String>(1)?,
4314 row.get::<_, String>(2)?,
4315 ))
4316 });
4317 match rows {
4318 Ok(rows) => rows.filter_map(Result::ok).collect(),
4319 Err(_) => Vec::new(),
4320 }
4321 }
4322
4323 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
4324 self.conn
4325 .query_row(
4326 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
4327 rusqlite::params![sub_id],
4328 |row| Ok((row.get(0)?, row.get(1)?)),
4329 )
4330 .ok()
4331 }
4332
4333 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) {
4334 let _ = self.conn.execute(
4335 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
4336 rusqlite::params![base_key, unit, sub_id],
4337 );
4338 }
4339
4340 fn delete_window_unit(&self, base_key: &str, unit: &str) {
4341 let _ = self.conn.execute(
4342 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
4343 rusqlite::params![base_key, unit],
4344 );
4345 }
4346
4347 fn evict_unit(&mut self, base_key: &str, base: &WindowBase, unit: &str, sub_id: &str) {
4354 let effective = self
4355 .subs
4356 .iter()
4357 .find(|s| s.id == sub_id)
4358 .and_then(|s| s.effective.clone())
4359 .unwrap_or_else(|| unit_scopes(base, unit));
4360 let pinned = self.pinned_row_ids(&base.table);
4361 let deferred = self
4362 .evict_scope_rows(&base.table, &effective, &pinned)
4363 .unwrap_or(false);
4364 self.delete_window_unit(base_key, unit);
4365 self.unsubscribe(sub_id);
4366 if deferred {
4367 self.save_pending_evict(sub_id, &base.table, &effective);
4368 } else {
4369 self.delete_pending_evict(sub_id);
4370 }
4371 self.rebuild_overlay();
4372 }
4373
4374 fn evict_scope_rows(
4378 &mut self,
4379 table_name: &str,
4380 effective: &[(String, Vec<String>)],
4381 pinned: &std::collections::HashSet<String>,
4382 ) -> Result<bool, ()> {
4383 if effective.is_empty() {
4384 return Ok(false);
4385 }
4386 let table = self.schema.table(table_name).ok_or(())?.clone();
4387 let mut clauses = Vec::new();
4388 let mut params: Vec<SqlValue> = Vec::new();
4389 for (variable, values) in effective {
4390 let column = table.scope_column(variable).ok_or(())?;
4391 let placeholders: Vec<String> = values
4392 .iter()
4393 .map(|v| {
4394 params.push(SqlValue::Text(v.clone()));
4395 "?".to_owned()
4396 })
4397 .collect();
4398 clauses.push(format!(
4399 "{} IN ({})",
4400 quote_ident(column),
4401 placeholders.join(", ")
4402 ));
4403 }
4404 let mut sql = format!(
4405 "DELETE FROM {} WHERE {}",
4406 base_table(table_name),
4407 clauses.join(" AND ")
4408 );
4409 if !pinned.is_empty() {
4410 let pk = quote_ident(&table.primary_key);
4411 let holes: Vec<String> = pinned
4412 .iter()
4413 .map(|id| {
4414 params.push(SqlValue::Text(id.clone()));
4415 "?".to_owned()
4416 })
4417 .collect();
4418 sql.push_str(&format!(" AND {} NOT IN ({})", pk, holes.join(", ")));
4419 }
4420 self.overlay_dirty.set(true);
4421 self.conn
4422 .execute(&sql, rusqlite::params_from_iter(params))
4423 .map_err(|_| ())?;
4424 if pinned.is_empty() {
4425 return Ok(false);
4426 }
4427 let mut where_params: Vec<SqlValue> = Vec::new();
4430 let mut where_clauses = Vec::new();
4431 for (variable, values) in effective {
4432 let column = table.scope_column(variable).ok_or(())?;
4433 let placeholders: Vec<String> = values
4434 .iter()
4435 .map(|v| {
4436 where_params.push(SqlValue::Text(v.clone()));
4437 "?".to_owned()
4438 })
4439 .collect();
4440 where_clauses.push(format!(
4441 "{} IN ({})",
4442 quote_ident(column),
4443 placeholders.join(", ")
4444 ));
4445 }
4446 let pk = quote_ident(&table.primary_key);
4447 let select = format!(
4448 "SELECT {} FROM {} WHERE {}",
4449 pk,
4450 base_table(table_name),
4451 where_clauses.join(" AND ")
4452 );
4453 let mut stmt = self.conn.prepare(&select).map_err(|_| ())?;
4454 let survivors: Vec<String> = stmt
4455 .query_map(rusqlite::params_from_iter(where_params), |row| {
4456 row.get::<_, String>(0)
4457 })
4458 .map_err(|_| ())?
4459 .filter_map(Result::ok)
4460 .collect();
4461 Ok(survivors.iter().any(|id| pinned.contains(id)))
4462 }
4463
4464 fn drain_pending_evictions(&mut self) {
4468 let pending = self.load_pending_evictions();
4469 if pending.is_empty() {
4470 return;
4471 }
4472 for (sub_id, table_name, effective) in pending {
4473 if self.schema.table(&table_name).is_none() {
4474 self.delete_pending_evict(&sub_id);
4475 continue;
4476 }
4477 let pinned = self.pinned_row_ids(&table_name);
4478 let deferred = self
4479 .evict_scope_rows(&table_name, &effective, &pinned)
4480 .unwrap_or(false);
4481 if !deferred {
4482 self.delete_pending_evict(&sub_id);
4483 }
4484 }
4485 self.rebuild_overlay_if_dirty();
4486 }
4487
4488 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
4491 let mut pinned = std::collections::HashSet::new();
4492 for commit in &self.outbox {
4493 for op in &commit.ops {
4494 if op.table == table {
4495 pinned.insert(op.row_id.clone());
4496 }
4497 }
4498 }
4499 pinned
4500 }
4501
4502 fn save_pending_evict(&self, sub_id: &str, table: &str, effective: &[(String, Vec<String>)]) {
4503 let _ = self.conn.execute(
4504 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes)
4505 VALUES (?1, ?2, ?3)",
4506 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
4507 );
4508 }
4509
4510 fn delete_pending_evict(&self, sub_id: &str) {
4511 let _ = self.conn.execute(
4512 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
4513 rusqlite::params![sub_id],
4514 );
4515 }
4516
4517 fn load_pending_evictions(&self) -> Vec<PendingEvict> {
4518 let mut stmt = match self
4519 .conn
4520 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
4521 {
4522 Ok(stmt) => stmt,
4523 Err(_) => return Vec::new(),
4524 };
4525 let rows = stmt.query_map([], |row| {
4526 Ok((
4527 row.get::<_, String>(0)?,
4528 row.get::<_, String>(1)?,
4529 row.get::<_, String>(2)?,
4530 ))
4531 });
4532 let mut out = Vec::new();
4533 if let Ok(rows) = rows {
4534 for entry in rows.filter_map(Result::ok) {
4535 let (sub_id, table, json) = entry;
4536 if let Ok(value) = serde_json::from_str::<Value>(&json) {
4537 if let Ok(effective) = json_to_scope_map(&value) {
4538 out.push((sub_id, table, effective));
4539 }
4540 }
4541 }
4542 }
4543 out
4544 }
4545
4546 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
4548 if mutations.is_empty() {
4549 return Err("a commit must contain at least one operation (§6.1)".to_owned());
4550 }
4551 let mut ops = Vec::with_capacity(mutations.len());
4552 for mutation in mutations {
4553 match mutation {
4554 Mutation::Upsert {
4555 table,
4556 values,
4557 base_version,
4558 } => {
4559 let schema_table = self
4560 .schema
4561 .table(&table)
4562 .ok_or_else(|| format!("unknown table {table:?}"))?;
4563 let values = normalize_values_casing(schema_table, values)?;
4567 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
4568 encode_row_json(schema_table, &row_id, &values, &self.encryption)?;
4571 ops.push(OutboxOp {
4572 upsert: true,
4573 table,
4574 row_id,
4575 base_version,
4576 values: Some(values),
4577 changed_fields: None,
4578 });
4579 }
4580 Mutation::Delete {
4581 table,
4582 row_id,
4583 base_version,
4584 } => {
4585 if self.schema.table(&table).is_none() {
4586 return Err(format!("unknown table {table:?}"));
4587 }
4588 ops.push(OutboxOp {
4589 upsert: false,
4590 table,
4591 row_id,
4592 base_version,
4593 values: None,
4594 changed_fields: None,
4595 });
4596 }
4597 }
4598 }
4599 self.record_outbox_commit(ops)
4600 }
4601
4602 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
4603 let commit = OutboxCommit {
4604 client_commit_id: uuid::Uuid::new_v4().to_string(),
4605 ops,
4606 };
4607 self.begin_observation("syncular_mutation")?;
4608 let mut batch = ChangeAccumulator::default();
4609 for op in &commit.ops {
4610 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
4611 if let Some(values) = &op.values {
4612 if let Some(table) = self.schema.table(&op.table) {
4613 for scope in &table.scope_variables {
4614 if let Some(Value::String(value)) = values.get(&scope.column) {
4615 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
4616 precise = true;
4617 }
4618 }
4619 }
4620 }
4621 if !precise {
4622 batch.table(&op.table);
4623 }
4624 }
4625 self.persist_outbox_insert(&commit);
4626 let id = commit.client_commit_id.clone();
4627 self.outbox.push(commit);
4628 self.overlay_dirty.set(true);
4629 self.rebuild_overlay();
4630 batch.status = true;
4631 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
4632 self.rollback_observation("syncular_mutation");
4633 return Err(error);
4634 }
4635 Ok(id)
4636 }
4637
4638 pub fn patch(
4643 &mut self,
4644 table: &str,
4645 row_id: &str,
4646 partial: Map<String, Value>,
4647 base_version: Option<i64>,
4648 ) -> Result<String, String> {
4649 let schema_table = self
4650 .schema
4651 .table(table)
4652 .ok_or_else(|| format!("unknown table {table:?}"))?;
4653 let partial = normalize_values_casing(schema_table, partial)?;
4654 let mut values = self
4655 .read_rows(table)?
4656 .into_iter()
4657 .find(|row| row.row_id == row_id)
4658 .map(|row| row.values)
4659 .ok_or_else(|| {
4660 format!(
4661 "sync.invalid_request: table {table:?} has no local row with primary key {row_id:?} to patch"
4662 )
4663 })?;
4664 let mut changed_fields = partial.keys().cloned().collect::<Vec<_>>();
4665 changed_fields.sort();
4666 values.extend(partial);
4667 let row_id_from_values = render_row_id_json(values.get(&schema_table.primary_key))?;
4668 if row_id_from_values != row_id {
4669 return Err("sync.invalid_request: patch cannot change the primary key".to_owned());
4670 }
4671 encode_row_json(schema_table, row_id, &values, &self.encryption)?;
4672 self.record_outbox_commit(vec![OutboxOp {
4673 upsert: true,
4674 table: table.to_owned(),
4675 row_id: row_id.to_owned(),
4676 base_version,
4677 values: Some(values),
4678 changed_fields: Some(changed_fields),
4679 }])
4680 }
4681
4682 pub fn pending_commit_ids(&self) -> Vec<String> {
4683 self.outbox
4684 .iter()
4685 .map(|c| c.client_commit_id.clone())
4686 .collect()
4687 }
4688
4689 pub fn conflicts(&self) -> &[ConflictRecord] {
4690 &self.conflicts
4691 }
4692
4693 pub fn rejections(&self) -> &[RejectionRecord] {
4694 &self.rejections
4695 }
4696
4697 pub fn resolve_commit_outcome(
4698 &mut self,
4699 input: ResolveCommitOutcomeInput,
4700 ) -> Result<CommitOutcome, String> {
4701 let current = self
4702 .commit_outcome(&input.client_commit_id)?
4703 .ok_or_else(|| {
4704 format!(
4705 "sync.outcome_not_found: no durable outcome exists for {:?}",
4706 input.client_commit_id
4707 )
4708 })?;
4709 if current.resolution != CommitOutcomeResolution::Active {
4710 return Ok(current);
4711 }
4712 if input.resolution == CommitOutcomeResolution::Active {
4713 return Err("sync.invalid_request: resolution must leave active state".to_owned());
4714 }
4715 match input.resolution {
4716 CommitOutcomeResolution::Superseded => {
4717 let replacement = input
4718 .replacement_client_commit_id
4719 .as_deref()
4720 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
4721 .ok_or_else(|| {
4722 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
4723 .to_owned()
4724 })?;
4725 let _ = replacement;
4726 }
4727 _ if input.replacement_client_commit_id.is_some() => {
4728 return Err(
4729 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
4730 .to_owned(),
4731 );
4732 }
4733 _ => {}
4734 }
4735 let allowed = match current.status {
4736 CommitOutcomeStatus::Conflict => matches!(
4737 input.resolution,
4738 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
4739 ),
4740 CommitOutcomeStatus::Rejected => {
4741 input.resolution == CommitOutcomeResolution::Superseded
4742 }
4743 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
4744 input.resolution == CommitOutcomeResolution::Dismissed
4745 }
4746 };
4747 if !allowed {
4748 return Err(format!(
4749 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
4750 input.resolution, current.status
4751 ));
4752 }
4753
4754 self.begin_observation("syncular_outcome_resolution")?;
4755 let result = (|| {
4756 self.conn
4757 .execute(
4758 "UPDATE _syncular_commit_outcomes
4759 SET resolution = ?1, resolved_at_ms = ?2,
4760 replacement_client_commit_id = ?3
4761 WHERE client_commit_id = ?4 AND resolution = 'active'",
4762 rusqlite::params![
4763 Self::outcome_resolution_name(input.resolution),
4764 self.clock_now_ms(),
4765 input.replacement_client_commit_id,
4766 input.client_commit_id
4767 ],
4768 )
4769 .map_err(|error| error.to_string())?;
4770 let resolved = self
4771 .commit_outcome(¤t.client_commit_id)?
4772 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
4773 self.prune_commit_outcomes()?;
4774 Ok(resolved)
4775 })();
4776 let resolved = match result {
4777 Ok(outcome) => outcome,
4778 Err(error) => {
4779 self.rollback_observation("syncular_outcome_resolution");
4780 return Err(error);
4781 }
4782 };
4783 self.conflicts
4784 .retain(|record| record.client_commit_id != current.client_commit_id);
4785 self.rejections
4786 .retain(|record| record.client_commit_id != current.client_commit_id);
4787 let batch = ChangeAccumulator {
4788 conflicts: current.status == CommitOutcomeStatus::Conflict,
4789 rejections: current.status == CommitOutcomeStatus::Rejected,
4790 outcomes: true,
4791 ..ChangeAccumulator::default()
4792 };
4793 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
4794 self.rollback_observation("syncular_outcome_resolution");
4795 return Err(error);
4796 }
4797 Ok(resolved)
4798 }
4799
4800 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
4801 self.schema_floor.as_ref()
4802 }
4803
4804 pub fn lease_state(&self) -> Option<&LeaseState> {
4806 self.lease_state.as_ref()
4807 }
4808
4809 fn record_lease_error(&mut self, code: &str) {
4812 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
4813 return;
4814 }
4815 let mut next = self.lease_state.clone().unwrap_or_default();
4816 next.error_code = Some(code.to_owned());
4817 self.set_lease_state(Some(next));
4818 }
4819
4820 fn set_lease_state(&mut self, next: Option<LeaseState>) {
4821 if self.lease_state == next {
4822 return;
4823 }
4824 if self.begin_observation("syncular_lease").is_err() {
4825 return;
4826 }
4827 self.lease_state = next;
4828 if let Some(lease) = &self.lease_state {
4829 if let Ok(json) = serde_json::to_string(lease) {
4830 self.set_meta(LEASE_STATE_KEY, &json);
4831 }
4832 } else {
4833 self.delete_meta(LEASE_STATE_KEY);
4834 }
4835 let batch = ChangeAccumulator {
4836 status: true,
4837 ..ChangeAccumulator::default()
4838 };
4839 if self.finish_observation("syncular_lease", batch).is_err() {
4840 self.rollback_observation("syncular_lease");
4841 }
4842 }
4843
4844 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
4845 if self.schema_floor == next {
4846 return;
4847 }
4848 if self.begin_observation("syncular_schema_floor").is_err() {
4849 return;
4850 }
4851 self.schema_floor = next;
4852 self.stopped = self.schema_floor.is_some();
4853 if let Some(floor) = &self.schema_floor {
4854 if let Ok(json) = serde_json::to_string(floor) {
4855 self.set_meta(SCHEMA_FLOOR_KEY, &json);
4856 }
4857 } else {
4858 self.delete_meta(SCHEMA_FLOOR_KEY);
4859 }
4860 let batch = ChangeAccumulator {
4861 status: true,
4862 ..ChangeAccumulator::default()
4863 };
4864 if self
4865 .finish_observation("syncular_schema_floor", batch)
4866 .is_err()
4867 {
4868 self.rollback_observation("syncular_schema_floor");
4869 }
4870 }
4871
4872 fn set_upgrading(&mut self, value: bool) {
4873 if self.upgrading == value {
4874 return;
4875 }
4876 if self.begin_observation("syncular_upgrading").is_err() {
4877 return;
4878 }
4879 self.upgrading = value;
4880 let batch = ChangeAccumulator {
4881 status: true,
4882 ..ChangeAccumulator::default()
4883 };
4884 if self
4885 .finish_observation("syncular_upgrading", batch)
4886 .is_err()
4887 {
4888 self.rollback_observation("syncular_upgrading");
4889 }
4890 }
4891
4892 pub fn sync_needed(&self) -> bool {
4893 self.sync_needed
4894 }
4895
4896 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
4897 let sub = self.subs.iter().find(|s| s.id == id)?;
4898 Some(SubscriptionStateView {
4899 id: sub.id.clone(),
4900 table: sub.table.clone(),
4901 status: sub.state.name().to_owned(),
4902 cursor: sub.cursor,
4903 has_resume_token: sub.bootstrap_state.is_some(),
4904 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
4905 reason_code: sub.reason_code.clone(),
4906 })
4907 }
4908
4909 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
4910 let schema_table = self
4911 .schema
4912 .table(table)
4913 .ok_or_else(|| format!("unknown table {table:?}"))?;
4914 let sql = format!(
4915 "SELECT * FROM {} ORDER BY {} ASC",
4916 visible_table(table),
4917 quote_ident(&schema_table.primary_key)
4918 );
4919 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
4920 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
4921 let mut out = Vec::new();
4922 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
4923 let mut values = Map::new();
4924 for (i, column) in schema_table.columns.iter().enumerate() {
4925 let value = row.get_ref(i).map_err(|e| e.to_string())?;
4926 values.insert(column.name.clone(), sql_ref_to_json(column, value));
4927 }
4928 let version: i64 = row
4929 .get(schema_table.columns.len())
4930 .map_err(|e| e.to_string())?;
4931 let row_id = match values.get(&schema_table.primary_key) {
4932 Some(Value::String(s)) => s.clone(),
4933 Some(Value::Number(n)) => n.to_string(),
4934 Some(Value::Bool(b)) => b.to_string(),
4935 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
4936 };
4937 out.push(RowState {
4938 row_id,
4939 version,
4940 values,
4941 });
4942 }
4943 Ok(out)
4944 }
4945
4946 #[cfg(feature = "crdt-yjs")]
4962 fn crdt_column_bytes(
4963 &self,
4964 table: &str,
4965 row_id: &str,
4966 column: &str,
4967 ) -> Result<Option<Vec<u8>>, String> {
4968 let schema_table = self
4969 .schema
4970 .table(table)
4971 .ok_or_else(|| format!("unknown table {table:?}"))?;
4972 let col = schema_table
4973 .columns
4974 .iter()
4975 .find(|c| c.name == column)
4976 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
4977 if col.ty != ColumnType::Crdt {
4978 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
4979 }
4980 let sql = format!(
4981 "SELECT {} FROM {} WHERE CAST({} AS TEXT) = ?1",
4982 quote_ident(column),
4983 visible_table(table),
4984 quote_ident(&schema_table.primary_key)
4985 );
4986 let bytes: Option<Vec<u8>> = self
4987 .conn
4988 .query_row(&sql, rusqlite::params![row_id], |row| {
4989 row.get::<_, Option<Vec<u8>>>(0)
4990 })
4991 .map_err(|e| match e {
4992 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
4993 other => other.to_string(),
4994 })?;
4995 Ok(bytes)
4996 }
4997
4998 #[cfg(feature = "crdt-yjs")]
5002 pub fn crdt_text(
5003 &self,
5004 table: &str,
5005 row_id: &str,
5006 column: &str,
5007 name: &str,
5008 ) -> Result<String, String> {
5009 let bytes = self
5010 .crdt_column_bytes(table, row_id, column)?
5011 .unwrap_or_default();
5012 crate::crdt::text(&bytes, name)
5013 }
5014
5015 #[cfg(feature = "crdt-yjs")]
5019 pub fn crdt_insert_text(
5020 &mut self,
5021 table: &str,
5022 row_id: &str,
5023 column: &str,
5024 name: &str,
5025 index: u32,
5026 value: &str,
5027 ) -> Result<String, String> {
5028 let current = self
5029 .crdt_column_bytes(table, row_id, column)?
5030 .unwrap_or_default();
5031 let update = crate::crdt::insert_text(¤t, name, index, value)?;
5032 self.crdt_push_update(table, row_id, column, &update)
5033 }
5034
5035 #[cfg(feature = "crdt-yjs")]
5038 pub fn crdt_delete_text(
5039 &mut self,
5040 table: &str,
5041 row_id: &str,
5042 column: &str,
5043 name: &str,
5044 index: u32,
5045 len: u32,
5046 ) -> Result<String, String> {
5047 let current = self
5048 .crdt_column_bytes(table, row_id, column)?
5049 .unwrap_or_default();
5050 let update = crate::crdt::delete_text(¤t, name, index, len)?;
5051 self.crdt_push_update(table, row_id, column, &update)
5052 }
5053
5054 #[cfg(feature = "crdt-yjs")]
5059 pub fn crdt_apply_update(
5060 &mut self,
5061 table: &str,
5062 row_id: &str,
5063 column: &str,
5064 update: &[u8],
5065 ) -> Result<String, String> {
5066 let current = self
5067 .crdt_column_bytes(table, row_id, column)?
5068 .unwrap_or_default();
5069 let next = crate::crdt::apply_update(¤t, update)?;
5070 self.crdt_push_update(table, row_id, column, &next)
5071 }
5072
5073 #[cfg(feature = "crdt-yjs")]
5079 fn crdt_push_update(
5080 &mut self,
5081 table: &str,
5082 row_id: &str,
5083 column: &str,
5084 crdt_bytes: &[u8],
5085 ) -> Result<String, String> {
5086 let schema_table = self
5087 .schema
5088 .table(table)
5089 .ok_or_else(|| format!("unknown table {table:?}"))?
5090 .clone();
5091 let mut values: Map<String, Value> = self
5094 .read_rows(table)?
5095 .into_iter()
5096 .find(|r| r.row_id == row_id)
5097 .map(|r| r.values)
5098 .unwrap_or_else(|| {
5099 let mut map = Map::new();
5100 map.insert(
5101 schema_table.primary_key.clone(),
5102 Value::from(row_id.to_owned()),
5103 );
5104 map
5105 });
5106 let mut bytes_obj = Map::new();
5108 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
5109 values.insert(column.to_owned(), Value::Object(bytes_obj));
5110 self.mutate(vec![Mutation::Upsert {
5111 table: table.to_owned(),
5112 values,
5113 base_version: None,
5114 }])
5115 }
5116
5117 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
5132 query_connection(&self.conn, sql, params)
5133 }
5134
5135 pub fn query_snapshot(
5137 &mut self,
5138 sql: &str,
5139 params: &[QueryValue],
5140 coverage: &[WindowCoverage],
5141 ) -> Result<QuerySnapshot, String> {
5142 snapshot_connection(&self.conn, sql, params, coverage)
5143 }
5144
5145 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
5148 let log_epoch = self.get_meta(LOG_EPOCH_KEY);
5149 let mut frames = vec![Frame::ReqHeader {
5150 client_id: self.client_id.clone(),
5151 schema_version: self.schema.version,
5152 log_epoch: log_epoch.clone(),
5153 }];
5154 let mut pushed_ids = Vec::new();
5155 let mut ops_in_request = 0usize;
5156 let mut deferred_commits = 0usize;
5157 for (index, commit) in self
5158 .outbox
5159 .iter()
5160 .take(if log_epoch.is_some() {
5161 self.outbox.len()
5162 } else {
5163 0
5164 })
5165 .enumerate()
5166 {
5167 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
5172 deferred_commits = self.outbox.len() - index;
5173 break;
5174 }
5175 ops_in_request += commit.ops.len();
5176 let operations = commit
5177 .ops
5178 .iter()
5179 .map(|op| {
5180 let payload = op.values.as_ref().and_then(|values| {
5181 let table = self.schema.table(&op.table)?;
5182 encode_row_json(table, &op.row_id, values, &self.encryption).ok()
5186 });
5187 ssp2::model::Operation {
5188 table: op.table.clone(),
5189 row_id: op.row_id.clone(),
5190 op: if op.upsert { Op::Upsert } else { Op::Delete },
5191 base_version: op.base_version,
5192 payload,
5193 }
5194 })
5195 .collect();
5196 frames.push(Frame::PushCommit {
5197 client_commit_id: commit.client_commit_id.clone(),
5198 operations,
5199 });
5200 pushed_ids.push(commit.client_commit_id.clone());
5201 }
5202 let accept = self.limits.accept.unwrap_or(if url_capable {
5205 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
5206 } else {
5207 DEFAULT_ACCEPT
5208 });
5209 frames.push(Frame::PullHeader {
5210 limit_commits: self.limits.limit_commits.unwrap_or(0),
5211 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
5212 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
5213 accept,
5214 });
5215 let mut fresh = Vec::new();
5216 for sub in &self.subs {
5217 if sub.state != SubState::Active {
5218 continue;
5219 }
5220 let mut scopes = sub.requested.clone();
5221 sort_scope_map(&mut scopes);
5222 frames.push(Frame::Subscription {
5223 id: sub.id.clone(),
5224 table: sub.table.clone(),
5225 scopes,
5226 params: sub.params.clone().map(RawJson),
5227 cursor: sub.cursor,
5228 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
5229 });
5230 fresh.push((
5231 sub.id.clone(),
5232 sub.cursor < 0 && sub.bootstrap_state.is_none(),
5233 ));
5234 }
5235 let message = Message {
5236 wire_version: WIRE_VERSION,
5237 msg_kind: MsgKind::Request,
5238 frames,
5239 };
5240 (
5241 message,
5242 RequestMeta {
5243 pushed_ids,
5244 fresh,
5245 accept,
5246 deferred_commits,
5247 },
5248 )
5249 }
5250
5251 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
5254 let started_at_ms = self.clock_now_ms();
5255 let outcome = self.sync_inner(transport);
5256 let completed_at_ms = self.clock_now_ms();
5257 self.last_round = Some(match &outcome {
5258 SyncOutcome::Ok(report) => DiagnosticLastRound {
5259 status: "succeeded".to_owned(),
5260 started_at_ms,
5261 completed_at_ms,
5262 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
5263 counters: Some(DiagnosticRoundCounters {
5264 pushed: report.pushed,
5265 applied: report.applied.len(),
5266 rejected: report.rejected.len(),
5267 retryable: report.retryable.len(),
5268 conflicts: report.conflicts,
5269 commits_applied: report.commits_applied,
5270 segment_rows_applied: report.segment_rows_applied,
5271 bootstrapping: report.bootstrapping.len(),
5272 resets: report.resets.len(),
5273 revoked: report.revoked.len(),
5274 failed: report.failed.len(),
5275 deferred_commits: report.deferred_commits,
5276 }),
5277 error_code: None,
5278 },
5279 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
5280 status: "failed".to_owned(),
5281 started_at_ms,
5282 completed_at_ms,
5283 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
5284 counters: None,
5285 error_code: Some(Self::diagnostic_code(error_code)),
5286 },
5287 });
5288 outcome
5289 }
5290
5291 fn sync_inner(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
5292 if self.stopped {
5293 return SyncOutcome::Ok(SyncReport {
5296 schema_floor: self.schema_floor.clone(),
5297 ..SyncReport::default()
5298 });
5299 }
5300 self.set_sync_needed(false, false);
5303 if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
5306 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
5307 if Self::retryable_transport_code(&code) {
5308 self.schedule_background_retry();
5309 }
5310 return SyncOutcome::Failed {
5311 error_code: code,
5312 message,
5313 };
5314 }
5315 }
5316 let (message, meta) = self.build_request(transport.supports_url_fetch());
5317 let request_bytes = encode_message(&message);
5318 let round = if self.realtime_connected {
5322 transport.realtime_sync(&request_bytes)
5323 } else {
5324 transport.sync(&request_bytes)
5325 };
5326 let response_bytes = match round {
5327 Ok(bytes) => bytes,
5328 Err(TransportError { code, message }) => {
5329 self.record_lease_error(&code);
5332 if Self::retryable_transport_code(&code) {
5333 self.schedule_background_retry();
5334 }
5335 return SyncOutcome::Failed {
5336 error_code: code,
5337 message,
5338 };
5339 }
5340 };
5341 let response = match decode_message(&response_bytes) {
5342 Ok(message) => message,
5343 Err(error) => {
5344 return SyncOutcome::Failed {
5347 error_code: error.code.as_str().to_owned(),
5348 message: error.detail,
5349 };
5350 }
5351 };
5352 if response.msg_kind != MsgKind::Response {
5353 return SyncOutcome::Failed {
5354 error_code: "sync.invalid_request".to_owned(),
5355 message: "expected a response message".to_owned(),
5356 };
5357 }
5358 let mut outcome = self.process_response(transport, response, &meta);
5359 if let SyncOutcome::Ok(report) = &mut outcome {
5360 report.deferred_commits = meta.deferred_commits;
5361 }
5362 match &outcome {
5363 SyncOutcome::Ok(_) => self.reset_background_retry(),
5364 SyncOutcome::Failed { error_code, .. }
5365 if Self::retryable_transport_code(error_code) =>
5366 {
5367 self.schedule_background_retry();
5368 }
5369 SyncOutcome::Failed { .. } => {}
5370 }
5371 if meta.deferred_commits > 0 {
5372 self.set_sync_needed(true, true);
5375 }
5376 outcome
5377 }
5378
5379 pub fn sync_until_idle(
5380 &mut self,
5381 transport: &mut dyn Transport,
5382 max_rounds: Option<u32>,
5383 ) -> SyncOutcome {
5384 let rounds = max_rounds.unwrap_or(12).max(1);
5385 let mut aggregate = SyncReport::default();
5386 for _ in 0..rounds {
5387 match self.sync(transport) {
5388 SyncOutcome::Failed {
5389 error_code,
5390 message,
5391 } => {
5392 return SyncOutcome::Failed {
5393 error_code,
5394 message,
5395 };
5396 }
5397 SyncOutcome::Ok(report) => {
5398 aggregate.pushed += report.pushed;
5399 aggregate.applied.extend(report.applied.iter().cloned());
5400 aggregate.rejected.extend(report.rejected.iter().cloned());
5401 aggregate.retryable.extend(report.retryable.iter().cloned());
5402 aggregate.conflicts += report.conflicts;
5403 aggregate.commits_applied += report.commits_applied;
5404 aggregate.segment_rows_applied += report.segment_rows_applied;
5405 aggregate.bootstrapping = report.bootstrapping.clone();
5406 aggregate.resets.extend(report.resets.iter().cloned());
5407 aggregate.revoked.extend(report.revoked.iter().cloned());
5408 aggregate.failed.extend(report.failed.iter().cloned());
5409 aggregate.deferred_commits = report.deferred_commits;
5410 if report.schema_floor.is_some() {
5411 aggregate.schema_floor = report.schema_floor.clone();
5412 }
5413 let more = !report.bootstrapping.is_empty()
5419 || report.commits_applied > 0
5420 || report.segment_rows_applied > 0
5421 || !report.resets.is_empty()
5422 || self.sync_needed;
5423 if !more {
5424 break;
5425 }
5426 }
5427 }
5428 }
5429 SyncOutcome::Ok(aggregate)
5430 }
5431
5432 fn process_response(
5433 &mut self,
5434 transport: &mut dyn Transport,
5435 response: Message,
5436 meta: &RequestMeta,
5437 ) -> SyncOutcome {
5438 let mut report = SyncReport {
5439 pushed: meta.pushed_ids.len() as u32,
5440 ..SyncReport::default()
5441 };
5442 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
5443 HashMap::new();
5444 let pushed_ids = meta
5445 .pushed_ids
5446 .iter()
5447 .map(String::as_str)
5448 .collect::<HashSet<_>>();
5449 let mut last_final_push_result_id: Option<String> = None;
5450 for frame in &response.frames {
5451 match frame {
5452 Frame::PushResultDetails {
5453 client_commit_id,
5454 entries,
5455 } => {
5456 let details = rejection_details_by_commit
5457 .entry(client_commit_id.clone())
5458 .or_default();
5459 for entry in entries {
5460 let parsed = match RejectionDetails::parse(&entry.details.0) {
5461 Ok(value) => value,
5462 Err(message) => {
5463 return SyncOutcome::Failed {
5464 error_code: "sync.invalid_request".to_owned(),
5465 message,
5466 };
5467 }
5468 };
5469 details.insert(entry.op_index, parsed);
5470 }
5471 }
5472 Frame::PushResult {
5473 client_commit_id,
5474 status,
5475 results,
5476 ..
5477 } if pushed_ids.contains(client_commit_id.as_str())
5478 && Self::push_result_is_final(*status, results) =>
5479 {
5480 last_final_push_result_id = Some(client_commit_id.clone());
5481 }
5482 _ => {}
5483 }
5484 }
5485 let mut frames = response.frames.into_iter();
5486 if response.wire_version < 2 {
5487 return SyncOutcome::Failed {
5488 error_code: "client.invalid_host_response".to_owned(),
5489 message: "server response does not carry wire version 2 log-epoch state".to_owned(),
5490 };
5491 }
5492 match frames.next() {
5493 Some(Frame::RespHeader {
5494 required_schema_version,
5495 latest_schema_version,
5496 log_epoch,
5497 reset_required,
5498 }) => {
5499 let Some(log_epoch) = log_epoch else {
5500 return SyncOutcome::Failed {
5501 error_code: "client.invalid_host_response".to_owned(),
5502 message: "response header omits logEpoch".to_owned(),
5503 };
5504 };
5505 let Some(reset_required) = reset_required else {
5506 return SyncOutcome::Failed {
5507 error_code: "client.invalid_host_response".to_owned(),
5508 message: "response header omits resetRequired".to_owned(),
5509 };
5510 };
5511 if let Some(required) = required_schema_version {
5512 let floor = SchemaFloor {
5515 required_schema_version: Some(required),
5516 latest_schema_version,
5517 };
5518 self.set_schema_floor(Some(floor.clone()));
5519 report.schema_floor = Some(floor);
5520 return SyncOutcome::Ok(report);
5521 }
5522 if reset_required {
5523 if frames.next().is_some() {
5524 return SyncOutcome::Failed {
5525 error_code: "client.invalid_host_response".to_owned(),
5526 message: "log-epoch reset response contains body frames".to_owned(),
5527 };
5528 }
5529 match self.run_log_epoch_reset(&log_epoch) {
5530 Ok(resets) => {
5531 report.resets = resets;
5532 return SyncOutcome::Ok(report);
5533 }
5534 Err(message) => {
5535 return SyncOutcome::Failed {
5536 error_code: "sync.local_corrupt".to_owned(),
5537 message,
5538 };
5539 }
5540 }
5541 }
5542 if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
5543 return SyncOutcome::Failed {
5544 error_code: "client.invalid_host_response".to_owned(),
5545 message: "server changed logEpoch without requiring a reset".to_owned(),
5546 };
5547 }
5548 }
5549 _ => {
5550 return SyncOutcome::Failed {
5551 error_code: "sync.invalid_request".to_owned(),
5552 message: "response does not start with RESP_HEADER".to_owned(),
5553 };
5554 }
5555 }
5556
5557 let mut failure: Option<(String, String)> = None;
5558 while let Some(frame) = frames.next() {
5559 match frame {
5560 Frame::PushResult {
5561 client_commit_id,
5562 status,
5563 commit_seq: _,
5564 results,
5565 } => {
5566 let prune_outcomes =
5567 last_final_push_result_id.as_deref() == Some(&client_commit_id);
5568 self.handle_push_result(
5569 &client_commit_id,
5570 status,
5571 &results,
5572 rejection_details_by_commit.get(&client_commit_id),
5573 &mut report,
5574 prune_outcomes,
5575 );
5576 }
5577 Frame::PushResultDetails { .. } => {}
5578 Frame::SubStart {
5579 id,
5580 status,
5581 reason_code,
5582 effective_scopes,
5583 bootstrap: _,
5584 } => {
5585 let mut body = Vec::new();
5586 let mut sub_end: Option<(i64, Option<String>)> = None;
5587 for inner in frames.by_ref() {
5588 match inner {
5589 Frame::SubEnd {
5590 next_cursor,
5591 bootstrap_state,
5592 } => {
5593 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
5594 break;
5595 }
5596 Frame::Unknown { .. } => {}
5597 other => body.push(other),
5598 }
5599 }
5600 let Some((next_cursor, bootstrap_state)) = sub_end else {
5601 failure = Some((
5602 "sync.invalid_request".to_owned(),
5603 "subscription section without SUB_END".to_owned(),
5604 ));
5605 break;
5606 };
5607 if let Err(SectionError::Abort(code, message)) = self.process_section(
5608 transport,
5609 &id,
5610 status,
5611 &reason_code,
5612 effective_scopes,
5613 body,
5614 next_cursor,
5615 bootstrap_state,
5616 meta,
5617 &mut report,
5618 ) {
5619 failure = Some((code, message));
5620 break;
5621 }
5622 }
5623 Frame::Lease {
5624 lease_id,
5625 expires_at_ms,
5626 } => {
5627 self.set_lease_state(Some(LeaseState {
5630 lease_id: Some(lease_id),
5631 expires_at_ms: Some(expires_at_ms),
5632 error_code: None,
5633 }));
5634 }
5635 Frame::Error { code, message, .. } => {
5636 failure = Some((code, message));
5639 break;
5640 }
5641 Frame::Unknown { .. } => {}
5642 _ => {
5643 failure = Some((
5644 "sync.invalid_request".to_owned(),
5645 "unexpected frame in response".to_owned(),
5646 ));
5647 break;
5648 }
5649 }
5650 }
5651
5652 if self.overlay_dirty.get() {
5657 self.rebuild_overlay();
5658 }
5659 self.reconcile_blob_refcounts(false);
5666
5667 if let Some((error_code, message)) = failure {
5668 return SyncOutcome::Failed {
5669 error_code,
5670 message,
5671 };
5672 }
5673 self.drain_pending_evictions();
5676 self.ack_after_pull(transport);
5677 if self.upgrading && report.bootstrapping.is_empty() {
5680 self.set_upgrading(false);
5681 }
5682 SyncOutcome::Ok(report)
5683 }
5684
5685 fn handle_push_result(
5688 &mut self,
5689 client_commit_id: &str,
5690 status: PushStatus,
5691 results: &[OpResult],
5692 rejection_details: Option<&BTreeMap<i32, RejectionDetails>>,
5693 report: &mut SyncReport,
5694 prune_outcomes: bool,
5695 ) {
5696 let Some(index) = self
5697 .outbox
5698 .iter()
5699 .position(|c| c.client_commit_id == client_commit_id)
5700 else {
5701 return;
5702 };
5703 if self.begin_observation("syncular_push_result").is_err() {
5704 return;
5705 }
5706 let mut batch = ChangeAccumulator::default();
5707 let operations = self.outbox[index].ops.clone();
5708 match status {
5709 PushStatus::Applied | PushStatus::Cached => {
5710 let journal_results = results
5713 .iter()
5714 .map(|result| {
5715 let op_index = match result {
5716 OpResult::Applied { op_index }
5717 | OpResult::Conflict { op_index, .. }
5718 | OpResult::Error { op_index, .. } => *op_index,
5719 };
5720 CommitOperationOutcome::Applied { op_index }
5721 })
5722 .collect::<Vec<_>>();
5723 let outcome_status = if status == PushStatus::Applied {
5724 CommitOutcomeStatus::Applied
5725 } else {
5726 CommitOutcomeStatus::Cached
5727 };
5728 let persisted = self
5729 .persist_commit_outcome(
5730 client_commit_id,
5731 outcome_status,
5732 &journal_results,
5733 None,
5734 )
5735 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
5736 let persisted = if prune_outcomes {
5737 persisted.and_then(|()| self.prune_commit_outcomes())
5738 } else {
5739 persisted
5740 };
5741 if persisted.is_err() {
5742 self.rollback_observation("syncular_push_result");
5743 return;
5744 }
5745 report.applied.push(client_commit_id.to_owned());
5746 self.outbox.remove(index);
5747 self.overlay_dirty.set(true);
5748 batch.status = true;
5749 batch.outcomes = true;
5750 }
5751 PushStatus::Rejected => {
5752 if results.iter().any(|result| {
5753 matches!(
5754 result,
5755 OpResult::Error {
5756 code,
5757 retryable: true,
5758 ..
5759 } if code == "sync.idempotency_cache_miss"
5760 )
5761 }) {
5762 report.retryable.push(client_commit_id.to_owned());
5765 if self
5766 .finish_observation("syncular_push_result", batch)
5767 .is_err()
5768 {
5769 self.rollback_observation("syncular_push_result");
5770 }
5771 return;
5772 }
5773
5774 let mut journal_results = Vec::with_capacity(results.len());
5775 let mut conflicts = Vec::new();
5776 let mut rejections = Vec::new();
5777 for result in results {
5778 match result {
5779 OpResult::Applied { op_index } => {
5780 journal_results.push(CommitOperationOutcome::Applied {
5781 op_index: *op_index,
5782 });
5783 }
5784 OpResult::Conflict {
5785 op_index,
5786 code,
5787 message,
5788 server_version,
5789 server_row,
5790 } => {
5791 let operation = operations
5792 .get(*op_index as usize)
5793 .map(CommitOperation::from);
5794 let (table, row_id) = operation
5795 .as_ref()
5796 .map(|op| (op.table.clone(), op.row_id.clone()))
5797 .unwrap_or_default();
5798 let server_row_json = self
5799 .schema
5800 .table(&table)
5801 .and_then(|t| {
5802 decode_row_bytes(t, server_row, &self.encryption)
5803 .ok()
5804 .map(|row| (t, row))
5805 })
5806 .map(|(t, row)| {
5807 let mut map = Map::new();
5808 for (i, column) in t.columns.iter().enumerate() {
5809 map.insert(
5810 column.name.clone(),
5811 column_value_to_json(row.get(i).unwrap_or(&None)),
5812 );
5813 }
5814 map
5815 })
5816 .unwrap_or_default();
5817 let conflict = ConflictRecord {
5818 client_commit_id: client_commit_id.to_owned(),
5819 op_index: *op_index,
5820 table,
5821 row_id,
5822 code: code.clone(),
5823 message: message.clone(),
5824 server_version: *server_version,
5825 server_row: server_row_json,
5826 operation,
5827 };
5828 journal_results.push(CommitOperationOutcome::Conflict {
5829 conflict: conflict.clone(),
5830 });
5831 conflicts.push(conflict);
5832 }
5833 OpResult::Error {
5834 op_index,
5835 code,
5836 message,
5837 retryable,
5838 } => {
5839 let rejection = RejectionRecord {
5840 client_commit_id: client_commit_id.to_owned(),
5841 op_index: *op_index,
5842 code: code.clone(),
5843 message: message.clone(),
5844 retryable: *retryable,
5845 details: rejection_details
5846 .and_then(|details| details.get(op_index))
5847 .cloned(),
5848 operation: operations
5849 .get(*op_index as usize)
5850 .map(CommitOperation::from),
5851 };
5852 journal_results.push(CommitOperationOutcome::Error {
5853 rejection: rejection.clone(),
5854 });
5855 rejections.push(rejection);
5856 }
5857 }
5858 }
5859 let outcome_status = if conflicts.is_empty() {
5860 CommitOutcomeStatus::Rejected
5861 } else {
5862 CommitOutcomeStatus::Conflict
5863 };
5864 let persisted = self
5865 .persist_commit_outcome(
5866 client_commit_id,
5867 outcome_status,
5868 &journal_results,
5869 Some(&operations),
5870 )
5871 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
5872 let persisted = if prune_outcomes {
5873 persisted.and_then(|()| self.prune_commit_outcomes())
5874 } else {
5875 persisted
5876 };
5877 if persisted.is_err() {
5878 self.rollback_observation("syncular_push_result");
5879 return;
5880 }
5881 report.conflicts += conflicts.len() as u32;
5882 report.rejected.push(client_commit_id.to_owned());
5883 batch.conflicts = !conflicts.is_empty();
5884 batch.rejections = !rejections.is_empty();
5885 batch.status = true;
5886 batch.outcomes = true;
5887 self.conflicts.extend(conflicts);
5888 self.rejections.extend(rejections);
5889 self.outbox.remove(index);
5890 self.overlay_dirty.set(true);
5891 }
5892 }
5893 if batch.status {
5894 for operation in &operations {
5895 if !self.record_row_scopes(&mut batch, &operation.table, &operation.row_id, false) {
5896 batch.table(&operation.table);
5897 }
5898 }
5899 }
5900 if self
5901 .finish_observation("syncular_push_result", batch)
5902 .is_err()
5903 {
5904 self.rollback_observation("syncular_push_result");
5905 }
5906 }
5907
5908 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
5909 status != PushStatus::Rejected
5910 || !results.iter().any(|result| {
5911 matches!(
5912 result,
5913 OpResult::Error {
5914 code,
5915 retryable: true,
5916 ..
5917 } if code == "sync.idempotency_cache_miss"
5918 )
5919 })
5920 }
5921
5922 #[allow(clippy::too_many_arguments)]
5925 fn process_section(
5926 &mut self,
5927 transport: &mut dyn Transport,
5928 id: &str,
5929 status: SubStatus,
5930 reason_code: &str,
5931 effective_scopes: Vec<(String, Vec<String>)>,
5932 body: Vec<Frame>,
5933 next_cursor: i64,
5934 bootstrap_state: Option<String>,
5935 meta: &RequestMeta,
5936 report: &mut SyncReport,
5937 ) -> Result<(), SectionError> {
5938 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
5939 return Ok(()); };
5941 match status {
5942 SubStatus::Revoked => {
5943 self.begin_observation("syncular_revocation")
5944 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
5945 let mut batch = ChangeAccumulator::default();
5946 let registered = self.window_unit_by_sub_id(id);
5947 let (table, effective) = {
5949 let sub = &self.subs[sub_index];
5950 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
5951 };
5952 let purged = self.purge_scope_rows(&table, &effective);
5953 match purged {
5954 Ok(()) => {
5955 self.record_scope_map(&mut batch, &table, &effective);
5956 let sub = &mut self.subs[sub_index];
5957 sub.state = SubState::Revoked;
5958 sub.reason_code = Some(if reason_code.is_empty() {
5959 "sync.scope_revoked".to_owned()
5960 } else {
5961 reason_code.to_owned()
5962 });
5963 report.revoked.push(id.to_owned());
5964 let doomed_effective = effective;
5965 let sub_table = table;
5966 self.persist_sub(&self.subs[sub_index].clone());
5967 let dropped = self
5968 .drop_doomed_outbox(&sub_table, &doomed_effective)
5969 .map_err(|message| {
5970 SectionError::Abort("storage.failed".to_owned(), message)
5971 })?;
5972 if dropped {
5973 batch.status = true;
5974 batch.rejections = true;
5975 batch.outcomes = true;
5976 }
5977 self.reconcile_blob_refcounts(true);
5980 }
5981 Err(()) => {
5982 let sub = &mut self.subs[sub_index];
5985 sub.state = SubState::Failed;
5986 sub.reason_code = Some("sync.scope_revoked".to_owned());
5987 report.failed.push(id.to_owned());
5988 self.persist_sub(&self.subs[sub_index].clone());
5989 }
5990 }
5991 if let Some((base_key, unit)) = registered {
5992 batch.window(&base_key, &self.subs[sub_index].table, &unit);
5993 }
5994 self.rebuild_overlay_if_dirty();
5995 self.finish_observation("syncular_revocation", batch)
5996 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
5997 Ok(())
5998 }
5999 SubStatus::Reset => {
6000 self.begin_observation("syncular_reset")
6001 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
6002 let mut batch = ChangeAccumulator::default();
6003 let registered = self.window_unit_by_sub_id(id);
6004 let sub = &mut self.subs[sub_index];
6007 sub.cursor = -1;
6008 sub.bootstrap_state = None;
6009 report.resets.push(id.to_owned());
6010 self.persist_sub(&self.subs[sub_index].clone());
6011 if let Some((base_key, unit)) = registered {
6012 batch.window(&base_key, &self.subs[sub_index].table, &unit);
6013 }
6014 self.finish_observation("syncular_reset", batch)
6015 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
6016 Ok(())
6017 }
6018 SubStatus::Active => {
6019 let fresh = meta
6020 .fresh
6021 .iter()
6022 .find(|(fid, _)| fid == id)
6023 .map(|(_, f)| *f)
6024 .unwrap_or(false);
6025 let was_pending = self.subs[sub_index].cursor < 0
6026 || self.subs[sub_index].bootstrap_state.is_some();
6027 let registered = self.window_unit_by_sub_id(id);
6028 self.subs[sub_index].effective = Some(effective_scopes);
6030 self.begin_observation("syncular_section")
6031 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
6032 let mut batch = ChangeAccumulator::default();
6033 let outcome = self.apply_section_body(
6034 transport, sub_index, body, fresh, meta, report, &mut batch,
6035 );
6036 match outcome {
6037 Ok(()) => {
6038 let sub = &mut self.subs[sub_index];
6039 sub.cursor = next_cursor;
6042 sub.bootstrap_state = bootstrap_state;
6043 sub.synced_once = true;
6044 if sub.bootstrap_state.is_some() {
6045 report.bootstrapping.push(id.to_owned());
6046 }
6047 let completed =
6048 was_pending && sub.cursor >= 0 && sub.bootstrap_state.is_none();
6049 self.persist_sub(&self.subs[sub_index].clone());
6050 if completed {
6051 if let Some((base_key, unit)) = registered.clone() {
6052 batch.window(&base_key, &self.subs[sub_index].table, &unit);
6053 }
6054 }
6055 self.rebuild_overlay_if_dirty();
6056 self.finish_observation("syncular_section", batch)
6057 .map_err(|message| {
6058 SectionError::Abort("storage.failed".to_owned(), message)
6059 })?;
6060 Ok(())
6061 }
6062 Err(SectionError::FailClosed) => {
6063 self.rollback_observation("syncular_section");
6066 self.begin_observation("syncular_section_failure")
6067 .map_err(|message| {
6068 SectionError::Abort("storage.failed".to_owned(), message)
6069 })?;
6070 let mut failure_batch = ChangeAccumulator::default();
6071 let sub = &mut self.subs[sub_index];
6072 sub.state = SubState::Failed;
6073 sub.reason_code = Some("sync.scope_revoked".to_owned());
6074 report.failed.push(id.to_owned());
6075 self.persist_sub(&self.subs[sub_index].clone());
6076 if let Some((base_key, unit)) = registered {
6077 failure_batch.window(&base_key, &self.subs[sub_index].table, &unit);
6078 }
6079 self.finish_observation("syncular_section_failure", failure_batch)
6080 .map_err(|message| {
6081 SectionError::Abort("storage.failed".to_owned(), message)
6082 })?;
6083 Ok(())
6084 }
6085 Err(SectionError::Abort(code, message)) => {
6086 self.rollback_observation("syncular_section");
6089 Err(SectionError::Abort(code, message))
6090 }
6091 }
6092 }
6093 }
6094 }
6095
6096 #[allow(clippy::too_many_arguments)]
6100 fn apply_section_body(
6101 &mut self,
6102 transport: &mut dyn Transport,
6103 sub_index: usize,
6104 body: Vec<Frame>,
6105 fresh: bool,
6106 meta: &RequestMeta,
6107 report: &mut SyncReport,
6108 batch: &mut ChangeAccumulator,
6109 ) -> Result<(), SectionError> {
6110 let mut saw_segment = false;
6111 for frame in body {
6112 match frame {
6113 Frame::Commit {
6114 tables, changes, ..
6115 } => {
6116 self.record_commit_changes(batch, &tables, &changes);
6117 self.apply_commit_changes(&tables, &changes)
6118 .map_err(|(c, m)| SectionError::Abort(c, m))?;
6119 report.commits_applied += 1;
6120 }
6121 Frame::SegmentInline { payload } => {
6122 let segment = decode_rows_segment(&payload)
6123 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
6124 let first = !saw_segment;
6125 saw_segment = true;
6126 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
6127 let cleared =
6128 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
6129 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
6130 if applied > 0 || cleared {
6131 batch.table(&segment.table);
6132 }
6133 report.segment_rows_applied += applied;
6134 }
6135 Frame::SegmentRef {
6136 segment_id,
6137 media_type,
6138 table,
6139 row_count,
6140 as_of_commit_seq,
6141 scope_digest,
6142 row_cursor,
6143 next_row_cursor,
6144 url,
6145 url_expires_at_ms,
6146 ..
6147 } => {
6148 let advertised = match media_type {
6151 MediaType::Rows => {
6152 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
6153 || meta.accept & ACCEPT_INLINE_ROWS != 0
6154 }
6155 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
6156 };
6157 if !advertised {
6158 return Err(SectionError::Abort(
6159 "sync.invalid_request".to_owned(),
6160 format!(
6161 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
6162 media_type.name()
6163 ),
6164 ));
6165 }
6166 let bytes = if let Some(url) = url {
6167 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
6172 return Err(SectionError::Abort(
6173 "sync.invalid_request".to_owned(),
6174 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
6175 .to_owned(),
6176 ));
6177 }
6178 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
6180 return Err(SectionError::Abort(
6181 "sync.segment_expired".to_owned(),
6182 format!(
6183 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
6184 ),
6185 ));
6186 }
6187 transport
6188 .fetch_url(&url)
6189 .map_err(|e| SectionError::Abort(e.code, e.message))?
6190 } else {
6191 let requested_scopes_json =
6192 canonical_scope_json(&self.subs[sub_index].requested);
6193 transport
6194 .download_segment(&SegmentRequest {
6195 segment_id: segment_id.clone(),
6196 table,
6197 requested_scopes_json,
6198 })
6199 .map_err(|e| SectionError::Abort(e.code, e.message))?
6200 };
6201 let digest = Sha256::digest(&bytes);
6203 let expected = segment_id
6204 .strip_prefix("sha256:")
6205 .unwrap_or(segment_id.as_str());
6206 if bytes_to_hex(&digest) != expected {
6207 return Err(SectionError::Abort(
6208 "sync.invalid_request".to_owned(),
6209 "segment bytes do not match the content address (§5.1)".to_owned(),
6210 ));
6211 }
6212 if media_type == MediaType::Sqlite {
6213 if row_cursor.is_some() || next_row_cursor.is_some() {
6216 return Err(SectionError::Abort(
6217 "sync.invalid_request".to_owned(),
6218 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
6219 .to_owned(),
6220 ));
6221 }
6222 let first = !saw_segment;
6223 saw_segment = true;
6224 let sub_table = self.subs[sub_index].table.clone();
6225 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
6226 let cleared =
6227 fresh && first && self.scoped_rows_exist(&sub_table, &effective);
6228 let applied = self.apply_sqlite_segment(
6229 sub_index,
6230 &bytes,
6231 fresh && first,
6232 row_count,
6233 as_of_commit_seq,
6234 &scope_digest,
6235 )?;
6236 if applied > 0 || cleared {
6237 batch.table(&sub_table);
6238 }
6239 report.segment_rows_applied += applied;
6240 } else {
6241 let segment = decode_rows_segment(&bytes).map_err(|e| {
6242 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
6243 })?;
6244 let first = row_cursor.is_none();
6245 saw_segment = true;
6246 let effective = self.subs[sub_index].effective.clone().unwrap_or_default();
6247 let cleared =
6248 fresh && first && self.scoped_rows_exist(&segment.table, &effective);
6249 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
6250 if applied > 0 || cleared {
6251 batch.table(&segment.table);
6252 }
6253 report.segment_rows_applied += applied;
6254 }
6255 }
6256 Frame::Unknown { .. } => {}
6257 _ => {
6258 return Err(SectionError::Abort(
6259 "sync.invalid_request".to_owned(),
6260 "unexpected frame inside a subscription section".to_owned(),
6261 ));
6262 }
6263 }
6264 }
6265 Ok(())
6266 }
6267
6268 fn apply_commit_changes(
6269 &mut self,
6270 tables: &[String],
6271 changes: &[ssp2::model::Change],
6272 ) -> Result<(), (String, String)> {
6273 for change in changes {
6274 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
6275 (
6276 "sync.invalid_request".to_owned(),
6277 "change tableIndex out of range".to_owned(),
6278 )
6279 })?;
6280 let table = self.schema.table(table_name).ok_or_else(|| {
6281 (
6282 "sync.schema_mismatch".to_owned(),
6283 format!("change targets unknown table {table_name:?}"),
6284 )
6285 })?;
6286 match change.op {
6287 Op::Upsert => {
6288 let payload = change.row.as_ref().ok_or_else(|| {
6289 (
6290 "sync.invalid_request".to_owned(),
6291 "upsert change without row payload".to_owned(),
6292 )
6293 })?;
6294 let row = decode_row_bytes(table, payload, &self.encryption)
6296 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
6297 let version = change.row_version.unwrap_or(0);
6298 let table_name = table.name.clone();
6299 self.write_base_row(&table_name, &row, version)
6300 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
6301 }
6302 Op::Delete => {
6303 self.delete_base_row(table_name, &change.row_id)
6304 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
6305 }
6306 }
6307 }
6308 Ok(())
6309 }
6310
6311 fn apply_segment(
6316 &mut self,
6317 sub_index: usize,
6318 segment: &RowsSegment,
6319 first_fresh_page: bool,
6320 ) -> Result<u32, SectionError> {
6321 let (sub_table, effective) = {
6322 let sub = &self.subs[sub_index];
6323 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
6324 };
6325 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
6326 SectionError::Abort(
6327 "sync.schema_mismatch".to_owned(),
6328 format!("subscription table {sub_table:?} is not in the client schema"),
6329 )
6330 })?;
6331 let matches = segment.table == table.name
6336 && segment.schema_version == self.schema.version
6337 && segment.columns.len() == table.wire_columns.len()
6338 && segment
6339 .columns
6340 .iter()
6341 .zip(table.wire_columns.iter())
6342 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
6343 if !matches {
6344 return Err(SectionError::Abort(
6345 "sync.schema_mismatch".to_owned(),
6346 "segment column table does not match the generated schema (§5.2)".to_owned(),
6347 ));
6348 }
6349 if first_fresh_page {
6350 self.purge_scope_rows(&table.name, &effective)
6354 .map_err(|()| SectionError::FailClosed)?;
6355 }
6356 let mut applied = 0u32;
6357 for block in &segment.blocks {
6358 for row in block {
6359 let decrypted;
6364 let values = if table.has_encrypted_columns() {
6365 let mut values = row.values.clone();
6366 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
6367 .map_err(|m| SectionError::Abort("client.decrypt_failed".to_owned(), m))?;
6368 decrypted = values;
6369 &decrypted
6370 } else {
6371 &row.values
6372 };
6373 self.write_base_row(&table.name, values, row.server_version)
6376 .map_err(|m| SectionError::Abort("sync.invalid_request".to_owned(), m))?;
6377 applied += 1;
6378 }
6379 }
6380 Ok(applied)
6381 }
6382
6383 fn apply_sqlite_segment(
6391 &mut self,
6392 sub_index: usize,
6393 bytes: &[u8],
6394 first_fresh_page: bool,
6395 row_count: i64,
6396 as_of_commit_seq: i64,
6397 scope_digest: &str,
6398 ) -> Result<u32, SectionError> {
6399 let invalid = |detail: &str| {
6400 SectionError::Abort(
6401 "sync.invalid_request".to_owned(),
6402 format!("sqlite segment rejected: {detail} (§5.3)"),
6403 )
6404 };
6405 let (sub_table, effective) = {
6406 let sub = &self.subs[sub_index];
6407 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
6408 };
6409 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
6410 SectionError::Abort(
6411 "sync.schema_mismatch".to_owned(),
6412 format!("subscription table {sub_table:?} is not in the client schema"),
6413 )
6414 })?;
6415
6416 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
6417 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
6418 let img = match rusqlite::Connection::open_with_flags(
6419 &path,
6420 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
6421 ) {
6422 Ok(conn) => conn,
6423 Err(_) => {
6424 let _ = std::fs::remove_file(&path);
6425 return Err(invalid("bytes do not open as a SQLite database"));
6426 }
6427 };
6428 let outcome = self.apply_sqlite_image(
6429 &img,
6430 &table,
6431 first_fresh_page,
6432 &effective,
6433 row_count,
6434 as_of_commit_seq,
6435 scope_digest,
6436 );
6437 drop(img);
6438 let _ = std::fs::remove_file(&path);
6439 outcome
6440 }
6441
6442 #[allow(clippy::too_many_arguments)]
6443 fn apply_sqlite_image(
6444 &mut self,
6445 img: &rusqlite::Connection,
6446 table: &crate::schema::TableSchema,
6447 first_fresh_page: bool,
6448 effective: &[(String, Vec<String>)],
6449 row_count: i64,
6450 as_of_commit_seq: i64,
6451 scope_digest: &str,
6452 ) -> Result<u32, SectionError> {
6453 let invalid = |detail: String| {
6454 SectionError::Abort(
6455 "sync.invalid_request".to_owned(),
6456 format!("sqlite segment rejected: {detail} (§5.3)"),
6457 )
6458 };
6459
6460 type MetaRow = (i64, String, i64, i64, String, i64, i64);
6463 let meta: MetaRow = img
6464 .query_row(
6465 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
6466 \"scopeDigest\", \"rowCount\",
6467 (SELECT count(*) FROM _syncular_segment)
6468 FROM _syncular_segment",
6469 [],
6470 |row| {
6471 Ok((
6472 row.get(0)?,
6473 row.get(1)?,
6474 row.get(2)?,
6475 row.get(3)?,
6476 row.get(4)?,
6477 row.get(5)?,
6478 row.get(6)?,
6479 ))
6480 },
6481 )
6482 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
6483 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
6484 if meta_count != 1 {
6485 return Err(invalid(format!(
6486 "_syncular_segment must contain exactly one row, found {meta_count}"
6487 )));
6488 }
6489 if format != 1 {
6490 return Err(invalid(format!("format {format}")));
6491 }
6492 if meta_table != table.name {
6493 return Err(invalid(format!("image table {meta_table:?}")));
6494 }
6495 if schema_version != i64::from(self.schema.version) {
6496 return Err(invalid(format!("schemaVersion {schema_version}")));
6497 }
6498 if pin != as_of_commit_seq {
6499 return Err(invalid(format!("asOfCommitSeq {pin}")));
6500 }
6501 if digest != scope_digest {
6502 return Err(invalid("scopeDigest mismatch".to_owned()));
6503 }
6504 if meta_rows != row_count {
6505 return Err(invalid(format!("rowCount {meta_rows}")));
6506 }
6507
6508 let mut names: Vec<String> = Vec::new();
6510 {
6511 let mut stmt = img
6512 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
6513 .map_err(|_| invalid("image data table missing".to_owned()))?;
6514 let mut rows = stmt
6515 .query([])
6516 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
6517 while let Some(row) = rows
6518 .next()
6519 .map_err(|_| invalid("image data table unreadable".to_owned()))?
6520 {
6521 names.push(
6522 row.get::<_, String>(1)
6523 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
6524 );
6525 }
6526 }
6527 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
6528 expected.push("_syncular_version");
6529 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
6530 return Err(SectionError::Abort(
6531 "sync.schema_mismatch".to_owned(),
6532 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
6533 ));
6534 }
6535
6536 if first_fresh_page {
6539 self.purge_scope_rows(&table.name, effective)
6540 .map_err(|()| SectionError::FailClosed)?;
6541 }
6542 self.overlay_dirty.set(true);
6548 let bulk_indexes: Vec<&crate::schema::IndexSchema> = if first_fresh_page {
6556 table.indexes.iter().filter(|i| !i.unique).collect()
6557 } else {
6558 Vec::new()
6559 };
6560 for index in &bulk_indexes {
6561 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
6562 self.conn
6563 .execute(&format!("DROP INDEX IF EXISTS {index_name}"), [])
6564 .map_err(|e| invalid(e.to_string()))?;
6565 }
6566 let insert = self.insert_row_sql(&base_table(&table.name), table);
6567 let applied = {
6568 let mut ins = self
6569 .conn
6570 .prepare_cached(&insert)
6571 .map_err(|e| invalid(e.to_string()))?;
6572 let column_list: Vec<String> = names.iter().map(|n| quote_ident(n)).collect();
6573 let mut stmt = img
6574 .prepare(&format!(
6575 "SELECT {} FROM {}",
6576 column_list.join(", "),
6577 quote_ident(&table.name)
6578 ))
6579 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
6580 let mut rows = stmt
6581 .query([])
6582 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
6583 let version_index = table.columns.len();
6584 let mut applied = 0u32;
6585 while let Some(row) = rows
6586 .next()
6587 .map_err(|_| invalid("image row unreadable".to_owned()))?
6588 {
6589 for (i, column) in table.columns.iter().enumerate() {
6590 let cell = row
6591 .get_ref(i)
6592 .map_err(|_| invalid("image row unreadable".to_owned()))?;
6593 let param = image_cell_param(column, cell).map_err(&invalid)?;
6594 ins.raw_bind_parameter(i + 1, param)
6595 .map_err(|e| invalid(e.to_string()))?;
6596 }
6597 let version: i64 = row
6598 .get(version_index)
6599 .map_err(|_| invalid("image row unreadable".to_owned()))?;
6600 if version < 1 {
6601 return Err(invalid(format!(
6602 "row _syncular_version must be >= 1, got {version}"
6603 )));
6604 }
6605 ins.raw_bind_parameter(version_index + 1, version)
6606 .map_err(|e| invalid(e.to_string()))?;
6607 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
6608 applied += 1;
6609 }
6610 applied
6611 };
6612 for index in &bulk_indexes {
6613 let index_name = quote_ident(&format!("_syncular_base_{}", index.name));
6614 let cols_sql = index
6615 .columns
6616 .iter()
6617 .map(|c| quote_ident(c))
6618 .collect::<Vec<_>>()
6619 .join(", ");
6620 self.conn
6621 .execute(
6622 &format!(
6623 "CREATE INDEX IF NOT EXISTS {index_name} ON {} ({cols_sql})",
6624 base_table(&table.name)
6625 ),
6626 [],
6627 )
6628 .map_err(|e| invalid(e.to_string()))?;
6629 }
6630 if i64::from(applied) != row_count {
6631 return Err(invalid(format!(
6632 "image holds {applied} rows, descriptor says {row_count}"
6633 )));
6634 }
6635 Ok(applied)
6636 }
6637
6638 fn compile_local_data_purge(
6641 &self,
6642 input: &LocalDataPurgeInput,
6643 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
6644 let invalid = |message: String| format!("sync.invalid_request: {message}");
6645 if input.purge_id.is_empty()
6646 || input.purge_id.len() > 128
6647 || !is_local_operation_code_like(&input.purge_id)
6648 {
6649 return Err(invalid(
6650 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
6651 ));
6652 }
6653 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
6654 return Err(invalid(format!(
6655 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
6656 )));
6657 }
6658
6659 let mut deduplicated: BTreeMap<
6660 String,
6661 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
6662 > = BTreeMap::new();
6663 for target in &input.targets {
6664 let table = self.schema.table(&target.table).ok_or_else(|| {
6665 invalid(format!(
6666 "local purge names unknown table {:?}",
6667 target.table
6668 ))
6669 })?;
6670 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
6671 return Err(invalid(format!(
6672 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
6673 target.table
6674 )));
6675 }
6676 let mut selectors = Vec::with_capacity(target.selectors.len());
6677 let mut canonical_selectors = BTreeMap::new();
6678 for (column_name, raw_values) in &target.selectors {
6679 let Some((column_index, column)) = table
6680 .columns
6681 .iter()
6682 .enumerate()
6683 .find(|(_, column)| column.name == *column_name)
6684 else {
6685 return Err(invalid(format!(
6686 "local purge target {:?} names unknown column {:?}",
6687 target.table, column_name
6688 )));
6689 };
6690 let encrypted = table
6691 .encrypted_columns
6692 .iter()
6693 .any(|candidate| candidate.index == column_index);
6694 if column.ty != ColumnType::String || encrypted {
6695 return Err(invalid(format!(
6696 "local purge selector {:?}.{:?} must be a plaintext string column",
6697 target.table, column_name
6698 )));
6699 }
6700 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
6701 return Err(invalid(format!(
6702 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
6703 target.table, column_name
6704 )));
6705 }
6706 let mut values = raw_values.clone();
6707 values.sort();
6708 values.dedup();
6709 if values.iter().any(|value| {
6710 value.is_empty()
6711 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
6712 || !is_local_operation_code_like(value)
6713 }) {
6714 return Err(invalid(format!(
6715 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
6716 )));
6717 }
6718 selectors.push((column_name.clone(), values.clone()));
6719 canonical_selectors.insert(column_name.clone(), values);
6720 }
6721 let canonical = LocalDataPurgeTarget {
6722 table: target.table.clone(),
6723 selectors: canonical_selectors,
6724 };
6725 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
6726 deduplicated.insert(
6727 key,
6728 (
6729 CompiledLocalDataPurgeTarget {
6730 table: target.table.clone(),
6731 selectors,
6732 },
6733 canonical,
6734 ),
6735 );
6736 }
6737 let targets = deduplicated
6738 .values()
6739 .map(|(compiled, _)| compiled.clone())
6740 .collect::<Vec<_>>();
6741 let canonical = deduplicated
6742 .values()
6743 .map(|(_, target)| target.clone())
6744 .collect::<Vec<_>>();
6745 let canonical_plan =
6746 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
6747 Ok((targets, canonical_plan))
6748 }
6749
6750 fn local_purge_base_row_ids(
6751 &self,
6752 targets: &[CompiledLocalDataPurgeTarget],
6753 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
6754 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
6755 for target in targets {
6756 let table = self
6757 .schema
6758 .table(&target.table)
6759 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
6760 let mut clauses = Vec::with_capacity(target.selectors.len());
6761 let mut params = Vec::new();
6762 for (column, values) in &target.selectors {
6763 clauses.push(format!(
6764 "{} IN ({})",
6765 quote_ident(column),
6766 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
6767 ));
6768 params.extend(values.iter().cloned().map(SqlValue::Text));
6769 }
6770 let sql = format!(
6771 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
6772 quote_ident(&table.primary_key),
6773 base_table(&target.table),
6774 clauses.join(" AND ")
6775 );
6776 let mut statement = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
6777 let rows = statement
6778 .query_map(rusqlite::params_from_iter(params), |row| {
6779 row.get::<_, String>(0)
6780 })
6781 .map_err(|error| error.to_string())?;
6782 let ids = by_table.entry(target.table.clone()).or_default();
6783 for row in rows {
6784 ids.insert(row.map_err(|error| error.to_string())?);
6785 }
6786 }
6787 Ok(by_table)
6788 }
6789
6790 fn local_purge_values_match(
6791 target: &CompiledLocalDataPurgeTarget,
6792 values: &Map<String, Value>,
6793 ) -> bool {
6794 target.selectors.iter().all(|(column, allowed)| {
6795 values
6796 .get(column)
6797 .and_then(Value::as_str)
6798 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
6799 })
6800 }
6801
6802 pub fn purge_local_data(
6806 &mut self,
6807 input: &LocalDataPurgeInput,
6808 ) -> Result<LocalDataPurgeResult, String> {
6809 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
6810 let meta_key = format!("localPurge:{}", input.purge_id);
6811 let applied_plan = self
6812 .conn
6813 .query_row(
6814 "SELECT value FROM _syncular_meta WHERE key = ?1",
6815 rusqlite::params![meta_key],
6816 |row| row.get::<_, String>(0),
6817 )
6818 .optional()
6819 .map_err(|error| error.to_string())?;
6820 if let Some(applied_plan) = applied_plan {
6821 if applied_plan != canonical_plan {
6822 return Err(format!(
6823 "sync.invalid_request: local purge id {:?} was already used with a different plan",
6824 input.purge_id
6825 ));
6826 }
6827 return Ok(LocalDataPurgeResult {
6828 already_applied: true,
6829 purged_rows: 0,
6830 dropped_commits: 0,
6831 });
6832 }
6833
6834 let prior_outbox = self.outbox.clone();
6835 let prior_rejection_count = self.rejections.len();
6836 let prior_overlay_dirty = self.overlay_dirty.get();
6837 self.begin_observation("syncular_local_purge")?;
6838 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
6839 let row_ids = self.local_purge_base_row_ids(&targets)?;
6840 let doomed = self
6841 .outbox
6842 .iter()
6843 .filter(|commit| {
6844 commit.ops.iter().any(|operation| {
6845 let matching_targets = targets
6846 .iter()
6847 .filter(|target| target.table == operation.table)
6848 .collect::<Vec<_>>();
6849 if matching_targets.is_empty() {
6850 return false;
6851 }
6852 if row_ids
6853 .get(&operation.table)
6854 .is_some_and(|ids| ids.contains(&operation.row_id))
6855 {
6856 return true;
6857 }
6858 operation.values.as_ref().is_some_and(|values| {
6859 matching_targets
6860 .iter()
6861 .any(|target| Self::local_purge_values_match(target, values))
6862 })
6863 })
6864 })
6865 .cloned()
6866 .collect::<Vec<_>>();
6867 let doomed_ids = doomed
6868 .iter()
6869 .map(|commit| commit.client_commit_id.clone())
6870 .collect::<BTreeSet<_>>();
6871 let mut batch = ChangeAccumulator::default();
6872 let mut rejections = Vec::new();
6873 for commit in &doomed {
6874 for operation in &commit.ops {
6875 batch.table(&operation.table);
6876 }
6877 let results = commit
6878 .ops
6879 .iter()
6880 .enumerate()
6881 .map(|(op_index, operation)| {
6882 let rejection = RejectionRecord {
6883 client_commit_id: commit.client_commit_id.clone(),
6884 op_index: op_index as i32,
6885 code: "client.local_data_purged".to_owned(),
6886 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
6887 retryable: false,
6888 details: None,
6889 operation: Some(CommitOperation::from(operation)),
6890 };
6891 rejections.push(rejection.clone());
6892 CommitOperationOutcome::Error { rejection }
6893 })
6894 .collect::<Vec<_>>();
6895 self.persist_commit_outcome(
6896 &commit.client_commit_id,
6897 CommitOutcomeStatus::Rejected,
6898 &results,
6899 Some(&commit.ops),
6900 )?;
6901 self.delete_outbox_persisted(&commit.client_commit_id)?;
6902 }
6903 if !doomed.is_empty() {
6904 self.outbox
6905 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
6906 self.rejections.extend(rejections);
6907 self.prune_commit_outcomes()?;
6908 self.overlay_dirty.set(true);
6909 batch.status = true;
6910 batch.rejections = true;
6911 batch.outcomes = true;
6912 }
6913
6914 let mut purged_rows = 0usize;
6915 for (table, ids) in &row_ids {
6916 if ids.is_empty() {
6917 continue;
6918 }
6919 batch.table(table);
6920 for row_id in ids {
6921 self.delete_base_row(table, row_id)?;
6922 purged_rows += 1;
6923 }
6924 }
6925 self.rebuild_overlay_if_dirty();
6926 self.reconcile_blob_refcounts(true);
6927 self.conn
6928 .execute(
6929 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
6930 rusqlite::params![meta_key, canonical_plan],
6931 )
6932 .map_err(|error| error.to_string())?;
6933 Ok((
6934 batch,
6935 LocalDataPurgeResult {
6936 already_applied: false,
6937 purged_rows,
6938 dropped_commits: doomed.len(),
6939 },
6940 ))
6941 })();
6942 let (batch, result) = match applied {
6943 Ok(value) => value,
6944 Err(error) => {
6945 self.rollback_observation("syncular_local_purge");
6946 self.outbox = prior_outbox;
6947 self.rejections.truncate(prior_rejection_count);
6948 self.overlay_dirty.set(prior_overlay_dirty);
6949 return Err(error);
6950 }
6951 };
6952 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
6953 self.rollback_observation("syncular_local_purge");
6954 self.outbox = prior_outbox;
6955 self.rejections.truncate(prior_rejection_count);
6956 self.overlay_dirty.set(prior_overlay_dirty);
6957 return Err(error);
6958 }
6959 Ok(result)
6960 }
6961
6962 pub fn rebootstrap_local_data(
6968 &mut self,
6969 input: &LocalDataRebootstrapInput,
6970 ) -> Result<LocalDataRebootstrapResult, String> {
6971 if input.rebootstrap_id.is_empty()
6972 || input.rebootstrap_id.len() > 128
6973 || !is_local_operation_code_like(&input.rebootstrap_id)
6974 {
6975 return Err(
6976 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
6977 .to_owned(),
6978 );
6979 }
6980 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
6981 if let Some(persisted) = self.get_meta_strict(&meta_key)? {
6982 let (retained_commits, reset_subscriptions) =
6983 decode_local_rebootstrap_receipt(&persisted)?;
6984 return Ok(LocalDataRebootstrapResult {
6985 already_applied: true,
6986 retained_commits,
6987 reset_subscriptions,
6988 });
6989 }
6990 if self.stopped || self.schema_floor.is_some() {
6991 return Err(
6992 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
6993 .to_owned(),
6994 );
6995 }
6996
6997 let retained_commits = self.outbox.len();
6998 let reset_subscriptions = self.subs.len();
6999 let prior_subs = self.subs.clone();
7000 let prior_upgrading = self.upgrading;
7001 let prior_stopped = self.stopped;
7002 let prior_schema_floor = self.schema_floor.clone();
7003 let prior_overlay_dirty = self.overlay_dirty.get();
7004 let prior_sync_needed = self.sync_needed;
7005 let prior_sync_intents = self.sync_intent_queue.clone();
7006 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
7007
7008 self.begin_observation("syncular_local_rebootstrap")?;
7009 let mut batch = ChangeAccumulator::default();
7010 let applied = (|| -> Result<(), String> {
7011 self.run_schema_reset_observed(&mut batch, false)?;
7012 self.conn
7013 .execute(
7014 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
7015 rusqlite::params![meta_key, receipt],
7016 )
7017 .map_err(|error| error.to_string())?;
7018 self.sync_needed = true;
7019 self.sync_intent_queue.push_back(SyncIntent::Interactive);
7020 batch.status = true;
7021 Ok(())
7022 })();
7023 if let Err(error) = applied {
7024 self.rollback_observation("syncular_local_rebootstrap");
7025 self.subs = prior_subs;
7026 self.upgrading = prior_upgrading;
7027 self.stopped = prior_stopped;
7028 self.schema_floor = prior_schema_floor;
7029 self.overlay_dirty.set(prior_overlay_dirty);
7030 self.sync_needed = prior_sync_needed;
7031 self.sync_intent_queue = prior_sync_intents;
7032 return Err(error);
7033 }
7034 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
7035 self.rollback_observation("syncular_local_rebootstrap");
7036 self.subs = prior_subs;
7037 self.upgrading = prior_upgrading;
7038 self.stopped = prior_stopped;
7039 self.schema_floor = prior_schema_floor;
7040 self.overlay_dirty.set(prior_overlay_dirty);
7041 self.sync_needed = prior_sync_needed;
7042 self.sync_intent_queue = prior_sync_intents;
7043 return Err(error);
7044 }
7045
7046 Ok(LocalDataRebootstrapResult {
7047 already_applied: false,
7048 retained_commits,
7049 reset_subscriptions,
7050 })
7051 }
7052
7053 fn purge_scope_rows(
7058 &mut self,
7059 table_name: &str,
7060 effective: &[(String, Vec<String>)],
7061 ) -> Result<(), ()> {
7062 if effective.is_empty() {
7063 return Ok(());
7064 }
7065 let table = self.schema.table(table_name).ok_or(())?.clone();
7066 let mut clauses = Vec::new();
7067 let mut params: Vec<SqlValue> = Vec::new();
7068 for (variable, values) in effective {
7069 let column = table.scope_column(variable).ok_or(())?;
7070 let placeholders: Vec<String> = values
7071 .iter()
7072 .map(|v| {
7073 params.push(SqlValue::Text(v.clone()));
7074 "?".to_owned()
7075 })
7076 .collect();
7077 clauses.push(format!(
7078 "{} IN ({})",
7079 quote_ident(column),
7080 placeholders.join(", ")
7081 ));
7082 }
7083 let sql = format!(
7084 "DELETE FROM {} WHERE {}",
7085 base_table(table_name),
7086 clauses.join(" AND ")
7087 );
7088 self.overlay_dirty.set(true);
7089 self.conn
7090 .execute(&sql, rusqlite::params_from_iter(params))
7091 .map_err(|_| ())?;
7092 Ok(())
7093 }
7094
7095 fn drop_doomed_outbox(
7098 &mut self,
7099 table_name: &str,
7100 effective: &[(String, Vec<String>)],
7101 ) -> Result<bool, String> {
7102 if effective.is_empty() {
7103 return Ok(false);
7104 }
7105 let Some(table) = self.schema.table(table_name).cloned() else {
7106 return Ok(false);
7107 };
7108 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
7109 for (variable, values) in effective {
7110 match table.scope_column(variable) {
7111 Some(column) => mappings.push((column, values)),
7112 None => return Ok(false), }
7114 }
7115 let doomed: Vec<OutboxCommit> = self
7116 .outbox
7117 .iter()
7118 .filter(|commit| {
7119 commit.ops.iter().any(|op| {
7120 op.upsert
7121 && op.table == table_name
7122 && op.values.as_ref().is_some_and(|values| {
7123 mappings
7124 .iter()
7125 .all(|(column, allowed)| match values.get(*column) {
7126 Some(Value::String(s)) => allowed.contains(s),
7127 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
7128 _ => false,
7129 })
7130 })
7131 })
7132 })
7133 .cloned()
7134 .collect();
7135 if doomed.is_empty() {
7136 return Ok(false);
7137 }
7138 let mut rejections = Vec::new();
7139 for commit in &doomed {
7140 let results = commit
7141 .ops
7142 .iter()
7143 .enumerate()
7144 .map(|(op_index, operation)| {
7145 let rejection = RejectionRecord {
7146 client_commit_id: commit.client_commit_id.clone(),
7147 op_index: op_index as i32,
7148 code: "sync.scope_revoked".to_owned(),
7149 message: "the commit was dropped because its effective scope was revoked"
7150 .to_owned(),
7151 retryable: false,
7152 details: None,
7153 operation: Some(CommitOperation::from(operation)),
7154 };
7155 rejections.push(rejection.clone());
7156 CommitOperationOutcome::Error { rejection }
7157 })
7158 .collect::<Vec<_>>();
7159 self.persist_commit_outcome(
7160 &commit.client_commit_id,
7161 CommitOutcomeStatus::Rejected,
7162 &results,
7163 Some(&commit.ops),
7164 )?;
7165 self.delete_outbox_persisted(&commit.client_commit_id)?;
7166 }
7167 self.prune_commit_outcomes()?;
7168 let doomed_ids = doomed
7169 .iter()
7170 .map(|commit| commit.client_commit_id.as_str())
7171 .collect::<BTreeSet<_>>();
7172 self.outbox
7173 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
7174 self.rejections.extend(rejections);
7175 self.overlay_dirty.set(true);
7176 Ok(true)
7177 }
7178
7179 pub fn upload_blob(
7186 &mut self,
7187 bytes: &[u8],
7188 media_type: Option<String>,
7189 name: Option<String>,
7190 ) -> Result<Value, String> {
7191 let blob_id = blob_id_for(bytes);
7192 let now = self.clock_now_ms();
7193 self.conn
7194 .execute(
7195 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,?,0,?,?)
7196 ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms",
7197 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now, now],
7198 )
7199 .map_err(|e| e.to_string())?;
7200 self.conn
7201 .execute(
7202 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
7203 rusqlite::params![blob_id, media_type, now],
7204 )
7205 .map_err(|e| e.to_string())?;
7206 self.enforce_blob_cache_cap();
7209 let mut obj = Map::new();
7210 obj.insert("blobId".to_owned(), Value::from(blob_id));
7211 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
7212 if let Some(mt) = media_type {
7213 obj.insert("mediaType".to_owned(), Value::from(mt));
7214 }
7215 if let Some(n) = name {
7216 obj.insert("name".to_owned(), Value::from(n));
7217 }
7218 Ok(Value::Object(obj))
7219 }
7220
7221 pub fn fetch_blob(
7225 &mut self,
7226 transport: &mut dyn Transport,
7227 blob_id_or_ref: &str,
7228 ) -> Result<Value, (String, String)> {
7229 let simple = |m: String| ("client.failed".to_owned(), m);
7230 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
7231 blob_id_or_ref.to_owned()
7232 } else {
7233 let value: Value = serde_json::from_str(blob_id_or_ref)
7234 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
7235 value
7236 .get("blobId")
7237 .and_then(Value::as_str)
7238 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
7239 .to_owned()
7240 };
7241 if let Some(cached) = self.get_cached_blob(&blob_id).map_err(simple)? {
7242 return Ok(cached);
7243 }
7244 let bytes = match transport
7250 .blob_download(&blob_id)
7251 .map_err(|e| (e.code, e.message))?
7252 {
7253 BlobDownload::Bytes(bytes) => bytes,
7254 BlobDownload::Url {
7255 url,
7256 url_expires_at_ms,
7257 } => {
7258 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
7260 return Err((
7261 "sync.segment_expired".to_owned(),
7262 format!(
7263 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
7264 ),
7265 ));
7266 }
7267 transport
7268 .fetch_blob_url(&url)
7269 .map_err(|e| (e.code, e.message))?
7270 }
7271 };
7272 if blob_id_for(&bytes) != blob_id {
7274 return Err(simple(format!(
7275 "blob content address mismatch for {blob_id}"
7276 )));
7277 }
7278 let now = self.clock_now_ms();
7279 self.conn
7280 .execute(
7281 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms) VALUES (?,?,?,NULL,0,?,?)",
7282 rusqlite::params![blob_id, bytes, bytes.len() as i64, now, now],
7283 )
7284 .map_err(|e| simple(e.to_string()))?;
7285 self.enforce_blob_cache_cap();
7286 self.get_cached_blob(&blob_id)
7287 .map_err(simple)?
7288 .ok_or_else(|| simple("blob cache write failed".to_owned()))
7289 }
7290
7291 fn get_cached_blob(&self, blob_id: &str) -> Result<Option<Value>, String> {
7292 let _ = self.conn.execute(
7295 "UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?",
7296 rusqlite::params![self.clock_now_ms(), blob_id],
7297 );
7298 let mut stmt = self
7299 .conn
7300 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
7301 .map_err(|e| e.to_string())?;
7302 let mut rows = stmt
7303 .query(rusqlite::params![blob_id])
7304 .map_err(|e| e.to_string())?;
7305 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
7306 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
7307 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
7308 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
7309 let mut obj = Map::new();
7310 obj.insert("blobId".to_owned(), Value::from(blob_id.to_owned()));
7311 obj.insert("byteLength".to_owned(), Value::from(byte_length));
7312 let mut bytes_obj = Map::new();
7313 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(&bytes)));
7314 obj.insert("bytes".to_owned(), Value::Object(bytes_obj));
7315 if let Some(mt) = media_type {
7316 obj.insert("mediaType".to_owned(), Value::from(mt));
7317 }
7318 return Ok(Some(Value::Object(obj)));
7319 }
7320 Ok(None)
7321 }
7322
7323 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
7325 let pending: Vec<(String, Option<String>)> = {
7326 let mut stmt = self
7327 .conn
7328 .prepare(
7329 "SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms",
7330 )
7331 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
7332 let rows = stmt
7333 .query_map([], |row| {
7334 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
7335 })
7336 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
7337 rows.filter_map(Result::ok).collect()
7338 };
7339 for (blob_id, media_type) in pending {
7340 let bytes: Option<Vec<u8>> = self
7341 .conn
7342 .query_row(
7343 "SELECT bytes FROM _syncular_blobs WHERE blob_id = ?",
7344 rusqlite::params![blob_id],
7345 |row| row.get(0),
7346 )
7347 .ok();
7348 if let Some(bytes) = bytes {
7349 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
7350 }
7351 let _ = self.conn.execute(
7352 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
7353 rusqlite::params![blob_id],
7354 );
7355 }
7356 Ok(())
7357 }
7358
7359 fn upload_one(
7366 &self,
7367 transport: &mut dyn Transport,
7368 blob_id: &str,
7369 bytes: &[u8],
7370 media_type: Option<&str>,
7371 ) -> Result<(), TransportError> {
7372 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
7373 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
7375 url,
7376 url_expires_at_ms,
7377 } => {
7378 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
7379 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
7380 return Ok(());
7381 }
7382 }
7384 BlobUploadGrant::None => {
7385 }
7387 }
7388 transport.blob_upload(blob_id, bytes, media_type)
7389 }
7390
7391 fn enforce_blob_cache_cap(&self) {
7400 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
7401 return;
7402 };
7403 let mut total: i64 = self
7404 .conn
7405 .query_row(
7406 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
7407 [],
7408 |row| row.get(0),
7409 )
7410 .unwrap_or(0);
7411 if total <= max_bytes {
7412 return;
7413 }
7414 let candidates: Vec<(String, i64)> = {
7415 let Ok(mut stmt) = self.conn.prepare(
7416 "SELECT blob_id, byte_length FROM _syncular_blobs
7417 WHERE refcount = 0
7418 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
7419 ORDER BY last_used_ms ASC, created_at_ms ASC",
7420 ) else {
7421 return;
7422 };
7423 let Ok(rows) = stmt.query_map([], |row| {
7424 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
7425 }) else {
7426 return;
7427 };
7428 rows.filter_map(Result::ok).collect()
7429 };
7430 for (blob_id, byte_length) in candidates {
7431 if total <= max_bytes {
7432 break;
7433 }
7434 let _ = self.conn.execute(
7435 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
7436 rusqlite::params![blob_id],
7437 );
7438 total -= byte_length;
7439 }
7440 }
7441
7442 fn reconcile_blob_refcounts(&mut self, delete_orphans: bool) {
7446 if !self.schema_has_blobs() {
7447 return;
7448 }
7449 let mut counts: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
7450 for table in self.schema.tables.clone() {
7451 let blob_cols: Vec<String> = table
7452 .columns
7453 .iter()
7454 .filter(|c| c.ty == ColumnType::BlobRef)
7455 .map(|c| c.name.clone())
7456 .collect();
7457 for column in blob_cols {
7458 let sql = format!(
7459 "SELECT {} FROM {} WHERE {} IS NOT NULL",
7460 quote_ident(&column),
7461 base_table(&table.name),
7462 quote_ident(&column)
7463 );
7464 let Ok(mut stmt) = self.conn.prepare(&sql) else {
7465 continue;
7466 };
7467 let Ok(rows) = stmt.query_map([], |row| row.get::<_, Option<String>>(0)) else {
7468 continue;
7469 };
7470 for raw in rows.flatten().flatten() {
7471 if let Ok(value) = serde_json::from_str::<Value>(&raw) {
7472 if let Some(id) = value.get("blobId").and_then(Value::as_str) {
7473 *counts.entry(id.to_owned()).or_insert(0) += 1;
7474 }
7475 }
7476 }
7477 }
7478 }
7479 let _ = self
7480 .conn
7481 .execute("UPDATE _syncular_blobs SET refcount = 0", []);
7482 for (blob_id, count) in &counts {
7483 let _ = self.conn.execute(
7484 "UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?",
7485 rusqlite::params![count, blob_id],
7486 );
7487 }
7488 if delete_orphans {
7489 let _ = self.conn.execute(
7490 "DELETE FROM _syncular_blobs WHERE refcount = 0 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)",
7491 [],
7492 );
7493 }
7494 }
7495
7496 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
7501 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
7502 return sql.clone();
7503 }
7504 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
7505 columns.push(quote_ident("_syncular_version"));
7506 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
7507 let primary_key = quote_ident(&table.primary_key);
7508 let updates = columns
7509 .iter()
7510 .filter(|column| **column != primary_key)
7511 .map(|column| format!("{column}=excluded.{column}"))
7512 .collect::<Vec<_>>()
7513 .join(", ");
7514 let sql = format!(
7515 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
7516 columns.join(", "),
7517 placeholders.join(", ")
7518 );
7519 self.insert_sql
7520 .borrow_mut()
7521 .insert(full_table.to_owned(), sql.clone());
7522 sql
7523 }
7524
7525 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
7526 self.overlay_dirty.set(true);
7527 self.write_row(&base_table(table_name), table_name, row, version)
7528 }
7529
7530 fn write_row(
7531 &self,
7532 full_table: &str,
7533 table_name: &str,
7534 row: &Row,
7535 version: i64,
7536 ) -> Result<(), String> {
7537 let table = self
7538 .schema
7539 .table(table_name)
7540 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
7541 let sql = self.insert_row_sql(full_table, table);
7542 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
7543 let params = row
7544 .iter()
7545 .map(RowParam::Cell)
7546 .chain(std::iter::once(RowParam::Version(version)));
7547 stmt.execute(rusqlite::params_from_iter(params))
7548 .map_err(|e| e.to_string())?;
7549 Ok(())
7550 }
7551
7552 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
7553 let table = self
7554 .schema
7555 .table(table_name)
7556 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
7557 self.overlay_dirty.set(true);
7558 let sql = format!(
7559 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
7560 base_table(table_name),
7561 quote_ident(&table.primary_key)
7562 );
7563 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
7564 stmt.execute(rusqlite::params![row_id])
7565 .map_err(|e| e.to_string())?;
7566 Ok(())
7567 }
7568
7569 fn rebuild_overlay_if_dirty(&mut self) {
7572 if self.overlay_dirty.get() {
7573 self.rebuild_overlay();
7574 }
7575 }
7576
7577 fn rebuild_overlay(&mut self) {
7581 #[cfg(test)]
7582 self.overlay_rebuild_count
7583 .set(self.overlay_rebuild_count.get() + 1);
7584 self.exec("SAVEPOINT syncular_overlay");
7585 for table in self.schema.tables.clone() {
7586 for index in &table.fts_indexes {
7587 let _ = self.drop_fts_triggers(index);
7588 }
7589 let visible = visible_table(&table.name);
7590 let base = base_table(&table.name);
7591 self.exec(&format!("DELETE FROM {visible}"));
7592 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
7593 }
7594 for commit in self.outbox.clone() {
7595 for op in &commit.ops {
7596 let Some(table) = self.schema.table(&op.table).cloned() else {
7597 continue;
7598 };
7599 if op.upsert {
7600 let Some(values) = op.values.as_ref() else {
7601 continue;
7602 };
7603 let mut row: Row = Vec::with_capacity(table.columns.len());
7604 let mut ok = true;
7605 for column in &table.columns {
7606 match json_to_column_value(column, values.get(&column.name)) {
7607 Ok(v) => row.push(v),
7608 Err(_) => {
7609 ok = false;
7610 break;
7611 }
7612 }
7613 }
7614 if ok {
7615 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, -1);
7616 }
7617 } else {
7618 let sql = format!(
7619 "DELETE FROM {} WHERE CAST({} AS TEXT) = ?1",
7620 visible_table(&table.name),
7621 quote_ident(&table.primary_key)
7622 );
7623 let _ = self.conn.execute(&sql, rusqlite::params![op.row_id]);
7624 }
7625 }
7626 }
7627 for table in self.schema.tables.clone() {
7628 for index in &table.fts_indexes {
7629 let _ = self.rebuild_fts_projection(&table, index);
7630 let _ = self.create_fts_triggers(&table, index);
7631 }
7632 }
7633 self.exec("RELEASE syncular_overlay");
7634 self.overlay_dirty.set(false);
7635 }
7636
7637 fn exec(&self, sql: &str) {
7638 let _ = self.conn.execute_batch(sql);
7639 }
7640
7641 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
7644 if self.realtime_connected {
7645 return Ok(());
7646 }
7647 transport
7648 .realtime_connect_for_client(&self.client_id)
7649 .map_err(|e| format!("{}: {}", e.code, e.message))?;
7650 self.realtime_connected = true;
7651 Ok(())
7652 }
7653
7654 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
7655 if !self.realtime_connected {
7656 return;
7657 }
7658 let _ = transport.realtime_close();
7659 self.realtime_connected = false;
7660 self.presence.clear(); }
7662
7663 pub fn set_presence(
7669 &mut self,
7670 transport: &mut dyn Transport,
7671 scope_key: &str,
7672 doc: Option<&Value>,
7673 ) -> Result<(), String> {
7674 if !self.realtime_connected {
7675 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
7676 }
7677 let text = encode_presence_publish(scope_key, doc);
7678 transport
7679 .realtime_send(&text)
7680 .map_err(|e| format!("{}: {}", e.code, e.message))
7681 }
7682
7683 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
7685 self.presence
7686 .get(scope_key)
7687 .map(|peers| peers.values().cloned().collect())
7688 .unwrap_or_default()
7689 }
7690
7691 fn apply_presence(
7693 &mut self,
7694 scope_key: String,
7695 kind: Option<PresenceKind>,
7696 actor_id: Option<String>,
7697 client_id: Option<String>,
7698 doc: Option<Value>,
7699 error: Option<String>,
7700 ) {
7701 if error.is_some() {
7704 return;
7705 }
7706 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
7707 return;
7708 };
7709 let peer_key = format!("{actor_id} {client_id}");
7710 match kind {
7711 PresenceKind::Leave => {
7712 if let Some(peers) = self.presence.get_mut(&scope_key) {
7713 peers.remove(&peer_key);
7714 if peers.is_empty() {
7715 self.presence.remove(&scope_key);
7716 }
7717 }
7718 }
7719 _ => {
7720 let doc = match doc {
7721 Some(Value::Object(_)) => doc.unwrap(),
7722 _ => return,
7723 };
7724 self.presence.entry(scope_key).or_default().insert(
7725 peer_key,
7726 PresencePeer {
7727 actor_id,
7728 client_id,
7729 doc,
7730 },
7731 );
7732 }
7733 }
7734 }
7735
7736 pub fn on_realtime_text(&mut self, text: &str) {
7738 match parse_control(text) {
7739 Ok(ControlMessage::Hello { requires_sync, .. }) => {
7740 if requires_sync {
7741 self.set_sync_needed(true, true);
7743 }
7744 }
7745 Ok(ControlMessage::Presence {
7746 scope_key,
7747 kind,
7748 actor_id,
7749 client_id,
7750 doc,
7751 error,
7752 ..
7753 }) => {
7754 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
7755 }
7756 Ok(ControlMessage::Wake { .. }) => {
7757 self.set_sync_needed(true, true);
7759 }
7760 _ => {}
7761 }
7762 }
7763
7764 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
7767 if self.stopped {
7768 return;
7769 }
7770 let message = match decode_message(bytes) {
7771 Ok(m) if m.msg_kind == MsgKind::Response => m,
7772 _ => {
7773 self.set_sync_needed(true, true);
7774 return;
7775 }
7776 };
7777 let mut frames = message.frames.into_iter();
7778 let mut applied_cursor: Option<i64> = None;
7779 let mut any_covered = false;
7780 let mut dropped = false;
7781 while let Some(frame) = frames.next() {
7782 let Frame::SubStart {
7783 id,
7784 status,
7785 effective_scopes,
7786 ..
7787 } = frame
7788 else {
7789 continue;
7790 };
7791 let mut body = Vec::new();
7792 let mut next_cursor: Option<i64> = None;
7793 for inner in frames.by_ref() {
7794 match inner {
7795 Frame::SubEnd {
7796 next_cursor: nc, ..
7797 } => {
7798 next_cursor = Some(nc);
7799 break;
7800 }
7801 Frame::Unknown { .. } => {}
7802 other => body.push(other),
7803 }
7804 }
7805 let Some(next_cursor) = next_cursor else {
7806 dropped = true;
7807 break;
7808 };
7809 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
7810 dropped = true;
7811 continue;
7812 };
7813 let sub = &self.subs[sub_index];
7814 if status != SubStatus::Active
7817 || sub.state != SubState::Active
7818 || sub.bootstrap_state.is_some()
7819 || !sub.synced_once
7820 {
7821 dropped = true;
7822 continue;
7823 }
7824 if next_cursor <= sub.cursor {
7825 any_covered = true;
7827 continue;
7828 }
7829 let previous_effective = self.subs[sub_index].effective.clone();
7830 let previous_cursor = self.subs[sub_index].cursor;
7831 if self.begin_observation("syncular_delta").is_err() {
7832 dropped = true;
7833 continue;
7834 }
7835 self.subs[sub_index].effective = Some(effective_scopes);
7836 let mut batch = ChangeAccumulator::default();
7837 let mut failed = false;
7838 for inner in body {
7839 if let Frame::Commit {
7840 tables, changes, ..
7841 } = inner
7842 {
7843 self.record_commit_changes(&mut batch, &tables, &changes);
7844 if self.apply_commit_changes(&tables, &changes).is_err() {
7845 failed = true;
7846 break;
7847 }
7848 }
7849 }
7850 if failed {
7851 self.rollback_observation("syncular_delta");
7852 self.subs[sub_index].effective = previous_effective;
7853 self.subs[sub_index].cursor = previous_cursor;
7854 self.overlay_dirty.set(true);
7855 self.rebuild_overlay();
7856 dropped = true;
7857 continue;
7858 }
7859 let sub = &mut self.subs[sub_index];
7860 sub.cursor = next_cursor;
7861 self.persist_sub(&self.subs[sub_index].clone());
7862 self.rebuild_overlay_if_dirty();
7863 if self.finish_observation("syncular_delta", batch).is_err() {
7864 self.rollback_observation("syncular_delta");
7865 self.subs[sub_index].effective = previous_effective;
7866 self.subs[sub_index].cursor = previous_cursor;
7867 self.overlay_dirty.set(true);
7868 self.rebuild_overlay();
7869 dropped = true;
7870 continue;
7871 }
7872 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
7873 }
7874 if let Some(cursor) = applied_cursor {
7875 self.reconcile_blob_refcounts(false);
7876 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
7878 let _ = transport.realtime_send(&ack);
7879 } else if !any_covered || dropped {
7880 self.set_sync_needed(true, true);
7882 }
7883 }
7884
7885 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
7889 if !self.realtime_connected {
7890 return;
7891 }
7892 let floor = self
7893 .subs
7894 .iter()
7895 .filter(|s| {
7896 s.state == SubState::Active
7897 && s.bootstrap_state.is_none()
7898 && s.synced_once
7899 && s.cursor >= 0
7900 })
7901 .map(|s| s.cursor)
7902 .min();
7903 if let Some(cursor) = floor {
7904 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
7905 let _ = transport.realtime_send(&ack);
7906 }
7907 }
7908}