1use crate::{ProgressObserver, ProgressPhase, ProgressState};
11use std::cell::{Cell, RefCell};
12use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
13
14use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
15use rusqlite::{Connection, OpenFlags, OptionalExtension};
16use serde::{Deserialize, Serialize};
17use serde_json::{Map, Value};
18use sha2::{Digest, Sha256};
19use ssp2::decode::WIRE_VERSION;
20use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
21use ssp2::primitives::RawJson;
22use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
23use ssp2::{
24 decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
25 PresenceKind,
26};
27
28use crate::api::{
29 ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
30 ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
31 ClientDiagnosticsStorage, ClientLimits, CommandEffects, CommitOperation,
32 CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution,
33 CommitOutcomeStatus, ConflictRecord, CoverageSnapshot, DiagnosticLastChange,
34 DiagnosticLastRound, DiagnosticRoundCounters, DiagnosticSubscription, FetchedBlob, LeaseState,
35 LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget, LocalDataRebootstrapInput,
36 LocalDataRebootstrapResult, Mutation, PresencePeer, QueryRow, QuerySnapshot, QueryValue,
37 RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
38 SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
39 WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
40 CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
41};
42#[cfg(feature = "bench-internals")]
43use crate::bench::{Phase, Recorder};
44use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
45use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
46use crate::values::{
47 bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
48 encode_sparse_row_json, full_row_values, json_to_column_value, json_to_scope_map,
49 normalize_values_casing, render_row_id_json, scope_map_to_json, sort_scope_map,
50};
51
52const DEFAULT_ACCEPT: u8 = 0b0111;
56const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
57const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
58const ACCEPT_SQLITE: u8 = 1 << 2;
59const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
60const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
61
62const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
64const LOCAL_REVISION_KEY: &str = "localRevision";
65const CLIENT_ID_KEY: &str = "clientId";
66const LEASE_STATE_KEY: &str = "leaseState";
67const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
68const LOG_EPOCH_KEY: &str = "logEpoch";
69const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
74const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
75const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
76const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
79const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
82
83#[derive(Debug, Serialize, Deserialize)]
84#[serde(rename_all = "camelCase", deny_unknown_fields)]
85struct PersistedLocalDataRebootstrapReceipt {
86 version: u8,
87 retained_commits: u64,
88 reset_subscriptions: u64,
89}
90
91fn invalid_local_rebootstrap_receipt() -> String {
92 "sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
93}
94
95fn encode_local_rebootstrap_receipt(
96 retained_commits: usize,
97 reset_subscriptions: usize,
98) -> Result<String, String> {
99 let retained_commits =
100 u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
101 let reset_subscriptions =
102 u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
103 if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
104 return Err(invalid_local_rebootstrap_receipt());
105 }
106 serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
107 version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
108 retained_commits,
109 reset_subscriptions,
110 })
111 .map_err(|_| invalid_local_rebootstrap_receipt())
112}
113
114fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
115 if value == "v1" {
117 return Ok((0, 0));
118 }
119 let receipt: PersistedLocalDataRebootstrapReceipt =
120 serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
121 if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
122 || receipt.retained_commits > MAX_JS_SAFE_INTEGER
123 || receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
124 {
125 return Err(invalid_local_rebootstrap_receipt());
126 }
127 Ok((
128 usize::try_from(receipt.retained_commits)
129 .map_err(|_| invalid_local_rebootstrap_receipt())?,
130 usize::try_from(receipt.reset_subscriptions)
131 .map_err(|_| invalid_local_rebootstrap_receipt())?,
132 ))
133}
134
135#[derive(Debug, Clone, Copy, PartialEq, Eq)]
136enum SubState {
137 Active,
138 Revoked,
139 Failed,
140}
141
142#[cfg(test)]
143mod observation_tests {
144 use super::*;
145 use crate::native_transport::HostTransport;
146 use crate::values::encode_row_json;
147 use serde_json::json;
148
149 fn client() -> SyncClient {
150 SyncClient::new(
151 "retry-test".to_owned(),
152 &json!({
153 "version": 1,
154 "tables": [{
155 "name": "tasks",
156 "primaryKey": "id",
157 "columns": [
158 { "name": "id", "type": "string", "nullable": false },
159 { "name": "project_id", "type": "string", "nullable": false }
160 ],
161 "scopes": [{ "pattern": "project:{project_id}" }]
162 }]
163 }),
164 ClientLimits::default(),
165 )
166 .expect("test client")
167 }
168
169 #[cfg(feature = "e2ee")]
170 #[test]
171 fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
172 let schema = json!({
173 "version": 1,
174 "tables": [{
175 "name": "secrets",
176 "primaryKey": "id",
177 "columns": [
178 { "name": "id", "type": "string", "nullable": false },
179 { "name": "encryption_key_id", "type": "string", "nullable": true },
180 { "name": "note", "type": "bytes", "nullable": true,
181 "encrypted": true, "declaredType": "string" }
182 ],
183 "scopes": []
184 }]
185 });
186 let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
187 .expect("test client");
188 let mut config = crate::values::EncryptionConfig::default();
189 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
190 config
191 .key_id_columns
192 .insert("secrets".to_owned(), "encryption_key_id".to_owned());
193 client.set_encryption(config);
194 let commit_id = client
197 .patch(
198 "secrets",
199 "ghost",
200 Map::from_iter([("note".to_owned(), json!("x"))]),
201 None,
202 )
203 .expect("patch records without an author-time encrypt failure");
204 assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
205 assert!(client.drop_unencodable_outbox().unwrap());
207 assert!(client.pending_commit_ids().is_empty());
208 let rejections = client.rejections();
209 assert_eq!(rejections.len(), 1);
210 assert_eq!(rejections[0].code, "client.encrypt_failed");
211 assert_eq!(rejections[0].client_commit_id, commit_id);
212 assert_eq!(
213 client
214 .commit_outcome(&commit_id)
215 .unwrap()
216 .map(|outcome| outcome.status),
217 Some(CommitOutcomeStatus::Rejected)
218 );
219 }
220
221 #[cfg(feature = "e2ee")]
222 #[test]
223 fn stored_key_fallback_resolves_an_integer_primary_key() {
224 let schema = json!({
225 "version": 1,
226 "tables": [{
227 "name": "patients",
228 "primaryKey": "id",
229 "columns": [
230 { "name": "id", "type": "integer", "nullable": false },
231 { "name": "encryption_key_id", "type": "string", "nullable": true },
232 { "name": "note", "type": "bytes", "nullable": true,
233 "encrypted": true, "declaredType": "string" }
234 ],
235 "scopes": []
236 }]
237 });
238 let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
239 .expect("test client");
240 client.create_synced_tables().unwrap();
241 client
242 .conn
243 .execute(
244 "INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
245 VALUES (5, 'k1', NULL, 1)",
246 [],
247 )
248 .unwrap();
249 let mut config = crate::values::EncryptionConfig::default();
250 config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
251 config
252 .key_id_columns
253 .insert("patients".to_owned(), "encryption_key_id".to_owned());
254 client.set_encryption(config);
255 let table = client.schema.table("patients").unwrap();
256 let fallback = client
257 .stored_key_fallback(table, "5")
258 .expect("the stored integer key resolves the fallback");
259 assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
260 }
261
262 #[test]
263 fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
264 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
265 {"name":"id","type":"string","nullable":false},
266 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
267 for state in ["absent", "cached", "pinned"] {
268 let path = std::env::temp_dir().join(format!(
269 "syncular-blob-stage-{}.sqlite",
270 uuid::Uuid::new_v4()
271 ));
272 let mut client = SyncClient::open_path(
273 "stage-test".into(),
274 &schema,
275 Default::default(),
276 path.to_str().unwrap(),
277 )
278 .unwrap();
279 let bytes = b"atomic stage";
280 if state != "absent" {
281 client
282 .upload_blob(bytes, Some("text/plain".into()), None)
283 .unwrap();
284 if state == "cached" {
285 client
286 .conn
287 .execute("DELETE FROM _syncular_blob_uploads", [])
288 .unwrap();
289 }
290 }
291 let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
292 client
293 .conn
294 .execute_batch(
295 "CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
296 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
297 CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
298 BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
299 )
300 .unwrap();
301 let error = client
302 .upload_blob(bytes, Some("application/octet-stream".into()), None)
303 .unwrap_err();
304 assert!(error.contains("injected stage failure"), "{state}: {error}");
305 assert!(client.conn.is_autocommit());
306 let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
307 drop(client);
308 let mut reopened = SyncClient::open_path(
309 "stage-test".into(),
310 &schema,
311 Default::default(),
312 path.to_str().unwrap(),
313 )
314 .unwrap();
315 let durable = reopened
316 .query("SELECT * FROM _syncular_blobs", &[])
317 .unwrap();
318 reopened
319 .conn
320 .execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
321 .unwrap();
322 let retried = reopened.upload_blob(bytes, None, None).unwrap();
323 assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
324 assert_eq!(
325 reopened
326 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
327 .unwrap(),
328 vec![Map::from_iter([(
329 "blob_id".into(),
330 Value::from(blob_id_for(bytes)),
331 )])]
332 );
333 drop(reopened);
334 std::fs::remove_file(&path).unwrap();
335 assert_eq!(durable, bodies, "{state}: reopened body state");
336 assert_eq!(immediate, bodies, "{state}: immediate body state");
337 }
338 }
339
340 #[test]
341 fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
342 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
343 {"name":"id","type":"string","nullable":false},
344 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
345 for fault in [
346 "missing",
347 "body-type",
348 "length",
349 "hash",
350 "pin-metadata",
351 "read",
352 "pin-delete",
353 ] {
354 let path = std::env::temp_dir().join(format!(
355 "syncular-blob-upload-{}.sqlite",
356 uuid::Uuid::new_v4()
357 ));
358 let mut client = SyncClient::open_path(
359 "upload-test".into(),
360 &schema,
361 Default::default(),
362 path.to_str().unwrap(),
363 )
364 .unwrap();
365 let bytes = b"durable upload";
366 let ref_value = client
367 .upload_blob(bytes, Some("text/plain".into()), None)
368 .unwrap();
369 let commit = client
370 .mutate(vec![Mutation::Upsert {
371 table: "attachments".into(),
372 values: Map::from_iter([
373 ("id".into(), Value::from("pending")),
374 (
375 "file".into(),
376 Value::from(format!(
377 "{{\"blobId\":\"{}\",\"byteLength\":{}}}",
378 blob_id_for(bytes),
379 bytes.len()
380 )),
381 ),
382 ]),
383 base_version: None,
384 }])
385 .unwrap();
386 client
387 .conn
388 .execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
389 .unwrap();
390 client.conn.execute_batch(match fault {
391 "missing" => "DELETE FROM _syncular_blobs",
392 "body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
393 "length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
394 "hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
395 "pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
396 "read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
397 _ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
398 }).unwrap();
399 let state = if fault == "read" {
400 Vec::new()
401 } else {
402 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
403 };
404 let mut transport = CountingRealtimeTransport::default();
405 let error = client.flush_blob_uploads(&mut transport).unwrap_err();
406 assert_eq!(
407 error.code,
408 if matches!(fault, "read" | "pin-delete") {
409 "client.failed"
410 } else {
411 "sync.local_corrupt"
412 },
413 "{fault}"
414 );
415 assert_eq!(
416 transport.blob_uploads.len(),
417 usize::from(fault == "pin-delete"),
418 "{fault}: corrupt bytes must never be uploaded"
419 );
420 assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
421 if fault != "read" {
422 assert_eq!(
423 client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
424 state
425 );
426 } else {
427 client
428 .conn
429 .execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
430 .unwrap();
431 }
432 drop(client);
433 let mut reopened = SyncClient::open_path(
434 "upload-test".into(),
435 &schema,
436 Default::default(),
437 path.to_str().unwrap(),
438 )
439 .unwrap();
440 assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
441 if fault != "read" {
442 assert_eq!(
443 reopened
444 .query("SELECT * FROM _syncular_blobs", &[])
445 .unwrap(),
446 state
447 );
448 }
449 if fault == "pin-delete" {
450 reopened
451 .conn
452 .execute_batch("DROP TRIGGER fail_pin_delete")
453 .unwrap();
454 }
455 if fault == "pin-metadata" {
456 reopened
457 .conn
458 .execute(
459 "UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
460 [],
461 )
462 .unwrap();
463 }
464 reopened
465 .conn
466 .execute_batch(
467 "DELETE FROM _syncular_blobs;
468 INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
469 )
470 .unwrap();
471 transport.blob_uploads.clear();
472 reopened.flush_blob_uploads(&mut transport).unwrap();
473 assert_eq!(
474 transport.blob_uploads,
475 [(blob_id_for(bytes), bytes.to_vec())]
476 );
477 assert_eq!(
478 reopened
479 .query("SELECT blob_id FROM _syncular_blob_uploads", &[])
480 .unwrap(),
481 Vec::<Map<String, Value>>::new()
482 );
483 assert_eq!(reopened.pending_commit_ids(), [commit]);
484 assert_eq!(
485 reopened
486 .get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
487 .unwrap()
488 .unwrap()
489 .bytes,
490 bytes
491 );
492 drop(reopened);
493 std::fs::remove_file(path).unwrap();
494 }
495 }
496
497 #[test]
498 fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
499 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
500 {"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
501 "scopes":[{"pattern":"project:{project_id}"}]}]});
502 for realtime in [false, true] {
503 for fault in [
504 "none",
505 "row",
506 "revision",
507 "commit",
508 "cursor",
509 "error",
510 "missing-end",
511 ] {
512 if realtime && fault == "missing-end" {
513 continue;
514 }
515 let path = std::env::temp_dir()
516 .join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
517 let mut client = SyncClient::open_path(
518 "frame-test".into(),
519 &schema,
520 Default::default(),
521 path.to_str().unwrap(),
522 )
523 .unwrap();
524 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
525 client
526 .subscribe(
527 "tasks".into(),
528 "tasks".into(),
529 vec![("project_id".into(), vec!["p1".into()])],
530 None,
531 )
532 .unwrap();
533 client.subs[0].cursor = 0;
534 client.subs[0].synced_once = true;
535 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
536 client.persist_sub(&client.subs[0]).unwrap();
537 client.realtime_connected = true;
538 client.drain_change_batches();
539 let mut transport = CountingRealtimeTransport::default();
540 let (_, meta) = client.build_request(false);
541 let mut response = Message {
542 wire_version: WIRE_VERSION,
543 msg_kind: MsgKind::Response,
544 frames: vec![
545 Frame::RespHeader {
546 required_schema_version: None,
547 latest_schema_version: None,
548 log_epoch: Some("epoch-1".into()),
549 reset_required: Some(false),
550 },
551 Frame::SubStart {
552 id: "tasks".into(),
553 status: SubStatus::Active,
554 reason_code: "".into(),
555 effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
556 bootstrap: false,
557 },
558 ],
559 };
560 for (index, id) in ["first", "second"].iter().enumerate() {
561 let values = Map::from_iter([
562 ("id".into(), Value::from(*id)),
563 ("project_id".into(), Value::from("p1")),
564 ]);
565 response.frames.push(Frame::Commit {
566 commit_seq: index as i64 + 1,
567 actor_id: "writer".into(),
568 created_at_ms: 1,
569 tables: vec!["tasks".into()],
570 changes: vec![ssp2::model::Change {
571 table_index: 0,
572 row_id: (*id).into(),
573 op: Op::Upsert,
574 row_version: Some(index as i64 + 1),
575 scopes: vec![("project_id".into(), "p1".into())],
576 row: Some(
577 encode_row_json(
578 client.schema.table("tasks").unwrap(),
579 id,
580 &values,
581 &client.encryption,
582 )
583 .unwrap(),
584 ),
585 }],
586 });
587 }
588 response.frames.push(Frame::SubEnd {
589 next_cursor: 2,
590 bootstrap_state: None,
591 });
592 let valid = response.clone();
593 match fault {
594 "row" => {
595 let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
596 let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
597 },
598 "revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
599 "commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
600 "cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
601 "error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
602 "missing-end" => {response.frames.truncate(3);},
603 _ => {}
604 }
605 if realtime {
606 client.on_realtime_binary(&mut transport, &encode_message(&response));
607 } else {
608 assert_eq!(
609 matches!(
610 client.process_response(&mut transport, response, &meta),
611 SyncOutcome::Ok(_)
612 ),
613 fault == "none",
614 "{fault}"
615 );
616 }
617 assert!(
618 client.conn.is_autocommit(),
619 "no open transaction after {realtime}/{fault}"
620 );
621 let count = if matches!(fault, "none" | "cursor") {
622 2
623 } else {
624 1
625 };
626 let expected_revision = count + u64::from(realtime && fault != "none");
627 assert_eq!(
628 client.local_revision(),
629 expected_revision,
630 "{realtime}/{fault}"
631 );
632 let batches = client.drain_change_batches();
633 assert_eq!(
634 batches.len(),
635 expected_revision as usize,
636 "{realtime}/{fault}"
637 );
638 let row_batches: Vec<_> = batches
639 .iter()
640 .filter(|batch| !batch.tables.is_empty())
641 .collect();
642 assert_eq!(row_batches.len(), count as usize);
643 assert_eq!(
644 row_batches
645 .iter()
646 .map(|b| b.revision.clone())
647 .collect::<Vec<_>>(),
648 (1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
649 );
650 for table in ["tasks", "_syncular_base_tasks"] {
651 assert_eq!(
652 client
653 .conn
654 .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
655 .get::<_, u64>(0))
656 .unwrap(),
657 count,
658 "{realtime}/{fault}/{table}"
659 );
660 }
661 assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
662 assert_eq!(
663 transport
664 .messages
665 .iter()
666 .filter(|m| m.contains("\"ack\""))
667 .count(),
668 usize::from(fault == "none")
669 );
670 if matches!(fault, "revision" | "commit" | "cursor") {
671 client
672 .conn
673 .execute_batch("DROP TRIGGER fail_frame")
674 .unwrap();
675 }
676 drop(client);
677 let mut reopened = SyncClient::open_path(
678 "frame-test".into(),
679 &schema,
680 Default::default(),
681 path.to_str().unwrap(),
682 )
683 .unwrap();
684 assert_eq!(reopened.local_revision(), expected_revision);
685 assert_eq!(
686 reopened
687 .query("SELECT id FROM tasks ORDER BY id", &[])
688 .unwrap()
689 .len(),
690 count as usize
691 );
692 assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
693 let (_, meta) = reopened.build_request(false);
694 assert!(matches!(
695 reopened.process_response(&mut transport, valid, &meta),
696 SyncOutcome::Ok(_)
697 ));
698 assert_eq!(
699 reopened
700 .query("SELECT id FROM tasks ORDER BY id", &[])
701 .unwrap()
702 .len(),
703 2
704 );
705 assert_eq!(reopened.subs[0].cursor, 2);
706 drop(reopened);
707 std::fs::remove_file(path).unwrap();
708 }
709 }
710 }
711
712 #[test]
713 fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
714 for status in [SubStatus::Reset, SubStatus::Revoked] {
715 for fault in ["subscription", "revision"] {
716 let mut client = client();
717 client
718 .subscribe(
719 "tasks".into(),
720 "tasks".into(),
721 vec![("project_id".into(), vec!["p1".into()])],
722 None,
723 )
724 .unwrap();
725 client.subs[0].cursor = 41;
726 client.subs[0].synced_once = true;
727 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
728 client.persist_sub(&client.subs[0]).unwrap();
729 client
730 .mutate(vec![Mutation::Upsert {
731 table: "tasks".into(),
732 values: Map::from_iter([
733 ("id".into(), Value::from("pending")),
734 ("project_id".into(), Value::from("p1")),
735 ]),
736 base_version: None,
737 }])
738 .unwrap();
739 let ids = client.pending_commit_ids();
740 let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
741 let revision = client.local_revision();
742 client.drain_change_batches();
743 let (_, meta) = client.build_request(false);
744 client.conn.execute_batch(if fault == "subscription" {
745 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
746 } else {
747 "CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
748 }).unwrap();
749 let mut transport = CountingRealtimeTransport::default();
750 let mut report = SyncReport::default();
751 if status == SubStatus::Reset {
753 client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
754 }
755 let result = client.process_section(
756 &mut transport,
757 "tasks",
758 status,
759 "sync.scope_revoked",
760 vec![],
761 vec![],
762 Some((42, None)),
763 &meta,
764 &mut report,
765 );
766 assert!(result.is_err(), "{status:?}/{fault}");
767 assert!(client.conn.is_autocommit());
768 assert_eq!(client.subs[0].cursor, 41);
769 assert_eq!(client.subs[0].state, SubState::Active);
770 assert_eq!(client.pending_commit_ids(), ids);
771 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
772 assert!(client.rejections.is_empty());
773 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
774 assert_eq!(client.local_revision(), revision);
775 assert!(client.drain_change_batches().is_empty());
776 assert!(report.revoked.is_empty() && report.resets.is_empty());
777 }
778 }
779 }
780
781 #[test]
782 fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
783 let mut client = client();
784 client
785 .subscribe(
786 "tasks".into(),
787 "tasks".into(),
788 vec![("project_id".into(), vec!["p1".into()])],
789 None,
790 )
791 .unwrap();
792 client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
793 client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
794 CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
795 let row = |id: &str| ssp2::segment::SegmentRow {
796 server_version: 1,
797 values: vec![
798 Some(ColumnValue::String(id.into())),
799 Some(ColumnValue::String("p1".into())),
800 ],
801 };
802 let segment = RowsSegment {
803 table: "tasks".into(),
804 schema_version: 1,
805 columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
806 blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
807 };
808 assert!(client.apply_segment(0, &segment, true).is_err());
809 assert!(client.conn.is_autocommit());
810 assert_eq!(
811 client
812 .query("SELECT id FROM tasks ORDER BY id", &[])
813 .unwrap(),
814 vec![Map::from_iter([("id".into(), Value::from("first"))])]
815 );
816 assert_eq!(client.local_revision(), 1);
817 assert_eq!(client.drain_change_batches().len(), 1);
818 assert_eq!(client.subs[0].cursor, -1);
819 client
820 .conn
821 .execute_batch("DROP TRIGGER fail_block")
822 .unwrap();
823 assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
824 assert_eq!(client.local_revision(), 3);
825 assert_eq!(client.drain_change_batches().len(), 2);
826 }
827
828 #[test]
829 fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
830 assert_eq!(
831 client().diagnostics_storage().blob_cache_bytes_approx,
832 Some(0)
833 );
834 let client = SyncClient::new(
835 "diagnostic-storage".to_owned(),
836 &json!({
837 "version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
838 "columns": [{"name": "id", "type": "string", "nullable": false},
839 {"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
840 }),
841 ClientLimits {
842 blob_cache_max_bytes: Some(5),
843 ..Default::default()
844 },
845 )
846 .unwrap();
847 let empty = client.diagnostics_storage();
848 assert_eq!(empty.status, "healthy");
849 assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
850 assert_eq!(empty.retained_outcome_entries, Some(0));
851 assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
852 assert_eq!(empty.blob_cache_bytes_approx, Some(0));
853 client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
854 INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
855 VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
856 INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
857 VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
858 let written = client.diagnostics_storage();
859 assert_eq!(written.status, "pressure");
860 assert_eq!(written.pending_outbox_bytes_approx, Some(2));
861 assert_eq!(written.retained_outcome_entries, Some(2));
862 assert_eq!(written.retained_outcome_bytes_approx, Some(6));
863 assert_eq!(written.blob_cache_bytes_approx, Some(8));
864 assert_eq!(
865 written.pressure_reason_code.as_deref(),
866 Some("client.blob_cache_over_limit")
867 );
868 client
869 .conn
870 .execute_batch(
871 "BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
872 DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
873 )
874 .unwrap();
875 let staged = client.diagnostics_storage();
876 assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
877 assert_eq!(staged.retained_outcome_entries, Some(0));
878 assert_eq!(staged.blob_cache_bytes_approx, Some(0));
879 assert_eq!(staged.status, "healthy");
880 client.conn.execute_batch("ROLLBACK").unwrap();
881 assert_eq!(client.diagnostics_storage(), written);
882 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
883 let unreadable = client.diagnostics_storage();
884 assert_eq!(unreadable.status, "unreadable");
885 assert!(unreadable.database_bytes_approx.is_none());
886 assert!(unreadable.pending_outbox_bytes_approx.is_none());
887 assert!(unreadable.retained_outcome_entries.is_none());
888 assert!(unreadable.retained_outcome_bytes_approx.is_none());
889 assert!(unreadable.blob_cache_bytes_approx.is_none());
890 client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
891 let restored = client.diagnostics_storage();
892 assert_eq!(restored.status, written.status);
893 assert_eq!(
894 restored.retained_outcome_bytes_approx,
895 written.retained_outcome_bytes_approx
896 );
897 client.conn.set_prepared_statement_cache_capacity(0);
898 assert_eq!(client.diagnostics_storage(), restored);
899 }
900
901 #[test]
902 fn background_retry_deadlines_back_off_and_reset() {
903 let mut client = client();
904 client.schedule_background_retry();
905 assert!(matches!(
906 client.drain_sync_intents().as_slice(),
907 [SyncIntent::Background { delay_ms: 250 }]
908 ));
909 client.schedule_background_retry();
910 assert!(matches!(
911 client.drain_sync_intents().as_slice(),
912 [SyncIntent::Background { delay_ms: 500 }]
913 ));
914 client.reset_background_retry();
915 client.schedule_background_retry();
916 assert!(matches!(
917 client.drain_sync_intents().as_slice(),
918 [SyncIntent::Background { delay_ms: 250 }]
919 ));
920 }
921
922 #[test]
923 fn log_epoch_reset_requests_an_immediate_follow_up_round() {
924 let mut client = client();
925 client
926 .subscribe(
927 "epoch-tasks".to_owned(),
928 "tasks".to_owned(),
929 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
930 None,
931 )
932 .expect("subscribe");
933 client.drain_sync_intents();
934
935 client
936 .run_log_epoch_reset("epoch-2")
937 .expect("reset partition log epoch");
938
939 assert!(matches!(
940 client.drain_sync_intents().as_slice(),
941 [SyncIntent::Interactive]
942 ));
943 }
944
945 #[derive(Default)]
946 struct CountingRealtimeTransport {
947 connects: usize,
948 closes: usize,
949 messages: Vec<String>,
950 blob_uploads: Vec<(String, Vec<u8>)>,
951 blob_download: Option<Vec<u8>>,
952 }
953
954 impl Transport for CountingRealtimeTransport {
955 fn blob_upload(
956 &mut self,
957 blob_id: &str,
958 bytes: &[u8],
959 _media_type: Option<&str>,
960 ) -> Result<(), TransportError> {
961 self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
962 Ok(())
963 }
964
965 fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
966 self.blob_download
967 .clone()
968 .map(BlobDownload::Bytes)
969 .ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
970 }
971
972 fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
973 Err(TransportError::new("sync.transport_failed", "offline"))
974 }
975
976 fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
977 Err(TransportError::new("sync.transport_failed", "offline"))
978 }
979
980 fn download_segment(
981 &mut self,
982 _request: &SegmentRequest,
983 _on_progress: &mut dyn FnMut(u64),
984 ) -> Result<Vec<u8>, TransportError> {
985 Err(TransportError::new("sync.transport_failed", "offline"))
986 }
987
988 fn realtime_connect(&mut self) -> Result<(), TransportError> {
989 self.connects += 1;
990 Ok(())
991 }
992
993 fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
994 self.messages.push(text.to_owned());
995 Ok(())
996 }
997
998 fn realtime_close(&mut self) -> Result<(), TransportError> {
999 self.closes += 1;
1000 Ok(())
1001 }
1002 }
1003
1004 #[test]
1005 fn blob_results_own_downloaded_bytes() {
1006 let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
1007 {"name":"id","type":"string","nullable":false},
1008 {"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
1009 let bytes = b"owned typed result".to_vec();
1010 let blob_id = blob_id_for(&bytes);
1011 let mut transport = CountingRealtimeTransport {
1012 blob_download: Some(bytes.clone()),
1013 ..Default::default()
1014 };
1015 let result = {
1016 let mut client =
1017 SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
1018 .unwrap();
1019 let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
1020 assert_eq!(result.blob_id, blob_id);
1021 assert_eq!(result.byte_length, bytes.len() as i64);
1022 assert_eq!(result.bytes, bytes);
1023 assert_eq!(result.media_type, None);
1024 assert_eq!(
1025 client
1026 .conn
1027 .query_row(
1028 "SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
1029 rusqlite::params![blob_id],
1030 |row| row.get::<_, i64>(0),
1031 )
1032 .unwrap(),
1033 0
1034 );
1035 result
1036 };
1037 assert_eq!(result.bytes, bytes);
1038
1039 let mut client =
1040 SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
1041 assert_eq!(
1042 client
1043 .fetch_blob_bytes(&mut transport, "not json")
1044 .unwrap_err(),
1045 (
1046 "client.failed".to_owned(),
1047 "blob ref is not JSON".to_owned()
1048 )
1049 );
1050 }
1051
1052 #[test]
1053 fn visible_blob_references_survive_cache_trimming() {
1054 let schema = json!({"version":1,"tables":[
1055 {"name":"attachments","primaryKey":"id","columns":[
1056 {"name":"id","type":"string","nullable":false},
1057 {"name":"body","type":"blob_ref","nullable":true},
1058 {"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
1059 {"name":"avatars","primaryKey":"id","columns":[
1060 {"name":"id","type":"string","nullable":false},
1061 {"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
1062 ]});
1063 let bytes = b"referenced above cap".to_vec();
1064 let blob_id = blob_id_for(&bytes);
1065 let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
1066 let mut client = SyncClient::new(
1067 "visible-blob-refs".to_owned(),
1068 &schema,
1069 ClientLimits {
1070 blob_cache_max_bytes: Some(1),
1071 ..Default::default()
1072 },
1073 )
1074 .unwrap();
1075 client.conn.execute(
1076 "INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
1077 rusqlite::params![reference, reference, reference, reference],
1078 ).unwrap();
1079 client
1080 .conn
1081 .execute(
1082 "INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
1083 rusqlite::params![reference],
1084 )
1085 .unwrap();
1086 client.conn.execute(
1087 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
1088 [],
1089 ).unwrap();
1090 let mut transport = CountingRealtimeTransport {
1091 blob_download: Some(bytes.clone()),
1092 ..Default::default()
1093 };
1094
1095 assert_eq!(
1096 client
1097 .fetch_blob_bytes(&mut transport, &blob_id)
1098 .unwrap()
1099 .bytes,
1100 bytes
1101 );
1102 let rows = client
1103 .conn
1104 .prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
1105 .unwrap()
1106 .query_map([], |row| row.get::<_, String>(0))
1107 .unwrap()
1108 .collect::<Result<Vec<_>, _>>()
1109 .unwrap();
1110 assert_eq!(rows, vec![blob_id]);
1111 }
1112
1113 #[test]
1114 fn realtime_connection_ownership_is_idempotent() {
1115 let mut client = client();
1116 let mut transport = CountingRealtimeTransport {
1117 connects: 0,
1118 closes: 0,
1119 ..Default::default()
1120 };
1121 client
1122 .connect_realtime(&mut transport)
1123 .expect("first connect");
1124 client
1125 .connect_realtime(&mut transport)
1126 .expect("idempotent connect");
1127 assert_eq!(transport.connects, 1);
1128 client.disconnect_realtime(&mut transport);
1129 client.disconnect_realtime(&mut transport);
1130 assert_eq!(transport.closes, 1);
1131 client
1132 .connect_realtime(&mut transport)
1133 .expect("deliberate reconnect");
1134 assert_eq!(transport.connects, 2);
1135 }
1136
1137 #[test]
1138 fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
1139 let mut client = client();
1140 client
1141 .subscribe(
1142 "repair-tasks".to_owned(),
1143 "tasks".to_owned(),
1144 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1145 None,
1146 )
1147 .expect("subscribe");
1148 {
1149 let sub = client
1150 .subs
1151 .iter_mut()
1152 .find(|sub| sub.id == "repair-tasks")
1153 .expect("subscription");
1154 sub.cursor = 42;
1155 sub.synced_once = true;
1156 let persisted = sub.clone();
1157 client.persist_sub(&persisted).unwrap();
1158 }
1159 for table in ["_syncular_base_tasks", "tasks"] {
1160 client
1161 .conn
1162 .execute(
1163 &format!(
1164 "INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
1165 ),
1166 rusqlite::params!["server-row", "p1"],
1167 )
1168 .expect("seed server row");
1169 }
1170 let pending = client
1171 .mutate(vec![Mutation::Upsert {
1172 table: "tasks".to_owned(),
1173 values: Map::from_iter([
1174 ("id".to_owned(), Value::from("offline-row")),
1175 ("project_id".to_owned(), Value::from("p1")),
1176 ]),
1177 base_version: None,
1178 }])
1179 .expect("queue offline work");
1180 client.drain_change_batches();
1181 client.drain_sync_intents();
1182
1183 assert_eq!(
1184 client
1185 .rebootstrap_local_data(&LocalDataRebootstrapInput {
1186 rebootstrap_id: "support-case-001".to_owned(),
1187 })
1188 .expect("rebootstrap"),
1189 LocalDataRebootstrapResult {
1190 already_applied: false,
1191 retained_commits: 1,
1192 reset_subscriptions: 1,
1193 }
1194 );
1195 let visible_ids = client
1196 .conn
1197 .prepare("SELECT id FROM tasks ORDER BY id")
1198 .expect("prepare visible ids")
1199 .query_map([], |row| row.get::<_, String>(0))
1200 .expect("query visible ids")
1201 .collect::<Result<Vec<_>, _>>()
1202 .expect("collect visible ids");
1203 assert_eq!(visible_ids, vec!["offline-row"]);
1204 assert_eq!(client.pending_commit_ids(), vec![pending]);
1205 assert_eq!(
1206 client
1207 .subscription_state("repair-tasks")
1208 .expect("subscription")
1209 .cursor,
1210 -1
1211 );
1212 assert!(client.upgrading());
1213 assert!(client.sync_needed());
1214 assert!(matches!(
1215 client.drain_sync_intents().as_slice(),
1216 [SyncIntent::Interactive]
1217 ));
1218 assert_eq!(client.drain_change_batches().len(), 1);
1219
1220 assert_eq!(
1221 client
1222 .rebootstrap_local_data(&LocalDataRebootstrapInput {
1223 rebootstrap_id: "support-case-001".to_owned(),
1224 })
1225 .expect("idempotent retry"),
1226 LocalDataRebootstrapResult {
1227 already_applied: true,
1228 retained_commits: 1,
1229 reset_subscriptions: 1,
1230 }
1231 );
1232 assert!(client.drain_change_batches().is_empty());
1233 }
1234
1235 #[test]
1236 fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
1237 let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
1238 assert_eq!(
1239 decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
1240 (3, 4)
1241 );
1242 assert_eq!(
1243 decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
1244 (0, 0)
1245 );
1246 for malformed in [
1247 "",
1248 "{}",
1249 r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
1250 r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
1251 r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
1252 ] {
1253 assert_eq!(
1254 decode_local_rebootstrap_receipt(malformed)
1255 .expect_err("malformed receipt must fail"),
1256 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
1257 );
1258 }
1259 }
1260
1261 #[test]
1262 fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
1263 let path = std::env::temp_dir().join(format!(
1264 "syncular-rebootstrap-receipt-{}.db",
1265 uuid::Uuid::new_v4()
1266 ));
1267 let schema = json!({
1268 "version": 1,
1269 "tables": [{
1270 "name": "tasks",
1271 "primaryKey": "id",
1272 "columns": [
1273 { "name": "id", "type": "string", "nullable": false },
1274 { "name": "project_id", "type": "string", "nullable": false }
1275 ],
1276 "scopes": [{ "pattern": "project:{project_id}" }]
1277 }]
1278 });
1279 let path_string = path.to_str().expect("UTF-8 temp path");
1280
1281 {
1282 let mut first = SyncClient::open_path(
1283 "repair-restart-client".to_owned(),
1284 &schema,
1285 ClientLimits::default(),
1286 path_string,
1287 )
1288 .expect("first open");
1289 first
1290 .subscribe(
1291 "repair-tasks".to_owned(),
1292 "tasks".to_owned(),
1293 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1294 None,
1295 )
1296 .expect("subscribe");
1297 first
1298 .mutate(vec![Mutation::Upsert {
1299 table: "tasks".to_owned(),
1300 values: Map::from_iter([
1301 ("id".to_owned(), Value::from("offline-row")),
1302 ("project_id".to_owned(), Value::from("p1")),
1303 ]),
1304 base_version: None,
1305 }])
1306 .expect("queue offline work");
1307 assert_eq!(
1308 first
1309 .rebootstrap_local_data(&LocalDataRebootstrapInput {
1310 rebootstrap_id: "restart-receipt".to_owned(),
1311 })
1312 .expect("first rebootstrap"),
1313 LocalDataRebootstrapResult {
1314 already_applied: false,
1315 retained_commits: 1,
1316 reset_subscriptions: 1,
1317 }
1318 );
1319 }
1320
1321 let mut reopened = SyncClient::open_path(
1322 "repair-restart-client".to_owned(),
1323 &schema,
1324 ClientLimits::default(),
1325 path_string,
1326 )
1327 .expect("reopen");
1328 reopened.drain_change_batches();
1329 reopened.drain_sync_intents();
1330 assert_eq!(
1331 reopened
1332 .rebootstrap_local_data(&LocalDataRebootstrapInput {
1333 rebootstrap_id: "restart-receipt".to_owned(),
1334 })
1335 .expect("receipt replay"),
1336 LocalDataRebootstrapResult {
1337 already_applied: true,
1338 retained_commits: 1,
1339 reset_subscriptions: 1,
1340 }
1341 );
1342 assert!(reopened.drain_change_batches().is_empty());
1343 assert!(reopened.drain_sync_intents().is_empty());
1344 drop(reopened);
1345 std::fs::remove_file(path).expect("remove temp database");
1346 }
1347
1348 #[test]
1349 fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
1350 let mut malformed = client();
1351 malformed
1352 .subscribe(
1353 "repair-tasks".to_owned(),
1354 "tasks".to_owned(),
1355 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
1356 None,
1357 )
1358 .expect("subscribe");
1359 malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
1360 malformed.drain_change_batches();
1361 malformed.drain_sync_intents();
1362 let malformed_error = malformed
1363 .rebootstrap_local_data(&LocalDataRebootstrapInput {
1364 rebootstrap_id: "malformed".to_owned(),
1365 })
1366 .expect_err("malformed receipt must fail closed");
1367 assert_eq!(
1368 malformed_error,
1369 "sync.local_corrupt: persisted local rebootstrap receipt is invalid"
1370 );
1371 assert!(!malformed.upgrading());
1372 assert_eq!(
1373 malformed
1374 .subscription_state("repair-tasks")
1375 .expect("unchanged subscription")
1376 .cursor,
1377 -1
1378 );
1379 assert!(malformed.drain_change_batches().is_empty());
1380 assert!(malformed.drain_sync_intents().is_empty());
1381
1382 let mut unreadable = client();
1383 unreadable
1384 .conn
1385 .execute(
1386 "INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
1387 rusqlite::params!["server-row", "p1"],
1388 )
1389 .expect("seed visible row");
1390 unreadable
1391 .conn
1392 .execute("DROP TABLE _syncular_meta", [])
1393 .expect("break marker storage");
1394 let unreadable_error = unreadable
1395 .rebootstrap_local_data(&LocalDataRebootstrapInput {
1396 rebootstrap_id: "unreadable".to_owned(),
1397 })
1398 .expect_err("unreadable marker storage must fail closed");
1399 assert_eq!(
1400 unreadable_error,
1401 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
1402 );
1403 let visible_rows = unreadable
1404 .conn
1405 .query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
1406 .expect("visible projection remains");
1407 assert_eq!(visible_rows, 1);
1408 assert!(!unreadable.upgrading());
1409 assert!(unreadable.drain_change_batches().is_empty());
1410 assert!(unreadable.drain_sync_intents().is_empty());
1411 }
1412
1413 #[test]
1414 fn local_rebootstrap_cannot_bypass_schema_floor() {
1415 let mut client = client();
1416 client.set_schema_floor(Some(SchemaFloor {
1417 required_schema_version: Some(2),
1418 latest_schema_version: Some(2),
1419 }));
1420 let error = client
1421 .rebootstrap_local_data(&LocalDataRebootstrapInput {
1422 rebootstrap_id: "blocked-floor".to_owned(),
1423 })
1424 .expect_err("schema floor must block repair");
1425 assert!(error.contains("cannot bypass an active schema-floor stop"));
1426 }
1427
1428 #[test]
1429 fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
1430 let mut client = client();
1431 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
1432 const COMMIT_COUNT: usize = 32;
1433
1434 for index in 0..COMMIT_COUNT {
1435 client
1436 .mutate(vec![Mutation::Upsert {
1437 table: "tasks".to_owned(),
1438 values: Map::from_iter([
1439 ("id".to_owned(), Value::from(format!("task-{index}"))),
1440 ("project_id".to_owned(), Value::from("project-1")),
1441 ]),
1442 base_version: None,
1443 }])
1444 .expect("queue commit");
1445 }
1446
1447 let (_, request_meta) = client.build_request(false);
1448 assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
1449 let mut frames = vec![Frame::RespHeader {
1450 required_schema_version: None,
1451 latest_schema_version: None,
1452 log_epoch: Some("epoch-1".to_owned()),
1453 reset_required: Some(false),
1454 }];
1455 frames.extend(request_meta.pushed_ids.iter().enumerate().map(
1456 |(index, client_commit_id)| Frame::PushResult {
1457 client_commit_id: client_commit_id.clone(),
1458 status: PushStatus::Applied,
1459 commit_seq: Some(index as i64 + 1),
1460 results: vec![OpResult::Applied { op_index: 0 }],
1461 },
1462 ));
1463 let response = Message {
1464 wire_version: WIRE_VERSION,
1465 msg_kind: MsgKind::Response,
1466 frames,
1467 };
1468 let mut transport =
1469 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
1470
1471 client.overlay_rebuild_count.set(0);
1472 client.outcome_prune_count.set(0);
1473 let outcome = client.process_response(&mut transport, response, &request_meta);
1474 assert!(matches!(outcome, SyncOutcome::Ok(_)));
1475 assert!(client.pending_commit_ids().is_empty());
1476 assert_eq!(
1477 client.overlay_rebuild_count.get(),
1478 1,
1479 "one response must reconcile its acknowledged commits with one overlay rebuild"
1480 );
1481 assert_eq!(
1482 client.outcome_prune_count.get(),
1483 1,
1484 "one response must enforce outcome retention once"
1485 );
1486 }
1487
1488 #[test]
1489 fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
1490 let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
1491 "columns": [
1492 {"name":"id", "type":"string", "nullable":false},
1493 {"name":"project_id", "type":"string", "nullable":false},
1494 {"name":"title", "type":"string", "nullable":false}
1495 ], "scopes": [{"pattern":"project:{project_id}"}],
1496 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
1497 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
1498 }] });
1499 let mut incremental =
1500 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
1501 let mut reference =
1502 SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
1503 let tables = vec!["tasks".to_owned()];
1504 for index in 0..32 {
1505 let id = format!("task-{}", index % 8);
1506 let values = Map::from_iter([
1507 ("id".to_owned(), Value::from(id.clone())),
1508 (
1509 "project_id".to_owned(),
1510 Value::from(format!("p{}", index % 2)),
1511 ),
1512 ("title".to_owned(), Value::from(format!("needle{index}"))),
1513 ]);
1514 let payload = encode_row_json(
1515 incremental.schema.table("tasks").unwrap(),
1516 &id,
1517 &values,
1518 &incremental.encryption,
1519 )
1520 .unwrap();
1521 let change = ssp2::model::Change {
1522 table_index: 0,
1523 row_id: id,
1524 op: if index % 5 == 0 {
1525 Op::Delete
1526 } else {
1527 Op::Upsert
1528 },
1529 row_version: Some(index + 1),
1530 scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
1531 row: Some(payload),
1532 };
1533 reference.overlay_dirty.set(true);
1534 for instance in [&mut incremental, &mut reference] {
1535 instance.begin_observation("mirror_test").unwrap();
1536 let mut batch = ChangeAccumulator::default();
1537 instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
1538 instance
1539 .apply_commit_changes(&tables, std::slice::from_ref(&change))
1540 .unwrap();
1541 instance.rebuild_overlay_if_dirty();
1542 instance.finish_observation("mirror_test", batch).unwrap();
1543 }
1544 for sql in [
1545 "SELECT * FROM tasks ORDER BY id",
1546 "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
1547 ] {
1548 assert_eq!(
1549 incremental.query(sql, &[]).unwrap(),
1550 reference.query(sql, &[]).unwrap()
1551 );
1552 }
1553 assert_eq!(incremental.local_revision(), reference.local_revision());
1554 assert_eq!(
1555 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
1556 serde_json::to_value(reference.drain_change_batches()).unwrap()
1557 );
1558 }
1559 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
1560 assert_eq!(reference.overlay_rebuild_count.get(), 32);
1561 }
1562
1563 #[test]
1564 fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
1565 let cases = [
1566 (
1567 "string",
1568 vec![
1569 json!("1"),
1570 json!("01"),
1571 json!(""),
1572 json!("nul\u{0}id"),
1573 json!("é"),
1574 ],
1575 ),
1576 (
1577 "integer",
1578 vec![
1579 json!(i64::MIN),
1580 json!(-1),
1581 json!(0),
1582 json!(1),
1583 json!(i64::MAX),
1584 ],
1585 ),
1586 ("boolean", vec![json!(false), json!(true)]),
1587 ("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
1588 (
1589 "float",
1590 vec![
1591 json!(1.0),
1592 json!(1.0000000000000002),
1593 json!(-1.0),
1594 json!(1e20),
1595 ],
1596 ),
1597 ];
1598 for (kind, ids) in cases {
1599 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
1600 "columns":[{"name":"id","type":kind,"nullable":false},
1601 {"name":"project_id","type":"string","nullable":false}],
1602 "scopes":[{"pattern":"project:{project_id}"}]}]});
1603 let mut client =
1604 SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
1605 for (index, id) in ids.iter().enumerate() {
1606 let table = client.schema.table("tasks").unwrap();
1607 client
1608 .write_base_row(
1609 "tasks",
1610 &vec![
1611 json_to_column_value(&table.columns[0], Some(id)).unwrap(),
1612 Some(ColumnValue::String(format!("p{index}"))),
1613 ],
1614 1,
1615 )
1616 .unwrap();
1617 }
1618 client.rebuild_overlay_if_dirty();
1619 for base in [true, false] {
1620 let full = if base {
1621 base_table("tasks")
1622 } else {
1623 visible_table("tasks")
1624 };
1625 let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
1626 for row_id in [
1627 "1",
1628 "01",
1629 "1.0",
1630 "1.0000000000000002",
1631 "+1",
1632 "1e0",
1633 " 1",
1634 "-1",
1635 "0",
1636 "9223372036854775807",
1637 "-9223372036854775808",
1638 "true",
1639 "false",
1640 "",
1641 "nul\u{0}id",
1642 "é",
1643 "null",
1644 "{\"a\":1}",
1645 ] {
1646 let old_sql =
1647 format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
1648 let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
1649 let old = client
1650 .conn
1651 .prepare(&old_sql)
1652 .unwrap()
1653 .query_map([row_id], |row| row.get::<_, String>(0))
1654 .unwrap()
1655 .collect::<Result<Vec<_>, _>>()
1656 .unwrap();
1657 let new = client
1658 .conn
1659 .prepare_cached(&new_sql)
1660 .unwrap()
1661 .query_map([row_id], |row| row.get::<_, String>(0))
1662 .unwrap()
1663 .collect::<Result<Vec<_>, _>>()
1664 .unwrap();
1665 assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
1666 if kind == "float" && row_id == "1.0" {
1669 assert_eq!(old.len(), 2);
1670 }
1671 let mut batch = ChangeAccumulator::default();
1672 assert_eq!(
1673 client.record_row_scopes(&mut batch, "tasks", row_id, base),
1674 !old.is_empty()
1675 );
1676 let expected = old.first().map(|scope| {
1677 BTreeMap::from([(
1678 "tasks".to_owned(),
1679 Some(BTreeSet::from([format!("project:{scope}")])),
1680 )])
1681 });
1682 assert_eq!(batch.tables, expected.unwrap_or_default());
1683 client
1684 .conn
1685 .execute_batch("SAVEPOINT lookup_delete")
1686 .unwrap();
1687 if base {
1688 client.delete_base_row("tasks", row_id).unwrap();
1689 } else {
1690 client.apply_outbox_ops(&[OutboxOp {
1691 upsert: false,
1692 table: "tasks".into(),
1693 row_id: row_id.into(),
1694 base_version: None,
1695 values: None,
1696 }]);
1697 }
1698 let remaining: i64 = client
1699 .conn
1700 .query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
1701 row.get(0)
1702 })
1703 .unwrap();
1704 assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
1705 client
1706 .conn
1707 .execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
1708 .unwrap();
1709 client.overlay_dirty.set(false);
1710 }
1711 for (predicate, expected) in [
1712 ("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
1713 (predicate, if kind == "float" { "SCAN" } else { "SEARCH" }),
1714 ] {
1715 let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
1716 let details = client
1717 .conn
1718 .prepare(&sql)
1719 .unwrap()
1720 .query_map(["1"], |row| row.get::<_, String>(3))
1721 .unwrap()
1722 .collect::<Result<Vec<_>, _>>()
1723 .unwrap();
1724 assert!(
1725 details.iter().any(|detail| detail.contains(expected)),
1726 "{kind}: {details:?}"
1727 );
1728 }
1729 }
1730 }
1731 }
1732
1733 #[test]
1734 fn pending_row_reconciliation_matches_full_fifo_replay() {
1735 for unique in [false, true] {
1736 let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
1737 "name": name, "primaryKey": "id", "columns": [
1738 {"name":"id", "type":"string", "nullable":false},
1739 {"name":"project_id", "type":"string", "nullable":false},
1740 {"name":"title", "type":"string", "nullable":false}],
1741 "scopes": [{"pattern":"project:{project_id}"}],
1742 "indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
1743 "ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
1744 })))});
1745 let mut incremental =
1746 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
1747 let mut reference =
1748 SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
1749 let tables = vec!["tasks".to_owned(), "notes".to_owned()];
1750 for instance in [&mut incremental, &mut reference] {
1751 for table in &tables {
1752 for index in 0..12 {
1753 instance
1754 .write_base_row(
1755 table,
1756 &vec![
1757 Some(ColumnValue::String(format!("row{index}"))),
1758 Some(ColumnValue::String("p0".into())),
1759 Some(ColumnValue::String(format!("base{index}"))),
1760 ],
1761 1,
1762 )
1763 .unwrap();
1764 }
1765 }
1766 instance.rebuild_overlay_if_dirty();
1767 }
1768 for index in 0..80 {
1769 let mutations = (0..3)
1770 .map(|offset| {
1771 let table = tables[(index + offset) % 2].clone();
1772 let id = format!("row{}", (index + offset) % 16);
1773 if (index + offset) % 7 == 0 {
1774 Mutation::Delete {
1775 table,
1776 row_id: id,
1777 base_version: None,
1778 }
1779 } else {
1780 Mutation::Upsert {
1781 table,
1782 values: Map::from_iter([
1783 ("id".into(), Value::from(id)),
1784 ("project_id".into(), Value::from(format!("p{}", index % 3))),
1785 (
1786 "title".into(),
1787 Value::from(format!("needle{}", (index + offset) % 5)),
1788 ),
1789 ]),
1790 base_version: None,
1791 }
1792 }
1793 })
1794 .collect::<Vec<_>>();
1795 for instance in [&mut incremental, &mut reference] {
1796 instance.mutate(mutations.clone()).unwrap();
1797 }
1798 let changes = (0..3)
1801 .map(|offset| {
1802 let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
1803 let id = format!("row{}", (index + offset) % 16);
1804 let values = Map::from_iter([
1805 ("id".into(), Value::from(id.clone())),
1806 (
1807 "project_id".into(),
1808 Value::from(format!("p{}", (index + 1) % 3)),
1809 ),
1810 (
1811 "title".into(),
1812 Value::from(format!("remote{index}_{offset}")),
1813 ),
1814 ]);
1815 ssp2::model::Change {
1816 table_index: table_index as u16,
1817 row_id: id.clone(),
1818 op: if (index + offset) % 5 == 0 {
1819 Op::Delete
1820 } else {
1821 Op::Upsert
1822 },
1823 row_version: Some(index as i64 + 2),
1824 scopes: vec![],
1825 row: Some(
1826 encode_row_json(
1827 incremental.schema.table(&tables[table_index]).unwrap(),
1828 &id,
1829 &values,
1830 &incremental.encryption,
1831 )
1832 .unwrap(),
1833 ),
1834 }
1835 })
1836 .collect::<Vec<_>>();
1837 reference.overlay_dirty.set(true);
1838 incremental.apply_commit_frame(&tables, &changes).unwrap();
1839 reference.apply_commit_frame(&tables, &changes).unwrap();
1840 for table in &tables {
1841 for sql in [format!("SELECT * FROM {table} ORDER BY id"),
1842 format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
1843 format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
1844 format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
1845 assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
1846 }
1847 }
1848 assert_eq!(incremental.local_revision(), reference.local_revision());
1849 assert_eq!(
1850 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
1851 serde_json::to_value(reference.drain_change_batches()).unwrap()
1852 );
1853 }
1854 assert_eq!(
1855 incremental.overlay_rebuild_count.get(),
1856 if unique { 81 } else { 21 }
1857 );
1858 assert_eq!(reference.overlay_rebuild_count.get(), 81);
1859 }
1860 }
1861
1862 #[test]
1863 fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
1864 let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
1865 "columns":[{"name":"id", "type":"string", "nullable":false},
1866 {"name":"title", "type":"string", "nullable":false}],
1867 "scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
1868 let mut client =
1869 SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
1870 for (id, title) in [("a", "original"), ("b", "occupied")] {
1871 client
1872 .write_base_row(
1873 "tasks",
1874 &vec![
1875 Some(ColumnValue::String(id.into())),
1876 Some(ColumnValue::String(title.into())),
1877 ],
1878 1,
1879 )
1880 .unwrap();
1881 }
1882 client.rebuild_overlay_if_dirty();
1883 let pending = client
1884 .mutate(vec![Mutation::Upsert {
1885 table: "tasks".into(),
1886 values: Map::from_iter([
1887 ("id".into(), Value::from("a")),
1888 ("title".into(), Value::from("occupied")),
1889 ]),
1890 base_version: None,
1891 }])
1892 .unwrap();
1893 assert_eq!(
1894 client
1895 .conn
1896 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
1897 .get::<_, String>(
1898 0
1899 ))
1900 .unwrap(),
1901 "original"
1902 );
1903 for (title, expected) in [("available", "occupied"), ("occupied", "original")] {
1904 let change = ssp2::model::Change {
1905 table_index: 0,
1906 row_id: "b".into(),
1907 op: Op::Upsert,
1908 row_version: Some(2),
1909 scopes: vec![],
1910 row: Some(
1911 encode_row_json(
1912 client.schema.table("tasks").unwrap(),
1913 "b",
1914 &Map::from_iter([
1915 ("id".into(), Value::from("b")),
1916 ("title".into(), Value::from(title)),
1917 ]),
1918 &client.encryption,
1919 )
1920 .unwrap(),
1921 ),
1922 };
1923 client
1924 .apply_commit_frame(&["tasks".into()], &[change])
1925 .unwrap();
1926 assert_eq!(
1927 client
1928 .conn
1929 .query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
1930 .get::<_, String>(
1931 0
1932 ))
1933 .unwrap(),
1934 expected
1935 );
1936 assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
1937 }
1938 assert_eq!(client.overlay_rebuild_count.get(), 3);
1939 }
1940
1941 #[test]
1942 fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
1943 for failure in ["visible", "revision", "commit"] {
1944 let mut client = client();
1945 let id = client
1946 .mutate(vec![Mutation::Upsert {
1947 table: "tasks".into(),
1948 values: Map::from_iter([
1949 ("id".into(), Value::from("t1")),
1950 ("project_id".into(), Value::from("pending")),
1951 ]),
1952 base_version: None,
1953 }])
1954 .unwrap();
1955 client.drain_change_batches();
1956 let revision = client.local_revision();
1957 let before = client.query("SELECT * FROM tasks", &[]).unwrap();
1958 let trigger = match failure {
1959 "visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
1960 "revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
1961 _ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
1962 };
1963 client.conn.execute_batch(&trigger).unwrap();
1964 let change = ssp2::model::Change {
1965 table_index: 0,
1966 row_id: "t1".into(),
1967 op: Op::Upsert,
1968 row_version: Some(2),
1969 scopes: vec![],
1970 row: Some(
1971 encode_row_json(
1972 client.schema.table("tasks").unwrap(),
1973 "t1",
1974 &Map::from_iter([
1975 ("id".into(), Value::from("t1")),
1976 ("project_id".into(), Value::from("remote")),
1977 ]),
1978 &client.encryption,
1979 )
1980 .unwrap(),
1981 ),
1982 };
1983 assert!(client
1984 .apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change))
1985 .is_err());
1986 assert!(client.conn.is_autocommit());
1987 assert!(!client.overlay_dirty.get());
1988 assert_eq!(client.local_revision(), revision);
1989 assert!(client.drain_change_batches().is_empty());
1990 assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
1991 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
1992 assert!(client
1993 .query("SELECT * FROM _syncular_base_tasks", &[])
1994 .unwrap()
1995 .is_empty());
1996 client
1997 .conn
1998 .execute_batch("DROP TRIGGER fail_pending")
1999 .unwrap();
2000 client
2001 .apply_commit_frame(&["tasks".into()], &[change])
2002 .unwrap();
2003 assert_eq!(client.local_revision(), revision + 1);
2004 assert_eq!(client.pending_commit_ids(), vec![id]);
2005 let mut expected = before[0].clone();
2008 expected.insert("_syncular_version".to_owned(), Value::from(2));
2009 assert_eq!(
2010 client.query("SELECT * FROM tasks", &[]).unwrap(),
2011 vec![expected]
2012 );
2013 assert_eq!(
2014 client
2015 .conn
2016 .query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
2017 .get::<_, String>(0))
2018 .unwrap(),
2019 "remote"
2020 );
2021 }
2022 }
2023
2024 #[test]
2025 fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
2026 let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
2027 "columns": [
2028 {"name":"id", "type":"string", "nullable":false},
2029 {"name":"project_id", "type":"string", "nullable":false},
2030 {"name":"title", "type":"string", "nullable":false}],
2031 "scopes": [{"pattern":"project:{project_id}"}],
2032 "indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
2033 "ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
2034 let mut incremental =
2035 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
2036 let mut reference =
2037 SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
2038 for instance in [&mut incremental, &mut reference] {
2039 for index in 0..16 {
2040 instance
2041 .write_base_row(
2042 "tasks",
2043 &vec![
2044 Some(ColumnValue::String(format!("task{index}"))),
2045 Some(ColumnValue::String("p0".to_owned())),
2046 Some(ColumnValue::String(format!("base{index}"))),
2047 ],
2048 1,
2049 )
2050 .unwrap();
2051 }
2052 instance.rebuild_overlay_if_dirty();
2053 instance.overlay_rebuild_count.set(0);
2054 }
2055 for index in 0..100 {
2056 let mutations = (0..1 + index % 4)
2057 .map(|offset| {
2058 let id = format!("task{}", (index + offset) % 24);
2059 if (index + offset) % 7 == 0 {
2060 Mutation::Delete {
2061 table: "tasks".to_owned(),
2062 row_id: id,
2063 base_version: None,
2064 }
2065 } else {
2066 Mutation::Upsert {
2067 table: "tasks".to_owned(),
2068 values: Map::from_iter([
2069 ("id".to_owned(), Value::from(id)),
2070 (
2071 "project_id".to_owned(),
2072 Value::from(format!("p{}", index % 2)),
2073 ),
2074 (
2075 "title".to_owned(),
2076 Value::from(format!("needle{}", (index + offset) % 5)),
2077 ),
2078 ]),
2079 base_version: None,
2080 }
2081 }
2082 })
2083 .collect::<Vec<_>>();
2084 reference.overlay_dirty.set(true);
2085 incremental.mutate(mutations.clone()).unwrap();
2086 reference.mutate(mutations).unwrap();
2087 for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
2088 "SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
2089 assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
2090 }
2091 assert_eq!(incremental.local_revision(), reference.local_revision());
2092 assert_eq!(
2093 serde_json::to_value(incremental.drain_change_batches()).unwrap(),
2094 serde_json::to_value(reference.drain_change_batches()).unwrap()
2095 );
2096 }
2097 assert_eq!(incremental.outbox.len(), 100);
2098 assert_eq!(incremental.overlay_rebuild_count.get(), 0);
2099 assert_eq!(reference.overlay_rebuild_count.get(), 100);
2100 }
2101
2102 #[test]
2103 fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
2104 for fail_at in ["outbox", "revision"] {
2105 let mut client = client();
2106 let values = Map::from_iter([
2107 ("id".to_owned(), Value::from("t1")),
2108 ("project_id".to_owned(), Value::from("p1")),
2109 ]);
2110 let mutation = Mutation::Upsert {
2111 table: "tasks".to_owned(),
2112 values,
2113 base_version: None,
2114 };
2115 let trigger = if fail_at == "outbox" {
2116 "CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
2117 } else {
2118 format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
2119 };
2120 client.conn.execute_batch(&trigger).unwrap();
2121 let revision = client.local_revision();
2122 assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
2123 assert_eq!(client.local_revision(), revision);
2124 assert!(client.outbox.is_empty());
2125 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
2126 assert_eq!(
2127 client
2128 .conn
2129 .query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
2130 .get::<_, i64>(0))
2131 .unwrap(),
2132 0
2133 );
2134 assert!(client.drain_change_batches().is_empty());
2135 assert!(!client.overlay_dirty.get());
2136 client
2137 .conn
2138 .execute_batch("DROP TRIGGER fail_append")
2139 .unwrap();
2140 client.mutate(vec![mutation]).unwrap();
2141 assert_eq!(client.outbox.len(), 1);
2142 assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
2143 }
2144 }
2145
2146 #[test]
2147 fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
2148 let mut client = client();
2149 let tables = vec!["tasks".to_owned(), "missing".to_owned()];
2150 let values = Map::from_iter([
2151 ("id".to_owned(), Value::from("t1")),
2152 ("project_id".to_owned(), Value::from("p1")),
2153 ]);
2154 let payload = encode_row_json(
2155 client.schema.table("tasks").unwrap(),
2156 "t1",
2157 &values,
2158 &client.encryption,
2159 )
2160 .unwrap();
2161 let first = ssp2::model::Change {
2162 table_index: 0,
2163 row_id: "t1".to_owned(),
2164 op: Op::Upsert,
2165 row_version: Some(1),
2166 scopes: vec![("project_id".to_owned(), "p1".to_owned())],
2167 row: Some(payload),
2168 };
2169 let mut invalid = first.clone();
2170 invalid.table_index = 1;
2171 let revision = client.local_revision();
2172 client.begin_observation("failed_mirror").unwrap();
2173 assert!(client
2174 .apply_commit_changes(&tables, &[first, invalid])
2175 .is_err());
2176 client.rollback_observation("failed_mirror");
2177 client.rebuild_overlay_if_dirty();
2178 assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
2179 assert_eq!(
2180 client
2181 .conn
2182 .query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
2183 .get::<_, i64>(
2184 0
2185 ))
2186 .unwrap(),
2187 0
2188 );
2189 assert_eq!(client.local_revision(), revision);
2190 assert!(client.drain_change_batches().is_empty());
2191 }
2192
2193 #[test]
2194 fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
2195 for failure in ["journal", "revision", "commit", "later-frame"] {
2196 let mut client = client();
2197 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2198 for id in ["first", "second"] {
2199 client
2200 .mutate(vec![Mutation::Upsert {
2201 table: "tasks".to_owned(),
2202 values: Map::from_iter([
2203 ("id".to_owned(), Value::from(id)),
2204 ("project_id".to_owned(), Value::from("p1")),
2205 ]),
2206 base_version: None,
2207 }])
2208 .unwrap();
2209 }
2210 let (_, meta) = client.build_request(false);
2211 client
2213 .mutate(vec![Mutation::Upsert {
2214 table: "tasks".to_owned(),
2215 values: Map::from_iter([
2216 ("id".to_owned(), Value::from("later")),
2217 ("project_id".to_owned(), Value::from("p1")),
2218 ]),
2219 base_version: None,
2220 }])
2221 .unwrap();
2222 let ids = client.pending_commit_ids();
2223 let rows = client
2224 .query("SELECT * FROM tasks ORDER BY id", &[])
2225 .unwrap();
2226 let revision = client.local_revision();
2227 client.drain_change_batches();
2228 if failure != "later-frame" {
2229 client.conn.execute_batch(match failure {
2230 "journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
2231 "revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
2232 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
2233 }).unwrap();
2234 }
2235 let mut response = Message {
2236 wire_version: WIRE_VERSION,
2237 msg_kind: MsgKind::Response,
2238 frames: vec![Frame::RespHeader {
2239 required_schema_version: None,
2240 latest_schema_version: None,
2241 log_epoch: Some("epoch-1".to_owned()),
2242 reset_required: Some(false),
2243 }],
2244 };
2245 for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
2246 response.frames.push(Frame::PushResult {
2247 client_commit_id: id.to_owned(),
2248 status: PushStatus::Applied,
2249 commit_seq: Some(1),
2250 results: vec![OpResult::Applied { op_index: 0 }],
2251 });
2252 }
2253 if failure == "later-frame" {
2254 response.frames.push(Frame::Error {
2255 code: "sync.invalid_request".to_owned(),
2256 message: "later error".to_owned(),
2257 category: "protocol".to_owned(),
2258 retryable: false,
2259 recommended_action: "retry".to_owned(),
2260 details: None,
2261 });
2262 }
2263 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
2264 let outcome = client.process_response(&mut transport, response.clone(), &meta);
2265 assert!(
2266 matches!(outcome, SyncOutcome::Failed { error_code, .. } if error_code == if failure == "later-frame" { "sync.invalid_request" } else { "client.outcome_persistence_failed" })
2267 );
2268 if failure == "later-frame" {
2269 assert_eq!(client.pending_commit_ids(), ids[2..]);
2270 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
2271 assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
2272 let batches = client.drain_change_batches();
2273 assert_eq!(batches.len(), 1);
2274 assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
2275 assert_eq!(client.local_revision(), revision + 1);
2276 response.frames.pop();
2277 } else {
2278 assert_eq!(client.pending_commit_ids(), ids);
2279 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
2280 assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
2281 assert_eq!(client.local_revision(), revision);
2282 assert!(client.drain_change_batches().is_empty());
2283 assert_eq!(
2284 client
2285 .query("SELECT * FROM tasks ORDER BY id", &[])
2286 .unwrap(),
2287 rows
2288 );
2289 client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
2290 }
2291 for frame in &mut response.frames {
2292 if let Frame::PushResult { status, .. } = frame {
2293 *status = PushStatus::Cached;
2294 }
2295 }
2296 let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
2297 else {
2298 panic!("retry failed");
2299 };
2300 assert_eq!(
2301 report.applied,
2302 if failure == "later-frame" {
2303 vec![]
2304 } else {
2305 ids[..2].to_vec()
2306 }
2307 );
2308 assert_eq!(client.pending_commit_ids(), ids[2..]);
2309 let batches = client.drain_change_batches();
2310 assert_eq!(batches.len(), usize::from(failure != "later-frame"));
2311 if let Some(batch) = batches.first() {
2312 assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
2313 }
2314 assert_eq!(client.local_revision(), revision + 1);
2315 }
2316 }
2317
2318 #[test]
2319 fn failed_push_result_preserves_memory_disk_and_retry_identity() {
2320 for failure in ["revision", "journal", "commit"] {
2321 for status in ["applied", "cached", "rejected", "conflict"] {
2322 let mut client = client();
2323 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2324 for id in ["first", "later"] {
2325 client
2326 .mutate(vec![Mutation::Upsert {
2327 table: "tasks".to_owned(),
2328 values: Map::from_iter([
2329 ("id".to_owned(), Value::from(id)),
2330 ("project_id".to_owned(), Value::from("p1")),
2331 ]),
2332 base_version: None,
2333 }])
2334 .unwrap();
2335 }
2336 let (_, meta) = client.build_request(false);
2337 let ids = client.pending_commit_ids();
2338 let rows = client
2339 .query("SELECT * FROM tasks ORDER BY id", &[])
2340 .unwrap();
2341 let revision = client.local_revision();
2342 client.drain_change_batches();
2343 let (push_status, results) = match status {
2344 "applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
2345 "cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
2346 "conflict" => (
2347 PushStatus::Rejected,
2348 vec![OpResult::Conflict {
2349 op_index: 0,
2350 code: "sync.version_conflict".to_owned(),
2351 message: "conflict".to_owned(),
2352 server_version: 1,
2353 server_row: encode_row_json(
2354 client.schema.table("tasks").unwrap(),
2355 "first",
2356 &Map::from_iter([
2357 ("id".to_owned(), Value::from("first")),
2358 ("project_id".to_owned(), Value::from("p1")),
2359 ]),
2360 &client.encryption,
2361 )
2362 .unwrap(),
2363 conflict_columns: Vec::new(),
2364 }],
2365 ),
2366 _ => (
2367 PushStatus::Rejected,
2368 vec![OpResult::Error {
2369 op_index: 0,
2370 code: "sync.validation_failed".to_owned(),
2371 message: "rejected".to_owned(),
2372 retryable: false,
2373 }],
2374 ),
2375 };
2376 let response = Message {
2377 wire_version: WIRE_VERSION,
2378 msg_kind: MsgKind::Response,
2379 frames: vec![
2380 Frame::RespHeader {
2381 required_schema_version: None,
2382 latest_schema_version: None,
2383 log_epoch: Some("epoch-1".to_owned()),
2384 reset_required: Some(false),
2385 },
2386 Frame::PushResult {
2387 client_commit_id: ids[0].clone(),
2388 status: push_status,
2389 commit_seq: Some(1),
2390 results,
2391 },
2392 ],
2393 };
2394 client.conn.execute_batch(match failure {
2395 "journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
2396 "revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
2397 _ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
2398 }).unwrap();
2399 let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
2400 let outcome = client.process_response(&mut transport, response.clone(), &meta);
2401 assert_eq!(
2402 client.pending_commit_ids(),
2403 ids,
2404 "{failure}/{status}: pending identity"
2405 );
2406 assert_eq!(
2407 client.local_revision(),
2408 revision,
2409 "{failure}/{status}: revision"
2410 );
2411 assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
2412 assert!(client.conflicts().is_empty());
2413 assert!(client.rejections().is_empty());
2414 assert!(client.drain_change_batches().is_empty());
2415 assert_eq!(
2416 client
2417 .query("SELECT * FROM tasks ORDER BY id", &[])
2418 .unwrap(),
2419 rows
2420 );
2421 assert!(
2422 matches!(outcome, SyncOutcome::Failed { error_code, .. } if error_code == "client.outcome_persistence_failed"),
2423 "{failure}/{status}: failed round"
2424 );
2425 client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
2426 assert!(matches!(
2427 client.process_response(&mut transport, response, &meta),
2428 SyncOutcome::Ok(_)
2429 ));
2430 assert_eq!(client.pending_commit_ids(), ids[1..]);
2431 assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
2432 assert_eq!(client.drain_change_batches().len(), 1);
2433 }
2434 }
2435 }
2436
2437 #[test]
2438 fn mixed_push_results_reconcile_and_prune_once_per_response() {
2439 let mut client = client();
2440 client.set_meta(LOG_EPOCH_KEY, "epoch-1");
2441 for index in 0..4 {
2442 client
2443 .mutate(vec![Mutation::Upsert {
2444 table: "tasks".to_owned(),
2445 values: Map::from_iter([
2446 ("id".to_owned(), Value::from(format!("task-{index}"))),
2447 ("project_id".to_owned(), Value::from("project-1")),
2448 ]),
2449 base_version: None,
2450 }])
2451 .expect("queue commit");
2452 }
2453
2454 let (_, request_meta) = client.build_request(false);
2455 let ids = &request_meta.pushed_ids;
2456 assert_eq!(ids.len(), 4);
2457 let response = Message {
2458 wire_version: WIRE_VERSION,
2459 msg_kind: MsgKind::Response,
2460 frames: vec![
2461 Frame::RespHeader {
2462 required_schema_version: None,
2463 latest_schema_version: None,
2464 log_epoch: Some("epoch-1".to_owned()),
2465 reset_required: Some(false),
2466 },
2467 Frame::PushResult {
2468 client_commit_id: ids[0].clone(),
2469 status: PushStatus::Applied,
2470 commit_seq: Some(1),
2471 results: vec![OpResult::Applied { op_index: 0 }],
2472 },
2473 Frame::PushResult {
2474 client_commit_id: ids[1].clone(),
2475 status: PushStatus::Cached,
2476 commit_seq: Some(2),
2477 results: vec![OpResult::Applied { op_index: 0 }],
2478 },
2479 Frame::PushResult {
2480 client_commit_id: ids[2].clone(),
2481 status: PushStatus::Rejected,
2482 commit_seq: None,
2483 results: vec![OpResult::Error {
2484 op_index: 0,
2485 code: "sync.validation_failed".to_owned(),
2486 message: "rejected".to_owned(),
2487 retryable: false,
2488 }],
2489 },
2490 Frame::PushResult {
2491 client_commit_id: ids[3].clone(),
2492 status: PushStatus::Rejected,
2493 commit_seq: None,
2494 results: vec![OpResult::Error {
2495 op_index: 0,
2496 code: "sync.idempotency_cache_miss".to_owned(),
2497 message: "retry".to_owned(),
2498 retryable: true,
2499 }],
2500 },
2501 ],
2502 };
2503 let mut transport =
2504 HostTransport::new_from_config(&json!({})).expect("no-network host transport");
2505
2506 client.overlay_rebuild_count.set(0);
2507 client.outcome_prune_count.set(0);
2508 let outcome = client.process_response(&mut transport, response, &request_meta);
2509 let SyncOutcome::Ok(report) = outcome else {
2510 panic!("mixed push-result response failed");
2511 };
2512
2513 assert_eq!(report.applied, ids[..2]);
2514 assert_eq!(report.rejected, ids[2..3]);
2515 assert_eq!(report.retryable, ids[3..4]);
2516 assert_eq!(client.pending_commit_ids(), ids[3..4]);
2517 assert_eq!(
2518 client
2519 .query("SELECT id FROM tasks ORDER BY id", &[])
2520 .expect("query visible overlay"),
2521 vec![Map::from_iter([("id".to_owned(), Value::from("task-3"))])]
2522 );
2523 assert_eq!(client.overlay_rebuild_count.get(), 1);
2524 assert_eq!(client.outcome_prune_count.get(), 1);
2525 }
2526
2527 #[test]
2528 fn secondary_unique_collision_preserves_existing_synced_row() {
2529 let client = SyncClient::new(
2530 "unique-upsert-test".to_owned(),
2531 &json!({
2532 "version": 1,
2533 "tables": [{
2534 "name": "tasks",
2535 "primaryKey": "id",
2536 "columns": [
2537 { "name": "id", "type": "string", "nullable": false },
2538 { "name": "project_id", "type": "string", "nullable": false },
2539 { "name": "title", "type": "string", "nullable": false }
2540 ],
2541 "scopes": [{ "pattern": "project:{project_id}" }],
2542 "indexes": [{
2543 "name": "tasks_by_project_title",
2544 "columns": ["project_id", "title"],
2545 "unique": true
2546 }]
2547 }]
2548 }),
2549 ClientLimits::default(),
2550 )
2551 .expect("test client");
2552 let table = client.schema.table("tasks").expect("tasks table");
2553 let sql = client.insert_row_sql(&base_table("tasks"), table);
2554
2555 client
2556 .conn
2557 .execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
2558 .expect("insert first row");
2559 client
2560 .conn
2561 .execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
2562 .expect("update same primary key");
2563 client
2564 .conn
2565 .execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
2566 .expect("insert second row");
2567 assert!(client
2568 .conn
2569 .execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
2570 .is_err());
2571
2572 let rows = client
2573 .conn
2574 .prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
2575 .expect("prepare rows")
2576 .query_map([], |row| {
2577 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
2578 })
2579 .expect("query rows")
2580 .collect::<Result<Vec<_>, _>>()
2581 .expect("collect rows");
2582 assert_eq!(
2583 rows,
2584 vec![
2585 ("t1".to_owned(), "updated".to_owned()),
2586 ("t2".to_owned(), "original".to_owned())
2587 ]
2588 );
2589 }
2590
2591 #[test]
2592 fn reopening_active_subscriptions_emits_a_catch_up_intent() {
2593 let path = std::env::temp_dir().join(format!(
2594 "syncular-startup-intent-{}.db",
2595 uuid::Uuid::new_v4()
2596 ));
2597 let schema = json!({
2598 "version": 1,
2599 "tables": [{
2600 "name": "tasks",
2601 "primaryKey": "id",
2602 "columns": [
2603 { "name": "id", "type": "string", "nullable": false },
2604 { "name": "project_id", "type": "string", "nullable": false }
2605 ],
2606 "scopes": [{ "pattern": "project:{project_id}" }]
2607 }]
2608 });
2609
2610 {
2611 let mut first = SyncClient::open_path_with_identity(
2612 None,
2613 &schema,
2614 ClientLimits::default(),
2615 path.to_str().expect("UTF-8 temp path"),
2616 )
2617 .expect("first open");
2618 first
2619 .set_window(
2620 &WindowBase {
2621 table: "tasks".to_owned(),
2622 variable: "project_id".to_owned(),
2623 fixed_scopes: Vec::new(),
2624 params: None,
2625 },
2626 &["persisted".to_owned()],
2627 )
2628 .expect("persist window");
2629 }
2630
2631 let mut reopened = SyncClient::open_path_with_identity(
2632 None,
2633 &schema,
2634 ClientLimits::default(),
2635 path.to_str().expect("UTF-8 temp path"),
2636 )
2637 .expect("reopen");
2638 assert!(reopened.sync_needed());
2639 assert!(matches!(
2640 reopened.drain_sync_intents().as_slice(),
2641 [SyncIntent::Interactive]
2642 ));
2643 drop(reopened);
2644 std::fs::remove_file(path).expect("remove temp database");
2645 }
2646
2647 #[test]
2648 fn reopening_preserves_immutable_subscription_identity_and_progress() {
2649 let path = std::env::temp_dir().join(format!(
2650 "syncular-subscription-identity-{}.db",
2651 uuid::Uuid::new_v4()
2652 ));
2653 let schema = json!({
2654 "version": 1,
2655 "tables": [
2656 {
2657 "name": "tasks",
2658 "primaryKey": "id",
2659 "columns": [
2660 { "name": "id", "type": "string", "nullable": false },
2661 { "name": "project_id", "type": "string", "nullable": false }
2662 ],
2663 "scopes": [{ "pattern": "project:{project_id}" }]
2664 },
2665 {
2666 "name": "docs",
2667 "primaryKey": "id",
2668 "columns": [
2669 { "name": "id", "type": "string", "nullable": false },
2670 { "name": "org_id", "type": "string", "nullable": false },
2671 { "name": "project_id", "type": "string", "nullable": false }
2672 ],
2673 "scopes": [
2674 { "pattern": "org:{org_id}" },
2675 { "pattern": "project:{projectId}", "column": "project_id" }
2676 ]
2677 }
2678 ]
2679 });
2680
2681 {
2682 let mut first = SyncClient::open_path_with_identity(
2683 None,
2684 &schema,
2685 ClientLimits::default(),
2686 path.to_str().expect("UTF-8 temp path"),
2687 )
2688 .expect("first open");
2689 first
2690 .subscribe(
2691 "stable-subscription".to_owned(),
2692 "tasks".to_owned(),
2693 vec![(
2694 "project_id".to_owned(),
2695 vec!["p2".to_owned(), "p1".to_owned()],
2696 )],
2697 Some(r#"{"view":"v1"}"#.to_owned()),
2698 )
2699 .expect("persist subscription");
2700 let persisted = {
2701 let subscription = first
2702 .subs
2703 .iter_mut()
2704 .find(|subscription| subscription.id == "stable-subscription")
2705 .expect("subscription");
2706 subscription.cursor = 41;
2707 subscription.bootstrap_state = Some("resume-token".to_owned());
2708 subscription.effective = Some(vec![(
2709 "project_id".to_owned(),
2710 vec!["p1".to_owned(), "p2".to_owned()],
2711 )]);
2712 subscription.synced_once = true;
2713 subscription.clone()
2714 };
2715 first.persist_sub(&persisted).unwrap();
2716 }
2717
2718 let mut reopened = SyncClient::open_path_with_identity(
2719 None,
2720 &schema,
2721 ClientLimits::default(),
2722 path.to_str().expect("UTF-8 temp path"),
2723 )
2724 .expect("reopen");
2725 let progress = reopened
2726 .subscription_state("stable-subscription")
2727 .expect("persisted state");
2728 assert_eq!(progress.cursor, 41);
2729 assert!(progress.has_resume_token);
2730
2731 reopened
2732 .subscribe(
2733 "stable-subscription".to_owned(),
2734 "tasks".to_owned(),
2735 vec![(
2736 "project_id".to_owned(),
2737 vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
2738 )],
2739 Some(r#"{"view":"v1"}"#.to_owned()),
2740 )
2741 .expect("canonical intent is idempotent");
2742 assert_eq!(
2743 reopened
2744 .subscription_state("stable-subscription")
2745 .expect("unchanged state")
2746 .cursor,
2747 progress.cursor
2748 );
2749
2750 for (table, scopes, params) in [
2751 (
2752 "tasks",
2753 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2754 Some(r#"{"view":"v1"}"#.to_owned()),
2755 ),
2756 (
2757 "tasks",
2758 vec![(
2759 "project_id".to_owned(),
2760 vec!["p2".to_owned(), "p1".to_owned()],
2761 )],
2762 Some(r#"{"view":"v2"}"#.to_owned()),
2763 ),
2764 (
2765 "docs",
2766 vec![
2767 ("org_id".to_owned(), vec!["o1".to_owned()]),
2768 ("projectId".to_owned(), vec!["p1".to_owned()]),
2769 ],
2770 Some(r#"{"view":"v1"}"#.to_owned()),
2771 ),
2772 ] {
2773 let error = reopened
2774 .subscribe(
2775 "stable-subscription".to_owned(),
2776 table.to_owned(),
2777 scopes,
2778 params,
2779 )
2780 .expect_err("identity rebind must fail");
2781 assert!(error.starts_with("client.subscription_intent_mismatch:"));
2782 assert_eq!(
2783 reopened
2784 .subscription_state("stable-subscription")
2785 .expect("unchanged state")
2786 .cursor,
2787 progress.cursor
2788 );
2789 }
2790
2791 drop(reopened);
2792 std::fs::remove_file(path).expect("remove temp database");
2793 }
2794
2795 #[test]
2796 fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
2797 let path = std::env::temp_dir().join(format!(
2798 "syncular-satisfied-schema-floor-{}.db",
2799 uuid::Uuid::new_v4()
2800 ));
2801 let schema = json!({
2802 "version": 23,
2803 "tables": [{
2804 "name": "tasks",
2805 "primaryKey": "id",
2806 "columns": [
2807 { "name": "id", "type": "string", "nullable": false },
2808 { "name": "project_id", "type": "string", "nullable": false }
2809 ],
2810 "scopes": [{ "pattern": "project:{project_id}" }]
2811 }]
2812 });
2813
2814 {
2815 let mut first = SyncClient::open_path_with_identity(
2816 None,
2817 &schema,
2818 ClientLimits::default(),
2819 path.to_str().expect("UTF-8 temp path"),
2820 )
2821 .expect("first open");
2822 first
2823 .subscribe(
2824 "tasks".to_owned(),
2825 "tasks".to_owned(),
2826 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2827 None,
2828 )
2829 .expect("persist subscription");
2830 first.set_schema_floor(Some(SchemaFloor {
2831 required_schema_version: Some(22),
2832 latest_schema_version: Some(22),
2833 }));
2834 }
2835
2836 let mut reopened = SyncClient::open_path_with_identity(
2837 None,
2838 &schema,
2839 ClientLimits::default(),
2840 path.to_str().expect("UTF-8 temp path"),
2841 )
2842 .expect("reopen");
2843 assert!(reopened.schema_floor().is_none());
2844 assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
2845 assert!(reopened.sync_needed());
2846 assert!(matches!(
2847 reopened.drain_sync_intents().as_slice(),
2848 [SyncIntent::Interactive]
2849 ));
2850 drop(reopened);
2851 std::fs::remove_file(path).expect("remove temp database");
2852 }
2853
2854 #[test]
2855 fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
2856 let path = std::env::temp_dir().join(format!(
2857 "syncular-unsatisfied-schema-floor-{}.db",
2858 uuid::Uuid::new_v4()
2859 ));
2860 let schema = json!({
2861 "version": 1,
2862 "tables": [{
2863 "name": "tasks",
2864 "primaryKey": "id",
2865 "columns": [
2866 { "name": "id", "type": "string", "nullable": false },
2867 { "name": "project_id", "type": "string", "nullable": false }
2868 ],
2869 "scopes": [{ "pattern": "project:{project_id}" }]
2870 }]
2871 });
2872
2873 {
2874 let mut first = SyncClient::open_path_with_identity(
2875 None,
2876 &schema,
2877 ClientLimits::default(),
2878 path.to_str().expect("UTF-8 temp path"),
2879 )
2880 .expect("first open");
2881 first
2882 .subscribe(
2883 "tasks".to_owned(),
2884 "tasks".to_owned(),
2885 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2886 None,
2887 )
2888 .expect("persist subscription");
2889 first.set_schema_floor(Some(SchemaFloor {
2890 required_schema_version: Some(2),
2891 latest_schema_version: Some(2),
2892 }));
2893 }
2894
2895 let reopened = SyncClient::open_path_with_identity(
2896 None,
2897 &schema,
2898 ClientLimits::default(),
2899 path.to_str().expect("UTF-8 temp path"),
2900 )
2901 .expect("reopen");
2902 assert_eq!(
2903 reopened.schema_floor(),
2904 Some(&SchemaFloor {
2905 required_schema_version: Some(2),
2906 latest_schema_version: Some(2),
2907 })
2908 );
2909 assert!(!reopened.sync_needed());
2910 drop(reopened);
2911 std::fs::remove_file(path).expect("remove temp database");
2912 }
2913
2914 #[test]
2915 fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
2916 let path = std::env::temp_dir().join(format!(
2917 "syncular-indexed-column-bump-{}.db",
2918 uuid::Uuid::new_v4()
2919 ));
2920 let old_schema = json!({
2921 "version": 1,
2922 "tables": [
2923 {
2924 "name": "tasks",
2925 "primaryKey": "id",
2926 "columns": [
2927 { "name": "id", "type": "string", "nullable": false },
2928 { "name": "project_id", "type": "string", "nullable": false }
2929 ],
2930 "scopes": [{ "pattern": "project:{project_id}" }]
2931 },
2932 {
2933 "name": "legacy",
2934 "primaryKey": "id",
2935 "columns": [
2936 { "name": "id", "type": "string", "nullable": false },
2937 { "name": "project_id", "type": "string", "nullable": false }
2938 ],
2939 "scopes": [{ "pattern": "project:{project_id}" }]
2940 }
2941 ]
2942 });
2943 {
2944 let mut first = SyncClient::open_path_with_identity(
2945 None,
2946 &old_schema,
2947 ClientLimits::default(),
2948 path.to_str().expect("UTF-8 temp path"),
2949 )
2950 .expect("open old schema");
2951 first
2952 .subscribe(
2953 "legacy-sub".to_owned(),
2954 "legacy".to_owned(),
2955 vec![("project_id".to_owned(), vec!["p1".to_owned()])],
2956 None,
2957 )
2958 .expect("persist legacy subscription");
2959 }
2960
2961 let new_schema = json!({
2962 "version": 2,
2963 "tables": [{
2964 "name": "tasks",
2965 "primaryKey": "id",
2966 "columns": [
2967 { "name": "id", "type": "string", "nullable": false },
2968 { "name": "project_id", "type": "string", "nullable": false },
2969 { "name": "facility_membership_id", "type": "string", "nullable": true }
2970 ],
2971 "scopes": [{ "pattern": "project:{project_id}" }],
2972 "indexes": [{
2973 "name": "tasks_by_membership",
2974 "columns": ["project_id", "facility_membership_id"],
2975 "unique": false
2976 }]
2977 }]
2978 });
2979 let upgraded = SyncClient::open_path_with_identity(
2980 None,
2981 &new_schema,
2982 ClientLimits::default(),
2983 path.to_str().expect("UTF-8 temp path"),
2984 )
2985 .expect("open upgraded schema");
2986 let columns = upgraded
2987 .conn
2988 .prepare("PRAGMA table_info(tasks)")
2989 .expect("prepare columns")
2990 .query_map([], |row| row.get::<_, String>(1))
2991 .expect("query columns")
2992 .collect::<Result<Vec<_>, _>>()
2993 .expect("collect columns");
2994 assert!(columns.contains(&"facility_membership_id".to_owned()));
2995 let index_count: i64 = upgraded
2996 .conn
2997 .query_row(
2998 "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
2999 [],
3000 |row| row.get(0),
3001 )
3002 .expect("query index");
3003 assert_eq!(index_count, 1);
3004 assert!(upgraded.subscription_state("legacy-sub").is_none());
3005 drop(upgraded);
3006 std::fs::remove_file(path).expect("remove temp database");
3007 }
3008
3009 #[test]
3010 fn migrates_pre_envelope_outcome_journal_additively() {
3011 let path = std::env::temp_dir().join(format!(
3012 "syncular-outcome-migration-{}.db",
3013 uuid::Uuid::new_v4()
3014 ));
3015 let conn = Connection::open(&path).expect("open legacy database");
3016 conn.execute_batch(
3017 "CREATE TABLE _syncular_commit_outcomes (
3018 seq INTEGER PRIMARY KEY AUTOINCREMENT,
3019 client_commit_id TEXT NOT NULL UNIQUE,
3020 status TEXT NOT NULL,
3021 recorded_at_ms INTEGER NOT NULL,
3022 results_json TEXT NOT NULL,
3023 resolution TEXT NOT NULL DEFAULT 'active',
3024 resolved_at_ms INTEGER,
3025 replacement_client_commit_id TEXT);",
3026 )
3027 .expect("create legacy outcome journal");
3028 drop(conn);
3029 let schema = json!({
3030 "version": 1,
3031 "tables": [{
3032 "name": "tasks",
3033 "primaryKey": "id",
3034 "columns": [
3035 { "name": "id", "type": "string", "nullable": false },
3036 { "name": "project_id", "type": "string", "nullable": false }
3037 ],
3038 "scopes": [{ "pattern": "project:{project_id}" }]
3039 }]
3040 });
3041 let client = SyncClient::open_path(
3042 "migration-native".to_owned(),
3043 &schema,
3044 ClientLimits::default(),
3045 path.to_str().expect("UTF-8 temp path"),
3046 )
3047 .expect("migrate database");
3048 let has_operations = client
3049 .conn
3050 .prepare("PRAGMA table_info(_syncular_commit_outcomes)")
3051 .expect("prepare table info")
3052 .query_map([], |row| row.get::<_, String>(1))
3053 .expect("query table info")
3054 .filter_map(Result::ok)
3055 .any(|column| column == "operations_json");
3056 assert!(has_operations);
3057 drop(client);
3058 std::fs::remove_file(path).expect("remove temp database");
3059 }
3060
3061 #[test]
3062 fn durable_conflict_outcome_and_resolution_survive_reopen() {
3063 let path = std::env::temp_dir().join(format!(
3064 "syncular-durable-outcome-{}.db",
3065 uuid::Uuid::new_v4()
3066 ));
3067 let schema = json!({
3068 "version": 1,
3069 "tables": [{
3070 "name": "tasks",
3071 "primaryKey": "id",
3072 "columns": [
3073 { "name": "id", "type": "string", "nullable": false },
3074 { "name": "project_id", "type": "string", "nullable": false }
3075 ],
3076 "scopes": [{ "pattern": "project:{project_id}" }]
3077 }]
3078 });
3079 let conflict = ConflictRecord {
3080 client_commit_id: "losing-commit".to_owned(),
3081 op_index: 0,
3082 table: "tasks".to_owned(),
3083 row_id: "t1".to_owned(),
3084 code: "sync.version_conflict".to_owned(),
3085 message: "stale base version".to_owned(),
3086 server_version: 2,
3087 server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
3088 conflict_columns: vec!["project_id".to_owned()],
3089 operation: Some(CommitOperation {
3090 table: "tasks".to_owned(),
3091 row_id: "t1".to_owned(),
3092 op: "upsert".to_owned(),
3093 base_version: Some(1),
3094 values: None,
3095 }),
3096 };
3097 let failed_operations = vec![
3098 OutboxOp {
3099 upsert: true,
3100 table: "tasks".to_owned(),
3101 row_id: "t1".to_owned(),
3102 base_version: Some(1),
3103 values: None,
3104 },
3105 OutboxOp {
3106 upsert: true,
3107 table: "tasks".to_owned(),
3108 row_id: "status-event-1".to_owned(),
3109 base_version: Some(0),
3110 values: None,
3111 },
3112 ];
3113
3114 {
3115 let mut first = SyncClient::open_path(
3116 "durable-native".to_owned(),
3117 &schema,
3118 ClientLimits::default(),
3119 path.to_str().expect("UTF-8 temp path"),
3120 )
3121 .expect("first open");
3122 first
3123 .begin_observation("test_outcome")
3124 .expect("begin outcome");
3125 first
3126 .persist_commit_outcome(
3127 "losing-commit",
3128 CommitOutcomeStatus::Conflict,
3129 &[CommitOperationOutcome::Conflict {
3130 conflict: conflict.clone(),
3131 }],
3132 Some(&failed_operations),
3133 )
3134 .expect("persist outcome");
3135 first.conflicts.push(conflict);
3136 first
3137 .finish_observation(
3138 "test_outcome",
3139 ChangeAccumulator {
3140 conflicts: true,
3141 outcomes: true,
3142 ..ChangeAccumulator::default()
3143 },
3144 )
3145 .expect("commit outcome");
3146 }
3147
3148 {
3149 let mut reopened = SyncClient::open_path(
3150 "durable-native".to_owned(),
3151 &schema,
3152 ClientLimits::default(),
3153 path.to_str().expect("UTF-8 temp path"),
3154 )
3155 .expect("reopen");
3156 assert_eq!(reopened.conflicts().len(), 1);
3157 assert_eq!(
3158 reopened.conflicts()[0].conflict_columns,
3159 vec!["project_id".to_owned()],
3160 "conflictColumns survives the journal round trip"
3161 );
3162 let outcome = reopened
3163 .commit_outcome("losing-commit")
3164 .expect("read outcome")
3165 .expect("outcome");
3166 assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
3167 let operations = outcome.operations.expect("aggregate envelope");
3168 assert_eq!(operations.len(), 2);
3169 assert_eq!(operations[1].row_id, "status-event-1");
3170 let resolved = reopened
3171 .resolve_commit_outcome(ResolveCommitOutcomeInput {
3172 client_commit_id: "losing-commit".to_owned(),
3173 resolution: CommitOutcomeResolution::ResolvedKeepServer,
3174 replacement_client_commit_id: None,
3175 })
3176 .expect("resolve");
3177 assert_eq!(
3178 resolved.resolution,
3179 CommitOutcomeResolution::ResolvedKeepServer
3180 );
3181 assert!(reopened.conflicts().is_empty());
3182 }
3183
3184 let reopened = SyncClient::open_path(
3185 "durable-native".to_owned(),
3186 &schema,
3187 ClientLimits::default(),
3188 path.to_str().expect("UTF-8 temp path"),
3189 )
3190 .expect("second reopen");
3191 assert!(reopened.conflicts().is_empty());
3192 assert_eq!(
3193 reopened
3194 .commit_outcome("losing-commit")
3195 .expect("read outcome")
3196 .expect("outcome")
3197 .resolution,
3198 CommitOutcomeResolution::ResolvedKeepServer
3199 );
3200 drop(reopened);
3201 std::fs::remove_file(path).expect("remove temp database");
3202 }
3203
3204 #[test]
3205 fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
3206 let path =
3207 std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
3208 let schema = json!({
3209 "version": 1,
3210 "tables": [{
3211 "name": "tasks",
3212 "primaryKey": "id",
3213 "columns": [
3214 { "name": "id", "type": "string", "nullable": false },
3215 { "name": "project_id", "type": "string", "nullable": false }
3216 ],
3217 "scopes": [{ "pattern": "project:{project_id}" }]
3218 }]
3219 });
3220 let mut client = SyncClient::open_path_with_identity(
3221 Some("sidecar-client".to_owned()),
3222 &schema,
3223 ClientLimits::default(),
3224 path.to_str().expect("UTF-8 temp path"),
3225 )
3226 .expect("open owner");
3227 let base = WindowBase {
3228 table: "tasks".to_owned(),
3229 variable: "project_id".to_owned(),
3230 fixed_scopes: Vec::new(),
3231 params: None,
3232 };
3233 client
3234 .set_window(&base, &["one".to_owned()])
3235 .expect("set window");
3236 client
3237 .mutate(vec![Mutation::Upsert {
3238 table: "tasks".to_owned(),
3239 values: Map::from_iter([
3240 ("id".to_owned(), Value::from("t1")),
3241 ("project_id".to_owned(), Value::from("one")),
3242 ]),
3243 base_version: None,
3244 }])
3245 .expect("local mutate");
3246
3247 let coverage = [WindowCoverage {
3248 base,
3249 units: vec!["one".to_owned(), "missing".to_owned()],
3250 }];
3251 let owner = client
3252 .query_snapshot(
3253 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
3254 &[],
3255 &coverage,
3256 )
3257 .expect("owner snapshot");
3258 let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
3259 let sidecar = reader
3260 .query_snapshot(
3261 "SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
3262 &[],
3263 &coverage,
3264 )
3265 .expect("sidecar snapshot");
3266
3267 assert_eq!(sidecar.revision, owner.revision);
3268 assert_eq!(sidecar.rows, owner.rows);
3269 assert_eq!(
3270 serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
3271 serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
3272 );
3273 assert_eq!(sidecar.revision, "2");
3274 assert_eq!(sidecar.rows[0]["id"], "t1");
3275 assert_eq!(sidecar.rows[0]["server_version"], -1);
3276 assert!(!sidecar.coverage.complete);
3277 assert_eq!(sidecar.coverage.pending.len(), 1);
3278 assert_eq!(sidecar.coverage.missing.len(), 1);
3279
3280 drop(reader);
3281 drop(client);
3282 std::fs::remove_file(path).expect("remove temp database");
3283 }
3284
3285 #[test]
3286 fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
3287 let mut client = client();
3288 let image = Connection::open_in_memory().unwrap();
3289 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
3290 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
3291 INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
3292 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
3293 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
3294 let table = client.schema.table("tasks").unwrap().clone();
3295 let scopes = vec![("project_id".into(), vec!["p1".into()])];
3296 assert!(client
3297 .apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
3298 .is_err());
3299 assert!(client.conn.is_autocommit());
3300 assert_eq!(
3301 client
3302 .query("SELECT count(*) AS n FROM tasks", &[])
3303 .unwrap()[0]["n"],
3304 1024
3305 );
3306 assert_eq!(client.drain_change_batches().len(), 1);
3307 image
3308 .execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
3309 .unwrap();
3310 assert!(matches!(
3311 client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
3312 Ok(2050)
3313 ));
3314 assert_eq!(
3315 client
3316 .query("SELECT count(*) AS n FROM tasks", &[])
3317 .unwrap()[0]["n"],
3318 2050
3319 );
3320 assert_eq!(client.drain_change_batches().len(), 3);
3321 }
3322
3323 #[test]
3324 fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
3325 let mut client = client();
3326 let base = WindowBase {
3327 table: "tasks".into(),
3328 variable: "project_id".into(),
3329 fixed_scopes: Vec::new(),
3330 params: None,
3331 };
3332 client
3333 .set_window(&base, &["p1".into(), "p2".into()])
3334 .unwrap();
3335 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
3336 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
3337 INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
3338 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
3339 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
3340 client.drain_change_batches();
3341 assert!(client.set_window(&base, &["p2".into()]).is_err());
3342 assert!(client.conn.is_autocommit());
3343 assert_eq!(
3344 client
3345 .query("SELECT count(*) AS n FROM tasks", &[])
3346 .unwrap()[0]["n"],
3347 1027
3348 );
3349 assert_eq!(client.window_state(&base).units, vec!["p2"]);
3350 assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
3351 assert_eq!(client.drain_change_batches().len(), 1);
3352 client
3353 .conn
3354 .execute_batch("DROP TRIGGER interrupt_evict")
3355 .unwrap();
3356 client.drain_pending_evictions().unwrap();
3357 assert!(client.load_pending_evictions().unwrap().is_empty());
3358 assert_eq!(
3359 client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
3360 "held"
3361 );
3362 assert_eq!(client.drain_change_batches().len(), 2);
3363 }
3364
3365 #[test]
3366 fn eviction_marker_failures_roll_back_window_changes() {
3367 let mut client = client();
3368 let base = WindowBase {
3369 table: "tasks".into(),
3370 variable: "project_id".into(),
3371 fixed_scopes: Vec::new(),
3372 params: None,
3373 };
3374 client.set_window(&base, &["p1".into()]).unwrap();
3375 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
3376 INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
3377 INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
3378 CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
3379 let revision = client.local_revision();
3380 assert!(client.set_window(&base, &[]).is_err());
3381 assert!(client.conn.is_autocommit());
3382 assert_eq!(client.local_revision(), revision);
3383 assert_eq!(
3384 client
3385 .query("SELECT count(*) AS n FROM tasks", &[])
3386 .unwrap()[0]["n"],
3387 2050
3388 );
3389 assert_eq!(client.window_state(&base).units, vec!["p1"]);
3390 client.conn.execute_batch("DROP TRIGGER fail_marker;
3391 CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
3392 assert!(client.set_window(&base, &[]).is_err());
3393 client.conn.execute_batch("DROP TRIGGER interrupt_evict;
3394 CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
3395 assert!(client.drain_pending_evictions().is_err());
3396 assert!(client.conn.is_autocommit());
3397 assert_eq!(
3398 client
3399 .query("SELECT count(*) AS n FROM tasks", &[])
3400 .unwrap()[0]["n"],
3401 2
3402 );
3403 assert!(client.set_window(&base, &["p1".into()]).is_err());
3404 assert!(client.conn.is_autocommit());
3405 assert!(client.window_state(&base).units.is_empty());
3406 assert!(client.subs.is_empty());
3407 assert_eq!(
3408 client
3409 .query(
3410 "SELECT count(*) AS n FROM _syncular_window_pending_evict",
3411 &[]
3412 )
3413 .unwrap()[0]["n"],
3414 1
3415 );
3416 client
3417 .conn
3418 .execute_batch("DROP TRIGGER fail_marker")
3419 .unwrap();
3420 client.set_window(&base, &["p1".into()]).unwrap();
3421 assert_eq!(client.window_state(&base).units, vec!["p1"]);
3422 }
3423
3424 #[test]
3425 fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
3426 let mut client = client();
3427 client
3428 .conn
3429 .execute(
3430 "INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
3431 [],
3432 )
3433 .unwrap();
3434 assert_eq!(
3435 client.drain_pending_evictions().unwrap_err(),
3436 "sync.local_corrupt: invalid pending eviction scopes"
3437 );
3438 assert!(client.conn.is_autocommit());
3439 assert!(client.enqueue_startup_sync_if_needed().is_err());
3440 }
3441
3442 #[test]
3443 fn image_chunks_reject_null_primary_keys() {
3444 let mut client = client();
3445 let image = Connection::open_in_memory().unwrap();
3446 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
3447 WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
3448 INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
3449 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
3450 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
3451 let table = client.schema.table("tasks").unwrap().clone();
3452 assert!(client
3453 .apply_sqlite_image(
3454 &image,
3455 &table,
3456 true,
3457 &[("project_id".into(), vec!["p1".into()])],
3458 1025,
3459 7,
3460 "digest"
3461 )
3462 .is_err());
3463 assert!(client.conn.is_autocommit());
3464 assert_eq!(
3465 client
3466 .query("SELECT count(*) AS n FROM tasks", &[])
3467 .unwrap()[0]["n"],
3468 0
3469 );
3470 }
3471
3472 #[test]
3473 fn import_chunks_with_unique_constraints_match_full_pending_replay() {
3474 for images in [false, true] {
3475 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
3476 "name":name,"primaryKey":"id","columns":[
3477 {"name":"id","type":"string","nullable":false},
3478 {"name":"project_id","type":"string","nullable":false},
3479 {"name":"title","type":"string","nullable":false}],
3480 "scopes":[{"pattern":"project:{project_id}"}],
3481 "indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
3482 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
3483 })))});
3484 let mut actual =
3485 SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
3486 let mut reference =
3487 SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
3488 let scopes = vec![("project_id".into(), vec!["p1".into()])];
3489 for client in [&mut actual, &mut reference] {
3490 client
3491 .subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
3492 .unwrap();
3493 client.subs[0].effective = Some(scopes.clone());
3494 for (table, id, title) in [
3495 ("tasks", "a", "original"),
3496 ("tasks", "b", "occupied"),
3497 ("tasks", "c", "deleted"),
3498 ("notes", "n", "untouched"),
3499 ] {
3500 client
3501 .write_base_row(
3502 table,
3503 &vec![
3504 Some(ColumnValue::String(id.into())),
3505 Some(ColumnValue::String("p1".into())),
3506 Some(ColumnValue::String(title.into())),
3507 ],
3508 1,
3509 )
3510 .unwrap();
3511 }
3512 client.rebuild_overlay_if_dirty();
3513 client
3514 .mutate(vec![
3515 Mutation::Upsert {
3516 table: "tasks".into(),
3517 values: Map::from_iter([
3518 ("id".into(), Value::from("a")),
3519 ("project_id".into(), Value::from("p1")),
3520 ("title".into(), Value::from("occupied")),
3521 ]),
3522 base_version: None,
3523 },
3524 Mutation::Delete {
3525 table: "tasks".into(),
3526 row_id: "c".into(),
3527 base_version: None,
3528 },
3529 Mutation::Upsert {
3530 table: "tasks".into(),
3531 values: Map::from_iter([
3532 ("id".into(), Value::from("local")),
3533 ("project_id".into(), Value::from("p1")),
3534 ("title".into(), Value::from("local value")),
3535 ]),
3536 base_version: None,
3537 },
3538 Mutation::Upsert {
3539 table: "notes".into(),
3540 values: Map::from_iter([
3541 ("id".into(), Value::from("n")),
3542 ("project_id".into(), Value::from("p1")),
3543 ("title".into(), Value::from("pending note")),
3544 ]),
3545 base_version: None,
3546 },
3547 ])
3548 .unwrap();
3549 }
3550 let pending = actual.pending_commit_ids();
3551 actual.overlay_rebuild_count.set(0);
3552 for (id, title) in [
3553 ("b", "available"),
3554 ("b", "occupied"),
3555 ("a", "server original"),
3556 ("d", "local value"),
3557 ("d", "released"),
3558 ("c", "redelivered"),
3559 ] {
3560 let table = actual.schema.table("tasks").unwrap().clone();
3561 let values = vec![
3562 Some(ColumnValue::String(id.into())),
3563 Some(ColumnValue::String("p1".into())),
3564 Some(ColumnValue::String(title.into())),
3565 ];
3566 if images {
3567 let image = Connection::open_in_memory().unwrap();
3568 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
3569 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
3570 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
3571 image
3572 .execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
3573 .unwrap();
3574 assert!(matches!(
3575 actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
3576 Ok(1)
3577 ));
3578 } else {
3579 let segment = RowsSegment {
3580 table: "tasks".into(),
3581 schema_version: 1,
3582 columns: table.wire_columns.clone(),
3583 blocks: vec![vec![ssp2::segment::SegmentRow {
3584 server_version: 2,
3585 values: values.clone(),
3586 }]],
3587 };
3588 assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
3589 }
3590 reference.write_base_row("tasks", &values, 2).unwrap();
3591 reference.rebuild_overlay();
3592 for table in ["tasks", "notes"] {
3593 for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
3594 assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
3595 }
3596 }
3597 assert_eq!(actual.pending_commit_ids(), pending);
3598 }
3599 assert_eq!(
3600 actual.overlay_rebuild_count.get(),
3601 0,
3602 "each chunk must leave unrelated tables alone"
3603 );
3604 let before = actual
3605 .query("SELECT * FROM tasks ORDER BY id", &[])
3606 .unwrap();
3607 let fts_before = actual
3608 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
3609 .unwrap();
3610 let revision = actual.local_revision();
3611 actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
3612 let table = actual.schema.table("tasks").unwrap().clone();
3613 let segment = RowsSegment {
3614 table: "tasks".into(),
3615 schema_version: 1,
3616 columns: table.wire_columns.clone(),
3617 blocks: vec![vec![ssp2::segment::SegmentRow {
3618 server_version: 3,
3619 values: vec![
3620 Some(ColumnValue::String("b".into())),
3621 Some(ColumnValue::String("p1".into())),
3622 Some(ColumnValue::String("fresh".into())),
3623 ],
3624 }]],
3625 };
3626 assert!(actual.apply_segment(0, &segment, false).is_err());
3627 assert!(actual.conn.is_autocommit());
3628 assert_eq!(
3629 actual
3630 .query("SELECT * FROM tasks ORDER BY id", &[])
3631 .unwrap(),
3632 before
3633 );
3634 assert_eq!(
3635 actual
3636 .query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
3637 .unwrap(),
3638 fts_before
3639 );
3640 assert_eq!(actual.local_revision(), revision);
3641 assert_eq!(actual.pending_commit_ids(), pending);
3642 }
3643 }
3644
3645 #[test]
3646 fn import_reconciliation_preserves_typed_pending_primary_keys() {
3647 for (kind, a, b) in [
3648 ("string", json!("a"), json!("b")),
3649 ("integer", json!(1), json!(2)),
3650 ("float", json!(1.5), json!(2.5)),
3651 ("boolean", json!(true), json!(false)),
3652 ("json", json!("1"), json!("2")),
3653 ] {
3654 let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
3655 for images in [false, true] {
3656 let mut client =
3657 SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
3658 let table = client.schema.table("tasks").unwrap().clone();
3659 client
3660 .subscribe("tasks".into(), "tasks".into(), vec![], None)
3661 .unwrap();
3662 for (id, title) in [(&a, "original"), (&b, "occupied")] {
3663 client
3664 .write_base_row(
3665 "tasks",
3666 &vec![
3667 json_to_column_value(&table.columns[0], Some(id)).unwrap(),
3668 Some(ColumnValue::String(title.into())),
3669 ],
3670 1,
3671 )
3672 .unwrap();
3673 }
3674 client.rebuild_overlay_if_dirty();
3675 client
3676 .mutate(vec![Mutation::Upsert {
3677 table: "tasks".into(),
3678 values: Map::from_iter([
3679 ("id".into(), a.clone()),
3680 ("title".into(), json!("occupied")),
3681 ]),
3682 base_version: None,
3683 }])
3684 .unwrap();
3685 client.overlay_rebuild_count.set(0);
3686 for (title, expected) in [("available", "occupied"), ("occupied", "original")] {
3687 let values = vec![
3688 json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
3689 Some(ColumnValue::String(title.into())),
3690 ];
3691 if images {
3692 let image = Connection::open_in_memory().unwrap();
3693 image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
3694 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
3695 INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
3696 image
3697 .execute(
3698 "INSERT INTO tasks VALUES (?1,?2,2)",
3699 rusqlite::params![RowParam::Cell(&values[0]), title],
3700 )
3701 .unwrap();
3702 assert!(
3703 matches!(
3704 client.apply_sqlite_image(
3705 &image,
3706 &table,
3707 false,
3708 &[],
3709 1,
3710 7,
3711 "digest"
3712 ),
3713 Ok(1)
3714 ),
3715 "{kind}, images={images}"
3716 );
3717 } else {
3718 let segment = RowsSegment {
3719 table: "tasks".into(),
3720 schema_version: 1,
3721 columns: table.wire_columns.clone(),
3722 blocks: vec![vec![ssp2::segment::SegmentRow {
3723 server_version: 2,
3724 values,
3725 }]],
3726 };
3727 assert!(
3728 matches!(client.apply_segment(0, &segment, false), Ok(1)),
3729 "{kind}"
3730 );
3731 }
3732 let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
3733 let actual: String = client
3734 .conn
3735 .query_row(
3736 "SELECT title FROM tasks WHERE id=?1",
3737 [RowParam::Cell(&key)],
3738 |row| row.get(0),
3739 )
3740 .unwrap();
3741 assert_eq!(actual, expected, "{kind}, images={images}");
3742 }
3743 assert_eq!(client.overlay_rebuild_count.get(), 0);
3744 }
3745 }
3746 }
3747
3748 fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
3749 let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
3750 "name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
3751 "scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
3752 "ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
3753 })))});
3754 let mut client =
3755 SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
3756 client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
3757 client.rebuild_overlay();
3758 client
3759 .mutate(vec![Mutation::Upsert {
3760 table: "tasks".into(),
3761 values: Map::from_iter([
3762 ("id".into(), json!("offline")),
3763 ("project_id".into(), json!("p1")),
3764 ("title".into(), json!("pending edit")),
3765 ]),
3766 base_version: None,
3767 }])
3768 .unwrap();
3769 let image = Connection::open_in_memory().unwrap();
3770 image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
3771 CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
3772 image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
3773 image
3774 .execute(
3775 "INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
3776 [rows],
3777 )
3778 .unwrap();
3779 client.overlay_rebuild_count.set(0);
3780 let table = client.schema.table("tasks").unwrap().clone();
3781 (client, image, table)
3782 }
3783
3784 #[test]
3785 fn large_unique_import_copies_no_unrelated_tables() {
3786 let (mut client, image, table) = unique_import_fixture(10240);
3787 let pending = client.pending_commit_ids();
3788 client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
3789 CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
3790 assert!(matches!(
3791 client.apply_sqlite_image(
3792 &image,
3793 &table,
3794 true,
3795 &[("project_id".into(), vec!["p1".into()])],
3796 10240,
3797 7,
3798 "digest"
3799 ),
3800 Ok(10240)
3801 ));
3802 assert_eq!(client.overlay_rebuild_count.get(), 0);
3803 assert_eq!(client.pending_commit_ids(), pending);
3804 assert_eq!(
3805 client
3806 .query("SELECT count(*) AS n FROM tasks", &[])
3807 .unwrap()[0]["n"],
3808 10241
3809 );
3810 assert_eq!(
3811 client
3812 .query("SELECT count(*) AS n FROM tasks_fts", &[])
3813 .unwrap()[0]["n"],
3814 10241
3815 );
3816 assert_eq!(
3817 client
3818 .query("SELECT count(*) AS n FROM notes_fts", &[])
3819 .unwrap()[0]["n"],
3820 8192
3821 );
3822 }
3823
3824 #[test]
3825 #[ignore = "manual scaling benchmark; see bench/README.md"]
3826 fn benchmark_unique_image_import() {
3827 let mut samples = Vec::new();
3828 for rows in [10240, 51200, 102400] {
3829 for trial in 0..4 {
3830 let (mut client, image, table) = unique_import_fixture(rows);
3831 let start = std::time::Instant::now();
3832 assert!(
3833 matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
3834 );
3835 let elapsed = start.elapsed().as_secs_f64() * 1000.0;
3836 assert_eq!(
3837 client
3838 .query("SELECT count(*) AS n FROM tasks", &[])
3839 .unwrap()[0]["n"],
3840 rows + 1
3841 );
3842 assert_eq!(
3843 client
3844 .query("SELECT count(*) AS n FROM tasks_fts", &[])
3845 .unwrap()[0]["n"],
3846 rows + 1
3847 );
3848 assert_eq!(
3849 client
3850 .query("SELECT count(*) AS n FROM notes_fts", &[])
3851 .unwrap()[0]["n"],
3852 8192
3853 );
3854 assert_eq!(client.pending_commit_ids().len(), 1);
3855 if trial > 0 {
3856 samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
3857 }
3858 }
3859 }
3860 let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
3861 let output = serde_json::to_string_pretty(&artifact).unwrap();
3862 if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
3863 use std::io::Write;
3864 let mut file = std::fs::OpenOptions::new()
3865 .write(true)
3866 .create_new(true)
3867 .open(path)
3868 .unwrap();
3869 file.write_all(output.as_bytes()).unwrap();
3870 } else {
3871 println!("{output}");
3872 }
3873 }
3874
3875 #[test]
3876 fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
3877 let schema = json!({
3878 "version": 1,
3879 "tables": [{
3880 "name": "catalogue_codes",
3881 "primaryKey": "id",
3882 "columns": [
3883 { "name": "id", "type": "string", "nullable": false },
3884 { "name": "release_id", "type": "string", "nullable": false },
3885 { "name": "code", "type": "string", "nullable": false },
3886 { "name": "title", "type": "string", "nullable": false }
3887 ],
3888 "scopes": [{ "pattern": "release:{release_id}" }],
3889 "ftsIndexes": [{
3890 "name": "catalogue_codes_fts",
3891 "columns": ["code", "title"],
3892 "tokenize": "unicode61 remove_diacritics 2"
3893 }]
3894 }]
3895 });
3896 let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
3897 .expect("FTS5 client");
3898 let insert_trigger: String = client
3899 .conn
3900 .query_row(
3901 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
3902 [],
3903 |row| row.get(0),
3904 )
3905 .expect("insert trigger");
3906 let replace_guard: String = client
3907 .conn
3908 .query_row(
3909 "SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
3910 [],
3911 |row| row.get(0),
3912 )
3913 .expect("replace guard");
3914 assert!(!insert_trigger.contains("DELETE FROM"));
3915 assert!(replace_guard.contains("BEFORE INSERT"));
3916 assert!(replace_guard.contains("WHEN EXISTS"));
3917 let search = |client: &SyncClient, query: &str| {
3918 client
3919 .query(
3920 "SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
3921 &[Value::from(query)],
3922 )
3923 .expect("FTS query")
3924 };
3925
3926 client
3927 .mutate(vec![Mutation::Upsert {
3928 table: "catalogue_codes".to_owned(),
3929 values: Map::from_iter([
3930 ("id".to_owned(), Value::from("c1")),
3931 ("release_id".to_owned(), Value::from("r1")),
3932 ("code".to_owned(), Value::from("A01")),
3933 ("title".to_owned(), Value::from("Cholera")),
3934 ]),
3935 base_version: None,
3936 }])
3937 .expect("insert code");
3938 assert_eq!(search(&client, "cholera").len(), 1);
3939 let prior = client
3940 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
3941 .unwrap();
3942 client
3943 .conn
3944 .execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
3945 .unwrap();
3946 client.create_synced_tables().unwrap();
3947 assert_eq!(
3948 client
3949 .query("SELECT rowid, * FROM catalogue_codes_fts", &[])
3950 .unwrap(),
3951 prior
3952 );
3953 let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
3954 let rendered = serde_json::to_string(&plan).unwrap();
3955 assert!(rendered.contains("COVERING INDEX"));
3956 assert!(rendered.contains("INDEX 0:="));
3957 client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
3958 assert_eq!(search(&client, "cholera").len(), 1);
3959
3960 client
3961 .mutate(vec![Mutation::Upsert {
3962 table: "catalogue_codes".to_owned(),
3963 values: Map::from_iter([
3964 ("id".to_owned(), Value::from("c1")),
3965 ("release_id".to_owned(), Value::from("r1")),
3966 ("code".to_owned(), Value::from("A01")),
3967 ("title".to_owned(), Value::from("Enteric infection")),
3968 ]),
3969 base_version: None,
3970 }])
3971 .expect("update code");
3972 assert!(search(&client, "cholera").is_empty());
3973 assert_eq!(search(&client, "enteric").len(), 1);
3974
3975 client
3976 .mutate(vec![Mutation::Delete {
3977 table: "catalogue_codes".to_owned(),
3978 row_id: "c1".to_owned(),
3979 base_version: None,
3980 }])
3981 .expect("delete code");
3982 assert!(search(&client, "enteric").is_empty());
3983 }
3984
3985 #[test]
3986 fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
3987 let schema = json!({
3988 "version": 1,
3989 "tables": [{
3990 "name": "patient_notes",
3991 "primaryKey": "id",
3992 "columns": [
3993 { "name": "id", "type": "string", "nullable": false },
3994 { "name": "practice_id", "type": "string", "nullable": false },
3995 { "name": "encryption_key_id", "type": "string", "nullable": false },
3996 { "name": "title", "type": "string", "nullable": false }
3997 ],
3998 "scopes": [{ "pattern": "practice:{practice_id}" }],
3999 "ftsIndexes": [{
4000 "name": "patient_notes_fts",
4001 "columns": ["title"],
4002 "tokenize": "unicode61 remove_diacritics 2"
4003 }]
4004 }]
4005 });
4006 let mut client = SyncClient::new(
4007 "local-purge-test".to_owned(),
4008 &schema,
4009 ClientLimits::default(),
4010 )
4011 .expect("local purge client");
4012 client
4013 .conn
4014 .execute(
4015 "INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
4016 ('target', 'practice-1', 'key-revoked', 'Target original', 1),
4017 ('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
4018 [],
4019 )
4020 .expect("seed base rows");
4021 client.overlay_dirty.set(true);
4022 client.rebuild_overlay_if_dirty();
4023
4024 let note = |id: &str, key_id: &str, title: &str| {
4025 Map::from_iter([
4026 ("id".to_owned(), Value::from(id)),
4027 ("practice_id".to_owned(), Value::from("practice-1")),
4028 ("encryption_key_id".to_owned(), Value::from(key_id)),
4029 ("title".to_owned(), Value::from(title)),
4030 ])
4031 };
4032 let doomed = client
4033 .mutate(vec![
4034 Mutation::Upsert {
4035 table: "patient_notes".to_owned(),
4036 values: note("target", "key-revoked", "Target changed"),
4037 base_version: None,
4038 },
4039 Mutation::Upsert {
4040 table: "patient_notes".to_owned(),
4041 values: note("unrelated", "key-held", "Unrelated changed"),
4042 base_version: None,
4043 },
4044 ])
4045 .expect("doomed commit");
4046 let kept = client
4047 .mutate(vec![Mutation::Upsert {
4048 table: "patient_notes".to_owned(),
4049 values: note("kept", "key-held", "Kept optimistic"),
4050 base_version: None,
4051 }])
4052 .expect("kept commit");
4053 client.drain_change_batches();
4054
4055 let input = LocalDataPurgeInput {
4056 purge_id: "purge-001".to_owned(),
4057 targets: vec![LocalDataPurgeTarget {
4058 table: "patient_notes".to_owned(),
4059 selectors: BTreeMap::from([(
4060 "encryption_key_id".to_owned(),
4061 vec!["key-revoked".to_owned()],
4062 )]),
4063 }],
4064 };
4065 assert_eq!(
4066 client.purge_local_data(&input).expect("apply purge"),
4067 LocalDataPurgeResult {
4068 already_applied: false,
4069 purged_rows: 1,
4070 dropped_commits: 1,
4071 }
4072 );
4073 let rows = client
4074 .query("SELECT id, title FROM patient_notes ORDER BY id", &[])
4075 .expect("visible rows");
4076 assert_eq!(rows.len(), 2);
4077 assert_eq!(rows[0]["id"], "kept");
4078 assert_eq!(rows[1]["id"], "unrelated");
4079 assert_eq!(rows[1]["title"], "Unrelated original");
4080 let fts = client
4081 .query(
4082 "SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
4083 &[],
4084 )
4085 .expect("fts query");
4086 assert!(fts.is_empty());
4087 assert_eq!(client.outbox.len(), 1);
4088 assert_eq!(client.outbox[0].client_commit_id, kept);
4089 let outcome = client
4090 .commit_outcome(&doomed)
4091 .expect("read doomed outcome")
4092 .expect("doomed outcome");
4093 assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
4094 match &outcome.results[0] {
4095 CommitOperationOutcome::Error { rejection } => {
4096 assert_eq!(rejection.code, "client.local_data_purged");
4097 assert_eq!(rejection.client_commit_id, doomed);
4098 }
4099 other => panic!("expected local purge rejection, got {other:?}"),
4100 }
4101 assert_eq!(client.drain_change_batches().len(), 1);
4102 assert_eq!(
4103 client.purge_local_data(&input).expect("retry purge"),
4104 LocalDataPurgeResult {
4105 already_applied: true,
4106 purged_rows: 0,
4107 dropped_commits: 0,
4108 }
4109 );
4110 let conflicting = LocalDataPurgeInput {
4111 purge_id: input.purge_id.clone(),
4112 targets: vec![LocalDataPurgeTarget {
4113 table: "patient_notes".to_owned(),
4114 selectors: BTreeMap::from([(
4115 "encryption_key_id".to_owned(),
4116 vec!["key-held".to_owned()],
4117 )]),
4118 }],
4119 };
4120 assert!(client
4121 .purge_local_data(&conflicting)
4122 .expect_err("id collision must fail")
4123 .contains("already used with a different plan"));
4124 }
4125}
4126
4127impl SubState {
4128 fn name(self) -> &'static str {
4129 match self {
4130 SubState::Active => "active",
4131 SubState::Revoked => "revoked",
4132 SubState::Failed => "failed",
4133 }
4134 }
4135
4136 fn parse(value: &str) -> Self {
4137 match value {
4138 "revoked" => Self::Revoked,
4139 "failed" => Self::Failed,
4140 _ => Self::Active,
4141 }
4142 }
4143}
4144
4145#[derive(Debug, Clone)]
4146struct Subscription {
4147 id: String,
4148 table: String,
4149 requested: Vec<(String, Vec<String>)>,
4150 params: Option<String>,
4151 cursor: i64,
4152 bootstrap_state: Option<String>,
4154 state: SubState,
4155 reason_code: Option<String>,
4156 effective: Option<Vec<(String, Vec<String>)>>,
4159 synced_once: bool,
4160}
4161
4162#[derive(Debug, Clone)]
4163struct OutboxOp {
4164 upsert: bool,
4165 table: String,
4166 row_id: String,
4167 base_version: Option<i64>,
4168 values: Option<Map<String, Value>>,
4172}
4173
4174impl From<&OutboxOp> for CommitOperation {
4175 fn from(operation: &OutboxOp) -> Self {
4176 Self {
4177 table: operation.table.clone(),
4178 row_id: operation.row_id.clone(),
4179 op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
4180 base_version: operation.base_version,
4181 values: operation.values.clone(),
4182 }
4183 }
4184}
4185
4186#[derive(Debug, Clone)]
4187struct OutboxCommit {
4188 client_commit_id: String,
4189 ops: Vec<OutboxOp>,
4190}
4191
4192#[derive(Debug, Clone)]
4193struct CompiledLocalDataPurgeTarget {
4194 table: String,
4195 selectors: Vec<(String, Vec<String>)>,
4196}
4197
4198struct StoredCommitOutcomeRow {
4199 sequence: i64,
4200 client_commit_id: String,
4201 status: String,
4202 recorded_at_ms: i64,
4203 results_json: String,
4204 operations_json: Option<String>,
4205 resolution: String,
4206 resolved_at_ms: Option<i64>,
4207 replacement_client_commit_id: Option<String>,
4208}
4209
4210enum SectionError {
4213 FailClosed,
4214 Abort(String, String),
4215}
4216
4217struct RequestMeta {
4218 pushed_ids: Vec<String>,
4219 fresh: Vec<(String, bool)>,
4222 accept: u8,
4223 deferred_commits: usize,
4227}
4228
4229#[derive(Default)]
4230struct ChangeAccumulator {
4231 tables: BTreeMap<String, Option<BTreeSet<String>>>,
4233 windows: BTreeMap<(String, String), BTreeSet<String>>,
4234 status: bool,
4235 conflicts: bool,
4236 rejections: bool,
4237 outcomes: bool,
4238}
4239
4240impl ChangeAccumulator {
4241 fn table(&mut self, table: &str) {
4242 self.tables.insert(table.to_owned(), None);
4243 }
4244
4245 fn scope(&mut self, table: &str, key: String) {
4246 match self.tables.get_mut(table) {
4247 Some(None) => {}
4248 Some(Some(keys)) => {
4249 keys.insert(key);
4250 }
4251 None => {
4252 self.tables
4253 .insert(table.to_owned(), Some(BTreeSet::from([key])));
4254 }
4255 }
4256 }
4257
4258 fn window(&mut self, base_key: &str, table: &str, unit: &str) {
4259 self.windows
4260 .entry((base_key.to_owned(), table.to_owned()))
4261 .or_default()
4262 .insert(unit.to_owned());
4263 }
4264
4265 fn touched(&self) -> bool {
4266 !self.tables.is_empty()
4267 || !self.windows.is_empty()
4268 || self.status
4269 || self.conflicts
4270 || self.rejections
4271 || self.outcomes
4272 }
4273}
4274
4275const PUSH_OPS_PER_REQUEST: usize = 500;
4281const MAX_LOCAL_PURGE_TARGETS: usize = 64;
4282const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
4283const MAX_LOCAL_PURGE_VALUES: usize = 128;
4284const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
4285pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
4286
4287pub struct SyncClient {
4288 #[cfg(feature = "bench-internals")]
4289 benchmark_phases: Recorder,
4290 conn: Connection,
4291 schema: ClientSchema,
4292 client_id: String,
4293 limits: ClientLimits,
4294 subs: Vec<Subscription>,
4295 outbox: Vec<OutboxCommit>,
4296 conflicts: Vec<ConflictRecord>,
4297 rejections: Vec<RejectionRecord>,
4298 schema_floor: Option<SchemaFloor>,
4299 lease_state: Option<LeaseState>,
4301 stopped: bool,
4303 upgrading: bool,
4305 sync_needed: bool,
4307 realtime_connected: bool,
4308 presence: HashMap<String, HashMap<String, PresencePeer>>,
4310 now_ms: Option<i64>,
4313 encryption: crate::values::EncryptionConfig,
4317 security_preflight: bool,
4320 insert_sql: RefCell<HashMap<String, String>>,
4326 overlay_dirty: Cell<bool>,
4330 #[cfg(test)]
4333 overlay_rebuild_count: Cell<usize>,
4334 #[cfg(test)]
4337 outcome_prune_count: Cell<usize>,
4338 progress: ProgressObserver,
4340 change_queue: VecDeque<ClientChangeBatch>,
4341 sync_intent_queue: VecDeque<SyncIntent>,
4342 retry_delay_ms: u64,
4344 last_round: Option<DiagnosticLastRound>,
4345 last_change: Option<DiagnosticLastChange>,
4346}
4347
4348fn quote_ident(name: &str) -> String {
4349 format!("\"{}\"", name.replace('"', "\"\""))
4350}
4351
4352fn row_id_predicate(table: &TableSchema) -> String {
4354 let key = quote_ident(&table.primary_key);
4355 let text_match = format!("CAST({key} AS TEXT) = ?1");
4356 match table.columns[table.pk_index].ty {
4357 ColumnType::String | ColumnType::Json => {
4358 format!("{key} = ?1 AND {text_match}")
4359 }
4360 ColumnType::Integer | ColumnType::Boolean => {
4364 format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
4365 }
4366 _ => text_match,
4369 }
4370}
4371
4372fn is_local_operation_code_like(value: &str) -> bool {
4373 let bytes = value.as_bytes();
4374 bytes.first().is_some_and(u8::is_ascii_alphanumeric)
4375 && bytes
4376 .iter()
4377 .all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
4378}
4379
4380fn base_table(name: &str) -> String {
4381 quote_ident(&format!("_syncular_base_{name}"))
4382}
4383
4384type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
4389
4390fn window_base_key(base: &WindowBase) -> String {
4391 format!(
4392 "{}\0{}\0{}",
4393 base.table,
4394 base.variable,
4395 canonical_scope_json(&base.fixed_scopes)
4396 )
4397}
4398
4399fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
4401 let mut scopes = base.fixed_scopes.clone();
4402 scopes.retain(|(k, _)| k != &base.variable);
4403 scopes.push((base.variable.clone(), vec![unit.to_owned()]));
4404 scopes
4405}
4406
4407fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
4411 let canonical = canonical_scope_json(&unit_scopes(base, unit));
4412 let digest = Sha256::digest(canonical.as_bytes());
4413 let hex = bytes_to_hex(&digest);
4414 format!("w:{}:{}", base.table, &hex[..16])
4415}
4416
4417fn visible_table(name: &str) -> String {
4418 quote_ident(name)
4419}
4420
4421const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
4422
4423fn is_synced_table_name(name: &str) -> bool {
4426 if name.starts_with("sqlite_") {
4427 return false;
4428 }
4429 if name.starts_with("_syncular_base_") {
4430 return true; }
4432 !name.starts_with("_syncular_")
4434}
4435
4436fn blob_id_for(bytes: &[u8]) -> String {
4438 let digest = Sha256::digest(bytes);
4439 format!("sha256:{}", bytes_to_hex(&digest))
4440}
4441
4442enum RowParam<'a> {
4446 Cell(&'a Option<ColumnValue>),
4447 Version(i64),
4448}
4449
4450impl rusqlite::ToSql for RowParam<'_> {
4451 fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
4452 Ok(match self {
4453 RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
4454 RowParam::Cell(cell) => match cell {
4455 None => ToSqlOutput::Owned(SqlValue::Null),
4456 Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
4457 Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
4458 Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
4459 Some(ColumnValue::Boolean(b)) => {
4460 ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
4461 }
4462 Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
4463 ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
4464 }
4465 Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
4467 ToSqlOutput::Borrowed(ValueRef::Blob(b))
4468 }
4469 },
4470 })
4471 }
4472}
4473
4474fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
4476 match param.to_sql().map_err(|error| error.to_string())? {
4477 ToSqlOutput::Owned(value) => Ok(value),
4478 ToSqlOutput::Borrowed(value) => Ok(value.into()),
4479 _ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
4480 }
4481}
4482
4483fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
4491 use ssp2::segment::ColumnType;
4492 let mismatch = || {
4493 Err(format!(
4494 "image column {:?} holds a value of the wrong type",
4495 column.name
4496 ))
4497 };
4498 match value {
4499 ValueRef::Null => {
4500 if !column.nullable {
4501 return Err(format!(
4502 "image column {:?} is NULL but not nullable",
4503 column.name
4504 ));
4505 }
4506 Ok(ToSqlOutput::Owned(SqlValue::Null))
4507 }
4508 ValueRef::Integer(i) => match column.ty {
4509 ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
4510 ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
4511 ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
4512 _ => mismatch(),
4513 },
4514 ValueRef::Real(_) => match column.ty {
4515 ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
4516 _ => mismatch(),
4517 },
4518 ValueRef::Text(t) => {
4519 std::str::from_utf8(t)
4520 .map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
4521 match column.ty {
4522 ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
4523 Ok(ToSqlOutput::Borrowed(value))
4524 }
4525 _ => mismatch(),
4526 }
4527 }
4528 ValueRef::Blob(_) => match column.ty {
4529 ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
4531 _ => mismatch(),
4532 },
4533 }
4534}
4535
4536fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
4537 use rusqlite::types::ValueRef;
4538 match value {
4539 ValueRef::Null => Value::Null,
4540 ValueRef::Integer(i) => match column.ty {
4541 ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
4542 ssp2::segment::ColumnType::Float => {
4543 serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
4544 }
4545 _ => Value::from(i),
4546 },
4547 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
4548 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
4549 ValueRef::Blob(b) => {
4550 let mut map = Map::new();
4551 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
4552 Value::Object(map)
4553 }
4554 }
4555}
4556
4557fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
4561 Ok(match value {
4562 Value::Null => SqlValue::Null,
4563 Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
4564 Value::Number(n) => {
4565 if let Some(i) = n.as_i64() {
4566 SqlValue::Integer(i)
4567 } else if let Some(f) = n.as_f64() {
4568 SqlValue::Real(f)
4569 } else {
4570 return Err(format!("query param number {n} is out of range"));
4571 }
4572 }
4573 Value::String(s) => SqlValue::Text(s.clone()),
4574 Value::Object(_) => {
4575 if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
4576 SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
4577 } else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
4578 SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
4579 format!("query bigint param {decimal:?} is outside SQLite's i64 range")
4580 })?)
4581 } else {
4582 return Err(
4583 "query object param must be a {$bytes: hex} or {$bigint: decimal} value"
4584 .to_owned(),
4585 );
4586 }
4587 }
4588 Value::Array(_) => return Err("query array params are not supported".to_owned()),
4589 })
4590}
4591
4592fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
4597 use rusqlite::types::ValueRef;
4598 match value {
4599 ValueRef::Null => Value::Null,
4600 ValueRef::Integer(i) => {
4601 const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
4605 if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
4606 Value::from(i)
4607 } else {
4608 let mut map = Map::new();
4609 map.insert("$bigint".to_owned(), Value::from(i.to_string()));
4610 Value::Object(map)
4611 }
4612 }
4613 ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
4614 ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
4615 ValueRef::Blob(b) => {
4616 let mut map = Map::new();
4617 map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
4618 Value::Object(map)
4619 }
4620 }
4621}
4622
4623fn query_connection(
4624 conn: &Connection,
4625 sql: &str,
4626 params: &[QueryValue],
4627) -> Result<Vec<QueryRow>, String> {
4628 crate::query_guard::assert_read_only_query(sql)?;
4629 let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
4630 let bound: Vec<SqlValue> = params
4631 .iter()
4632 .map(json_param_to_sql)
4633 .collect::<Result<_, _>>()?;
4634 let mut stmt = conn.prepare(&lowered_sql).map_err(|e| e.to_string())?;
4635 let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
4636 let bound_refs: Vec<&dyn rusqlite::ToSql> =
4637 bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
4638 let mut sql_rows = stmt
4639 .query(bound_refs.as_slice())
4640 .map_err(|e| e.to_string())?;
4641 let mut out = Vec::new();
4642 while let Some(row) = sql_rows.next().map_err(|e| e.to_string())? {
4643 let mut record = Map::new();
4644 for (i, name) in column_names.iter().enumerate() {
4645 let value = row.get_ref(i).map_err(|e| e.to_string())?;
4646 record.insert(name.clone(), sql_ref_to_json_dynamic(value));
4647 }
4648 out.push(record);
4649 }
4650 Ok(out)
4651}
4652
4653fn persisted_window_state(conn: &Connection, base: &WindowBase) -> Result<WindowState, String> {
4654 let mut stmt = conn
4655 .prepare(
4656 "SELECT windows.unit, subscriptions.state_json
4657 FROM _syncular_windows AS windows
4658 JOIN _syncular_subscriptions AS subscriptions
4659 ON subscriptions.id = windows.sub_id
4660 WHERE windows.base = ?1
4661 ORDER BY windows.unit ASC",
4662 )
4663 .map_err(|error| error.to_string())?;
4664 let rows = stmt
4665 .query_map(rusqlite::params![window_base_key(base)], |row| {
4666 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
4667 })
4668 .map_err(|error| error.to_string())?;
4669 let mut units = Vec::new();
4670 let mut pending = Vec::new();
4671 for row in rows {
4672 let (unit, raw) = row.map_err(|error| error.to_string())?;
4673 let state: Value = serde_json::from_str(&raw)
4674 .map_err(|error| format!("invalid persisted window subscription: {error}"))?;
4675 let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
4676 || state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
4677 || state
4678 .get("bootstrapState")
4679 .is_some_and(|value| !value.is_null());
4680 if is_pending {
4681 pending.push(unit.clone());
4682 }
4683 units.push(unit);
4684 }
4685 Ok(WindowState { units, pending })
4686}
4687
4688fn snapshot_connection(
4689 conn: &Connection,
4690 sql: &str,
4691 params: &[Value],
4692 coverage: &[WindowCoverage],
4693) -> Result<QuerySnapshot, String> {
4694 conn.execute_batch("SAVEPOINT syncular_snapshot_read")
4695 .map_err(|error| error.to_string())?;
4696 let result = (|| {
4697 let revision = conn
4698 .query_row(
4699 "SELECT value FROM _syncular_meta WHERE key = ?1",
4700 rusqlite::params![LOCAL_REVISION_KEY],
4701 |row| row.get::<_, String>(0),
4702 )
4703 .ok()
4704 .and_then(|value| value.parse::<u64>().ok())
4705 .unwrap_or(0);
4706 let rows = query_connection(conn, sql, params)?;
4707 let mut pending = Vec::new();
4708 let mut missing = Vec::new();
4709 for requested in coverage {
4710 let base_key = window_base_key(&requested.base);
4711 let state = persisted_window_state(conn, &requested.base)?;
4712 for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
4713 let reference = WindowUnitRef {
4714 base_key: base_key.clone(),
4715 unit: unit.clone(),
4716 };
4717 if !state.units.iter().any(|held| held == &unit) {
4718 missing.push(reference);
4719 } else if state.pending.iter().any(|held| held == &unit) {
4720 pending.push(reference);
4721 }
4722 }
4723 }
4724 Ok(QuerySnapshot {
4725 revision: revision.to_string(),
4726 rows,
4727 coverage: CoverageSnapshot {
4728 complete: pending.is_empty() && missing.is_empty(),
4729 pending,
4730 missing,
4731 },
4732 })
4733 })();
4734 match result {
4735 Ok(snapshot) => {
4736 conn.execute_batch("RELEASE syncular_snapshot_read")
4737 .map_err(|error| error.to_string())?;
4738 Ok(snapshot)
4739 }
4740 Err(error) => {
4741 let _ = conn.execute_batch(
4742 "ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
4743 );
4744 Err(error)
4745 }
4746 }
4747}
4748
4749pub struct FileQuerySnapshotReader {
4754 path: String,
4755 conn: Option<Connection>,
4756}
4757
4758impl FileQuerySnapshotReader {
4759 #[must_use]
4760 pub fn new(path: impl Into<String>) -> Self {
4761 Self {
4762 path: path.into(),
4763 conn: None,
4764 }
4765 }
4766
4767 fn connection(&mut self) -> Result<&Connection, String> {
4768 if self.conn.is_none() {
4769 let conn = Connection::open_with_flags(
4770 &self.path,
4771 OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
4772 )
4773 .map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
4774 conn.busy_timeout(std::time::Duration::from_millis(250))
4775 .map_err(|error| error.to_string())?;
4776 self.conn = Some(conn);
4777 }
4778 self.conn
4779 .as_ref()
4780 .ok_or_else(|| "read sidecar connection missing".to_owned())
4781 }
4782
4783 pub fn query_snapshot(
4784 &mut self,
4785 sql: &str,
4786 params: &[Value],
4787 coverage: &[WindowCoverage],
4788 ) -> Result<QuerySnapshot, String> {
4789 snapshot_connection(self.connection()?, sql, params, coverage)
4790 }
4791}
4792
4793impl SyncClient {
4794 pub fn new_with_identity(
4795 client_id: Option<String>,
4796 schema_json: &Value,
4797 limits: ClientLimits,
4798 ) -> Result<Self, String> {
4799 let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
4800 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
4801 Self::with_connection(resolved, schema_json, limits, conn)
4802 }
4803
4804 pub fn new(
4805 client_id: String,
4806 schema_json: &Value,
4807 limits: ClientLimits,
4808 ) -> Result<Self, String> {
4809 let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
4810 Self::with_connection(client_id, schema_json, limits, conn)
4811 }
4812
4813 pub fn open_path(
4819 client_id: String,
4820 schema_json: &Value,
4821 limits: ClientLimits,
4822 path: &str,
4823 ) -> Result<Self, String> {
4824 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
4825 Self::with_connection(client_id, schema_json, limits, conn)
4826 }
4827
4828 pub fn open_path_with_identity(
4829 client_id: Option<String>,
4830 schema_json: &Value,
4831 limits: ClientLimits,
4832 path: &str,
4833 ) -> Result<Self, String> {
4834 let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
4835 conn.busy_timeout(std::time::Duration::from_millis(250))
4841 .map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
4842 conn.pragma_update(None, "journal_mode", "WAL")
4843 .map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
4844 let persisted = conn
4845 .query_row(
4846 "SELECT value FROM _syncular_meta WHERE key = 'clientId'",
4847 [],
4848 |row| row.get::<_, String>(0),
4849 )
4850 .ok();
4851 let resolved = persisted
4852 .clone()
4853 .or(client_id.clone())
4854 .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
4855 if let (Some(existing), Some(requested)) = (persisted, client_id) {
4856 if existing != requested {
4857 return Err(format!(
4858 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
4859 ));
4860 }
4861 }
4862 Self::with_connection(resolved, schema_json, limits, conn)
4863 }
4864
4865 #[must_use]
4866 pub fn client_id(&self) -> &str {
4867 &self.client_id
4868 }
4869
4870 pub fn with_connection(
4877 client_id: String,
4878 schema_json: &Value,
4879 limits: ClientLimits,
4880 conn: Connection,
4881 ) -> Result<Self, String> {
4882 if limits.outcome_retention_max_entries == Some(0) {
4883 return Err(
4884 "sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
4885 );
4886 }
4887 let schema = parse_schema_json(schema_json)?;
4888 let mut client = SyncClient {
4889 #[cfg(feature = "bench-internals")]
4890 benchmark_phases: Recorder::default(),
4891 conn,
4892 schema,
4893 client_id,
4894 limits,
4895 subs: Vec::new(),
4896 outbox: Vec::new(),
4897 conflicts: Vec::new(),
4898 rejections: Vec::new(),
4899 schema_floor: None,
4900 lease_state: None,
4901 stopped: false,
4902 upgrading: false,
4903 sync_needed: false,
4904 realtime_connected: false,
4905 presence: HashMap::new(),
4906 now_ms: None,
4907 encryption: crate::values::EncryptionConfig::default(),
4908 security_preflight: false,
4909 insert_sql: RefCell::new(HashMap::new()),
4910 overlay_dirty: Cell::new(false),
4911 #[cfg(test)]
4912 overlay_rebuild_count: Cell::new(0),
4913 #[cfg(test)]
4914 outcome_prune_count: Cell::new(0),
4915 progress: ProgressObserver::default(),
4916 change_queue: VecDeque::new(),
4917 sync_intent_queue: VecDeque::new(),
4918 retry_delay_ms: 250,
4919 last_round: None,
4920 last_change: None,
4921 };
4922 client
4926 .conn
4927 .set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
4928 client.create_bookkeeping_tables()?;
4932 match client.get_meta(CLIENT_ID_KEY) {
4933 Some(existing) if existing != client.client_id => {
4934 return Err(format!(
4935 "client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
4936 client.client_id
4937 ));
4938 }
4939 None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
4940 _ => {}
4941 }
4942 client.restore_persisted_state()?;
4943 let marker = client
4944 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
4945 .and_then(|value| value.parse::<i32>().ok());
4946 match marker {
4947 None => {
4948 client.create_synced_tables()?;
4949 client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
4950 }
4951 Some(version) if version == client.schema.version => {
4952 client.create_synced_tables()?;
4953 }
4954 Some(_) => client.run_schema_reset()?,
4955 }
4956 client.clear_satisfied_persisted_schema_floor();
4957 client.prune_unknown_subscriptions()?;
4958 if marker == Some(client.schema.version) && !client.outbox.is_empty() {
4959 client.overlay_dirty.set(true);
4962 client.rebuild_overlay();
4963 }
4964 client.enqueue_startup_sync_if_needed()?;
4970 Ok(client)
4971 }
4972
4973 pub fn set_now_ms(&mut self, now_ms: i64) {
4976 self.now_ms = Some(now_ms);
4977 }
4978
4979 pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
4983 self.encryption = encryption;
4984 }
4985
4986 #[must_use]
4987 pub fn security_lifecycle(&self) -> &'static str {
4988 if self.security_preflight {
4989 "preflight"
4990 } else {
4991 "active"
4992 }
4993 }
4994
4995 #[must_use]
4996 pub fn security_preflight(&self) -> bool {
4997 self.security_preflight
4998 }
4999
5000 pub fn begin_security_preflight(&mut self) {
5007 self.seal_security_on_teardown();
5008 self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
5011 }
5012
5013 pub fn seal_security_on_teardown(&mut self) {
5023 self.security_preflight = true;
5024 self.encryption = crate::values::EncryptionConfig::default();
5025 self.sync_intent_queue.clear();
5026 }
5027
5028 pub fn activate_security(
5030 &mut self,
5031 encryption: crate::values::EncryptionConfig,
5032 ) -> Result<(), String> {
5033 if !self.security_preflight {
5034 return Err(
5035 "sync.invalid_request: activateSecurity requires security preflight".to_owned(),
5036 );
5037 }
5038 self.enqueue_startup_sync_if_needed()?;
5039 self.encryption = encryption;
5040 self.security_preflight = false;
5041 self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
5042 Ok(())
5043 }
5044
5045 fn clock_now_ms(&self) -> i64 {
5046 self.now_ms.unwrap_or_else(|| {
5047 std::time::SystemTime::now()
5048 .duration_since(std::time::UNIX_EPOCH)
5049 .map(|d| d.as_millis() as i64)
5050 .unwrap_or(0)
5051 })
5052 }
5053
5054 fn create_bookkeeping_tables(&self) -> Result<(), String> {
5055 self.conn
5057 .execute_batch(
5058 "CREATE TABLE IF NOT EXISTS _syncular_outbox (
5059 seq INTEGER PRIMARY KEY AUTOINCREMENT,
5060 commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
5061 CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
5062 seq INTEGER PRIMARY KEY AUTOINCREMENT,
5063 client_commit_id TEXT NOT NULL UNIQUE,
5064 status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
5065 recorded_at_ms INTEGER NOT NULL,
5066 results_json TEXT NOT NULL,
5067 operations_json TEXT,
5068 resolution TEXT NOT NULL DEFAULT 'active'
5069 CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
5070 resolved_at_ms INTEGER,
5071 replacement_client_commit_id TEXT);
5072 CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
5073 ON _syncular_commit_outcomes(resolution, seq);
5074 CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
5075 id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
5076 CREATE TABLE IF NOT EXISTS _syncular_meta (
5077 key TEXT PRIMARY KEY, value TEXT NOT NULL);
5078 CREATE TABLE IF NOT EXISTS _syncular_windows (
5079 base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
5080 PRIMARY KEY (base, unit));
5081 CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
5082 sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
5083 effective_scopes TEXT NOT NULL);
5084 CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
5085 commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
5086 PRIMARY KEY(commit_id, blob_id));
5087 CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
5088 ON _syncular_blob_commit_refs(blob_id);",
5089 )
5090 .map_err(|e| e.to_string())?;
5091 let _ = self
5094 .conn
5095 .execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
5096 if self.get_meta(LOCAL_REVISION_KEY).is_none() {
5097 self.set_meta(LOCAL_REVISION_KEY, "0");
5098 }
5099 if self.schema_has_blobs() {
5102 let exists = self
5103 .conn
5104 .query_row(
5105 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
5106 [],
5107 |row| row.get::<_, bool>(0),
5108 )
5109 .map_err(|e| e.to_string())?;
5110 if exists {
5111 let mut statement = self
5112 .conn
5113 .prepare("PRAGMA table_info(_syncular_blobs)")
5114 .map_err(|e| e.to_string())?;
5115 let mut columns = statement
5116 .query_map([], |row| row.get::<_, String>(1))
5117 .map_err(|e| e.to_string())?
5118 .collect::<Result<Vec<_>, _>>()
5119 .map_err(|e| e.to_string())?;
5120 columns.sort();
5121 let mut expected = vec![
5122 "blob_id",
5123 "byte_length",
5124 "bytes",
5125 "created_at_ms",
5126 "media_type",
5127 ];
5128 expected.sort_unstable();
5129 if columns != expected {
5130 return Err(
5131 "sync.schema_mismatch: Local blob schema is incompatible".to_owned()
5132 );
5133 }
5134 } else {
5135 self.conn
5136 .execute_batch(
5137 "CREATE TABLE _syncular_blobs(
5138 blob_id TEXT PRIMARY KEY,
5139 bytes BLOB NOT NULL,
5140 byte_length INTEGER NOT NULL,
5141 media_type TEXT,
5142 created_at_ms INTEGER NOT NULL);",
5143 )
5144 .map_err(|e| e.to_string())?;
5145 }
5146 self.conn
5147 .execute_batch(
5148 "CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
5149 blob_id TEXT PRIMARY KEY,
5150 media_type TEXT,
5151 created_at_ms INTEGER NOT NULL);",
5152 )
5153 .map_err(|e| e.to_string())?;
5154 }
5155 Ok(())
5156 }
5157
5158 fn schema_has_blobs(&self) -> bool {
5160 self.schema
5161 .tables
5162 .iter()
5163 .any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
5164 }
5165
5166 fn get_meta(&self, key: &str) -> Option<String> {
5169 self.conn
5170 .query_row(
5171 "SELECT value FROM _syncular_meta WHERE key = ?1",
5172 rusqlite::params![key],
5173 |row| row.get::<_, String>(0),
5174 )
5175 .ok()
5176 }
5177
5178 fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
5179 self.conn
5180 .query_row(
5181 "SELECT value FROM _syncular_meta WHERE key = ?1",
5182 rusqlite::params![key],
5183 |row| row.get::<_, String>(0),
5184 )
5185 .optional()
5186 .map_err(|_| {
5187 "sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
5188 })
5189 }
5190
5191 fn set_meta(&self, key: &str, value: &str) {
5192 let _ = self.conn.execute(
5193 "INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
5194 rusqlite::params![key, value],
5195 );
5196 }
5197
5198 fn delete_meta(&self, key: &str) {
5199 let _ = self.conn.execute(
5200 "DELETE FROM _syncular_meta WHERE key = ?1",
5201 rusqlite::params![key],
5202 );
5203 }
5204
5205 fn restore_persisted_state(&mut self) -> Result<(), String> {
5206 self.subs = {
5207 let mut stmt = self
5208 .conn
5209 .prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
5210 .map_err(|error| error.to_string())?;
5211 let rows = stmt
5212 .query_map([], |row| {
5213 Ok((
5214 row.get::<_, String>(0)?,
5215 row.get::<_, String>(1)?,
5216 row.get::<_, String>(2)?,
5217 ))
5218 })
5219 .map_err(|error| error.to_string())?;
5220 let mut subscriptions = Vec::new();
5221 for row in rows {
5222 let (id, table, raw) = row.map_err(|error| error.to_string())?;
5223 let state: Value = serde_json::from_str(&raw)
5224 .map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
5225 let requested = json_to_scope_map(
5226 state.get("requested").unwrap_or(&Value::Object(Map::new())),
5227 )?;
5228 let effective = state
5229 .get("effectiveScopes")
5230 .filter(|value| !value.is_null())
5231 .map(json_to_scope_map)
5232 .transpose()?;
5233 subscriptions.push(Subscription {
5234 id,
5235 table,
5236 requested,
5237 params: state
5238 .get("params")
5239 .and_then(Value::as_str)
5240 .map(str::to_owned),
5241 cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
5242 bootstrap_state: state
5243 .get("bootstrapState")
5244 .and_then(Value::as_str)
5245 .map(str::to_owned),
5246 state: SubState::parse(
5247 state
5248 .get("status")
5249 .and_then(Value::as_str)
5250 .unwrap_or("active"),
5251 ),
5252 reason_code: state
5253 .get("reasonCode")
5254 .and_then(Value::as_str)
5255 .map(str::to_owned),
5256 effective,
5257 synced_once: state
5258 .get("syncedOnce")
5259 .and_then(Value::as_bool)
5260 .unwrap_or(false),
5261 });
5262 }
5263 subscriptions
5264 };
5265
5266 self.outbox = {
5267 let mut stmt = self
5268 .conn
5269 .prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
5270 .map_err(|error| error.to_string())?;
5271 let rows = stmt
5272 .query_map([], |row| {
5273 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
5274 })
5275 .map_err(|error| error.to_string())?;
5276 let mut commits = Vec::new();
5277 for row in rows {
5278 let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
5279 let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
5280 format!("invalid persisted outbox {client_commit_id:?}: {error}")
5281 })?;
5282 let mut ops = Vec::with_capacity(entries.len());
5283 for entry in entries {
5284 let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
5285 ops.push(OutboxOp {
5286 upsert: op == "upsert",
5287 table: entry
5288 .get("table")
5289 .and_then(Value::as_str)
5290 .ok_or_else(|| "persisted outbox operation missing table".to_owned())?
5291 .to_owned(),
5292 row_id: entry
5293 .get("rowId")
5294 .and_then(Value::as_str)
5295 .ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
5296 .to_owned(),
5297 base_version: entry.get("baseVersion").and_then(Value::as_i64),
5298 values: entry.get("values").and_then(Value::as_object).cloned(),
5301 });
5302 }
5303 commits.push(OutboxCommit {
5304 client_commit_id,
5305 ops,
5306 });
5307 }
5308 commits
5309 };
5310
5311 self.prune_commit_outcomes()?;
5312 let active = self.commit_outcomes(CommitOutcomeQuery {
5313 active_only: true,
5314 ..CommitOutcomeQuery::default()
5315 })?;
5316 self.conflicts = active
5317 .iter()
5318 .flat_map(|outcome| outcome.results.iter())
5319 .filter_map(|result| match result {
5320 CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
5321 _ => None,
5322 })
5323 .collect();
5324 self.rejections = active
5325 .iter()
5326 .flat_map(|outcome| outcome.results.iter())
5327 .filter_map(|result| match result {
5328 CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
5329 _ => None,
5330 })
5331 .collect();
5332
5333 self.lease_state = self
5334 .get_meta(LEASE_STATE_KEY)
5335 .map(|raw| serde_json::from_str(&raw))
5336 .transpose()
5337 .map_err(|error| format!("invalid persisted lease state: {error}"))?;
5338 self.schema_floor = self
5339 .get_meta(SCHEMA_FLOOR_KEY)
5340 .map(|raw| serde_json::from_str(&raw))
5341 .transpose()
5342 .map_err(|error| format!("invalid persisted schema floor: {error}"))?;
5343 self.stopped = self.schema_floor.is_some();
5344 if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
5347 self.security_preflight = true;
5348 }
5349 Ok(())
5350 }
5351
5352 fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
5355 let valid_tables: BTreeSet<String> = self
5356 .schema
5357 .tables
5358 .iter()
5359 .map(|table| table.name.clone())
5360 .collect();
5361 let stale_ids: Vec<String> = self
5362 .subs
5363 .iter()
5364 .filter(|sub| !valid_tables.contains(&sub.table))
5365 .map(|sub| sub.id.clone())
5366 .collect();
5367 for id in &stale_ids {
5368 self.conn
5369 .execute(
5370 "DELETE FROM _syncular_windows WHERE sub_id = ?1",
5371 rusqlite::params![id],
5372 )
5373 .map_err(|error| error.to_string())?;
5374 self.conn
5375 .execute(
5376 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
5377 rusqlite::params![id],
5378 )
5379 .map_err(|error| error.to_string())?;
5380 self.conn
5381 .execute(
5382 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
5383 rusqlite::params![id],
5384 )
5385 .map_err(|error| error.to_string())?;
5386 }
5387 self.subs.retain(|sub| valid_tables.contains(&sub.table));
5388 Ok(())
5389 }
5390
5391 #[must_use]
5392 pub fn local_revision(&self) -> u64 {
5393 self.get_meta(LOCAL_REVISION_KEY)
5394 .and_then(|value| value.parse().ok())
5395 .unwrap_or(0)
5396 }
5397
5398 #[must_use]
5399 pub fn status_snapshot(&self) -> SyncStatusSnapshot {
5400 SyncStatusSnapshot {
5401 current_schema_version: self.schema.version,
5402 outbox: self.outbox.len(),
5403 upgrading: self.upgrading,
5404 lease_state: self.lease_state.clone(),
5405 schema_floor: self.schema_floor.clone(),
5406 sync_needed: self.sync_needed,
5407 }
5408 }
5409
5410 pub fn diagnostics_snapshot(
5411 &self,
5412 request: &ClientDiagnosticsRequest,
5413 ) -> Result<ClientDiagnosticsSnapshot, String> {
5414 if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
5415 return Err(format!(
5416 "sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
5417 ));
5418 }
5419 let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
5420 for sub in &self.subs {
5421 let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
5422 let complete =
5423 sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
5424 let state = match sub.state {
5425 SubState::Revoked => "revoked",
5426 SubState::Failed => "failed",
5427 SubState::Active if reset => "reset",
5428 SubState::Active if complete => "complete",
5429 SubState::Active => "bootstrapping",
5430 };
5431 subscriptions.insert(
5432 sub.id.clone(),
5433 DiagnosticSubscription {
5434 id: sub.id.clone(),
5435 table: sub.table.clone(),
5436 state: state.to_owned(),
5437 complete,
5438 cursor: Some(sub.cursor),
5439 reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
5440 },
5441 );
5442 }
5443 for expected in &request.expected_subscriptions {
5444 if expected.id.is_empty() || expected.table.is_empty() {
5445 return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
5446 }
5447 if subscriptions
5448 .get(&expected.id)
5449 .is_some_and(|registered| registered.table != expected.table)
5450 {
5451 subscriptions.insert(
5452 expected.id.clone(),
5453 DiagnosticSubscription {
5454 id: expected.id.clone(),
5455 table: expected.table.clone(),
5456 state: "failed".to_owned(),
5457 complete: false,
5458 cursor: None,
5459 reason_code: Some("client.subscription_intent_mismatch".to_owned()),
5460 },
5461 );
5462 } else {
5463 subscriptions.entry(expected.id.clone()).or_insert_with(|| {
5464 DiagnosticSubscription {
5465 id: expected.id.clone(),
5466 table: expected.table.clone(),
5467 state: "unregistered".to_owned(),
5468 complete: false,
5469 cursor: None,
5470 reason_code: None,
5471 }
5472 });
5473 }
5474 }
5475 let mut ordered_subscriptions = Vec::new();
5476 let mut included = BTreeSet::new();
5477 for expected in &request.expected_subscriptions {
5478 if included.insert(expected.id.clone()) {
5479 if let Some(subscription) = subscriptions.get(&expected.id) {
5480 ordered_subscriptions.push(subscription.clone());
5481 }
5482 }
5483 }
5484 for (id, subscription) in &subscriptions {
5485 if included.insert(id.clone()) {
5486 ordered_subscriptions.push(subscription.clone());
5487 }
5488 }
5489 let subscriptions_truncated =
5490 ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
5491 ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
5492 let captured_at_ms = self.clock_now_ms();
5493 let lease = if let Some(error_code) = self
5494 .lease_state
5495 .as_ref()
5496 .and_then(|state| state.error_code.clone())
5497 {
5498 ClientDiagnosticsLease {
5499 state: "stopped".to_owned(),
5500 expires_at_ms: self
5501 .lease_state
5502 .as_ref()
5503 .and_then(|state| state.expires_at_ms),
5504 error_code: Some(Self::diagnostic_code(&error_code)),
5505 }
5506 } else if let Some(expires_at_ms) = self
5507 .lease_state
5508 .as_ref()
5509 .and_then(|state| state.expires_at_ms)
5510 {
5511 ClientDiagnosticsLease {
5512 state: if expires_at_ms <= captured_at_ms {
5513 "expired".to_owned()
5514 } else {
5515 "active".to_owned()
5516 },
5517 expires_at_ms: Some(expires_at_ms),
5518 error_code: None,
5519 }
5520 } else {
5521 ClientDiagnosticsLease {
5522 state: "none".to_owned(),
5523 expires_at_ms: None,
5524 error_code: None,
5525 }
5526 };
5527 let connectivity = match self.last_round.as_ref() {
5528 Some(round) if round.status == "succeeded" => "online",
5529 Some(round)
5530 if round.status == "failed"
5531 && round
5532 .error_code
5533 .as_deref()
5534 .is_some_and(Self::retryable_transport_code) =>
5535 {
5536 "offline"
5537 }
5538 _ => "unknown",
5539 };
5540 Ok(ClientDiagnosticsSnapshot {
5541 version: CLIENT_DIAGNOSTICS_VERSION,
5542 captured_at_ms,
5543 host: ClientDiagnosticsHost {
5544 kind: "direct".to_owned(),
5545 role: "single".to_owned(),
5546 connectivity: connectivity.to_owned(),
5547 realtime: if self.realtime_connected {
5548 "connected".to_owned()
5549 } else {
5550 "disconnected".to_owned()
5551 },
5552 },
5553 security_lifecycle: self.security_lifecycle().to_owned(),
5554 schema: ClientDiagnosticsSchema {
5555 current_version: self.schema.version,
5556 upgrading: self.upgrading,
5557 required_version: self
5558 .schema_floor
5559 .as_ref()
5560 .and_then(|floor| floor.required_schema_version),
5561 latest_version: self
5562 .schema_floor
5563 .as_ref()
5564 .and_then(|floor| floor.latest_schema_version),
5565 },
5566 replica: ClientDiagnosticsReplica {
5567 local_revision: self.local_revision().to_string(),
5568 sync_needed: self.sync_needed,
5569 pending_outbox: self.outbox.len(),
5570 },
5571 lease,
5572 subscriptions: ordered_subscriptions,
5573 subscriptions_truncated,
5574 last_round: self.last_round.clone(),
5575 last_change: self.last_change.clone(),
5576 storage: self.diagnostics_storage(),
5577 })
5578 }
5579
5580 fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
5581 let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
5582 let page_count: i64 = self
5583 .conn
5584 .query_row("PRAGMA page_count", [], |row| row.get(0))?;
5585 let page_size: i64 = self
5586 .conn
5587 .query_row("PRAGMA page_size", [], |row| row.get(0))?;
5588 let outbox_bytes: i64 = self
5590 .conn
5591 .prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
5592 .query_row([], |row| row.get(0))?;
5593 let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
5594 "SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
5595 )?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
5596 let blob_bytes = if self.schema_has_blobs() {
5597 self.conn
5598 .prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
5599 .query_row([], |row| row.get(0))?
5600 } else {
5601 0
5602 };
5603 let pressure = self
5604 .limits
5605 .blob_cache_max_bytes
5606 .is_some_and(|limit| blob_bytes > limit);
5607 Ok(ClientDiagnosticsStorage {
5608 status: if pressure { "pressure" } else { "healthy" }.to_owned(),
5609 database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
5610 pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
5611 retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
5612 retained_outcome_entries: Some(outcome_entries.max(0)),
5613 blob_cache_bytes_approx: Some(blob_bytes.max(0)),
5614 pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
5615 })
5616 };
5617 read().unwrap_or_else(|_| ClientDiagnosticsStorage {
5618 status: "unreadable".to_owned(),
5619 database_bytes_approx: None,
5620 pending_outbox_bytes_approx: None,
5621 retained_outcome_bytes_approx: None,
5622 retained_outcome_entries: None,
5623 blob_cache_bytes_approx: None,
5624 pressure_reason_code: None,
5625 })
5626 }
5627
5628 pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
5629 self.change_queue.drain(..).collect()
5630 }
5631
5632 pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
5633 self.sync_intent_queue.drain(..).collect()
5634 }
5635
5636 fn schedule_background_retry(&mut self) {
5637 self.sync_intent_queue.push_back(SyncIntent::Background {
5638 delay_ms: self.retry_delay_ms,
5639 });
5640 self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
5641 }
5642
5643 fn reset_background_retry(&mut self) {
5644 self.retry_delay_ms = 250;
5645 }
5646
5647 fn retryable_transport_code(code: &str) -> bool {
5648 code == "transport.failed"
5649 || code == "transport.unavailable"
5650 || code == "sync.transport_failed"
5651 }
5652
5653 fn diagnostic_code(code: &str) -> String {
5654 let valid = !code.is_empty()
5655 && code.len() <= 96
5656 && code.contains('.')
5657 && code
5658 .bytes()
5659 .next()
5660 .is_some_and(|byte| byte.is_ascii_lowercase())
5661 && code.bytes().all(|byte| {
5662 byte.is_ascii_lowercase()
5663 || byte.is_ascii_digit()
5664 || matches!(byte, b'.' | b'_' | b'-')
5665 });
5666 if valid {
5667 code.to_owned()
5668 } else {
5669 "client.unknown_failure".to_owned()
5670 }
5671 }
5672
5673 fn set_sync_needed(&mut self, value: bool, interactive: bool) {
5674 if self.sync_needed != value {
5675 if self.begin_observation("syncular_status").is_ok() {
5676 self.sync_needed = value;
5677 let batch = ChangeAccumulator {
5678 status: true,
5679 ..ChangeAccumulator::default()
5680 };
5681 if self.finish_observation("syncular_status", batch).is_err() {
5682 self.rollback_observation("syncular_status");
5683 }
5684 } else {
5685 self.sync_needed = value;
5686 }
5687 }
5688 if value && interactive {
5689 self.sync_intent_queue.push_back(SyncIntent::Interactive);
5690 }
5691 }
5692
5693 fn begin_observation(&self, name: &str) -> Result<(), String> {
5694 self.conn
5695 .execute_batch(&format!("SAVEPOINT {name}"))
5696 .map_err(|error| error.to_string())
5697 }
5698
5699 fn rollback_observation(&self, name: &str) {
5700 let _ = self
5701 .conn
5702 .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
5703 }
5704
5705 fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
5706 #[cfg(feature = "bench-internals")]
5707 let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
5708 if !batch.touched() {
5709 self.conn
5710 .execute_batch(&format!("RELEASE {name}"))
5711 .map_err(|error| error.to_string())?;
5712 return Ok(());
5713 }
5714 let revision = self
5715 .local_revision()
5716 .checked_add(1)
5717 .ok_or_else(|| "local revision exhausted u64".to_owned())?;
5718 self.conn
5719 .execute(
5720 "INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
5721 rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
5722 )
5723 .map_err(|error| error.to_string())?;
5724 let status = batch.status.then(|| self.status_snapshot());
5725 let event = ClientChangeBatch {
5726 revision: revision.to_string(),
5727 tables: batch
5728 .tables
5729 .into_iter()
5730 .map(|(table, scope_keys)| TableChange {
5731 table,
5732 scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
5733 })
5734 .collect(),
5735 windows: batch
5736 .windows
5737 .into_iter()
5738 .map(|((base_key, table), units)| WindowChange {
5739 base_key,
5740 table,
5741 units: units.into_iter().collect(),
5742 })
5743 .collect(),
5744 status,
5745 conflicts_changed: batch.conflicts,
5746 rejections_changed: batch.rejections,
5747 outcomes_changed: batch.outcomes,
5748 };
5749 self.conn
5750 .execute_batch(&format!("RELEASE {name}"))
5751 .map_err(|error| error.to_string())?;
5752 let mut diagnostic_tables = event
5753 .tables
5754 .iter()
5755 .map(|entry| entry.table.clone())
5756 .collect::<BTreeSet<_>>()
5757 .into_iter()
5758 .collect::<Vec<_>>();
5759 let mut diagnostic_windows = event
5760 .windows
5761 .iter()
5762 .map(|entry| entry.table.clone())
5763 .collect::<BTreeSet<_>>()
5764 .into_iter()
5765 .collect::<Vec<_>>();
5766 let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
5767 || diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
5768 diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
5769 diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
5770 self.last_change = Some(DiagnosticLastChange {
5771 revision: event.revision.clone(),
5772 recorded_at_ms: self.clock_now_ms(),
5773 tables: diagnostic_tables,
5774 windows: diagnostic_windows,
5775 domains_truncated,
5776 status_changed: event.status.is_some(),
5777 conflicts_changed: event.conflicts_changed,
5778 rejections_changed: event.rejections_changed,
5779 outcomes_changed: event.outcomes_changed,
5780 });
5781 self.change_queue.push_back(event);
5782 Ok(())
5783 }
5784
5785 fn record_scope_map(
5786 &self,
5787 batch: &mut ChangeAccumulator,
5788 table_name: &str,
5789 scopes: &[(String, Vec<String>)],
5790 ) {
5791 let Some(table) = self.schema.table(table_name) else {
5792 return;
5793 };
5794 for (variable, values) in scopes {
5795 let Some(scope) = table
5796 .scope_variables
5797 .iter()
5798 .find(|scope| &scope.variable == variable)
5799 else {
5800 continue;
5801 };
5802 for value in values {
5803 batch.scope(table_name, format!("{}:{value}", scope.prefix));
5804 }
5805 }
5806 }
5807
5808 fn record_row_scopes(
5810 &self,
5811 batch: &mut ChangeAccumulator,
5812 table_name: &str,
5813 row_id: &str,
5814 base: bool,
5815 ) -> bool {
5816 let Some(table) = self.schema.table(table_name) else {
5817 return false;
5818 };
5819 if table.scope_variables.is_empty() {
5820 return false;
5821 }
5822 let columns = table
5823 .scope_variables
5824 .iter()
5825 .map(|scope| quote_ident(&scope.column))
5826 .collect::<Vec<_>>()
5827 .join(", ");
5828 let full_table = if base {
5829 base_table(table_name)
5830 } else {
5831 visible_table(table_name)
5832 };
5833 let sql = format!(
5834 "SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
5835 row_id_predicate(table)
5836 );
5837 let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
5838 return false;
5839 };
5840 let values = stmt.query_row(rusqlite::params![row_id], |row| {
5841 let mut values = Vec::with_capacity(table.scope_variables.len());
5842 for index in 0..table.scope_variables.len() {
5843 values.push(row.get::<_, Option<String>>(index)?);
5844 }
5845 Ok(values)
5846 });
5847 let Ok(values) = values else {
5848 return false;
5849 };
5850 let mut recorded = false;
5851 for (scope, value) in table.scope_variables.iter().zip(values) {
5852 if let Some(value) = value {
5853 batch.scope(table_name, format!("{}:{value}", scope.prefix));
5854 recorded = true;
5855 }
5856 }
5857 recorded
5858 }
5859
5860 fn record_commit_changes(
5861 &self,
5862 batch: &mut ChangeAccumulator,
5863 tables: &[String],
5864 changes: &[ssp2::model::Change],
5865 ) {
5866 #[cfg(feature = "bench-internals")]
5867 let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
5868 for change in changes {
5869 let Some(table_name) = tables.get(change.table_index as usize) else {
5870 continue;
5871 };
5872 let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
5873 if let Some(table) = self.schema.table(table_name) {
5874 for (variable, value) in &change.scopes {
5875 if let Some(scope) = table
5876 .scope_variables
5877 .iter()
5878 .find(|scope| &scope.variable == variable)
5879 {
5880 batch.scope(table_name, format!("{}:{value}", scope.prefix));
5881 precise = true;
5882 }
5883 }
5884 }
5885 if !precise {
5886 batch.table(table_name);
5887 }
5888 }
5889 }
5890
5891 fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
5892 if effective.is_empty() {
5893 return false;
5894 }
5895 let Some(table) = self.schema.table(table_name) else {
5896 return false;
5897 };
5898 let mut clauses = Vec::new();
5899 let mut params = Vec::new();
5900 for (variable, values) in effective {
5901 let Some(column) = table.scope_column(variable) else {
5902 return false;
5903 };
5904 if values.is_empty() {
5905 return false;
5906 }
5907 let placeholders = values
5908 .iter()
5909 .map(|value| {
5910 params.push(SqlValue::Text(value.clone()));
5911 "?"
5912 })
5913 .collect::<Vec<_>>()
5914 .join(", ");
5915 clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
5916 }
5917 let sql = format!(
5918 "SELECT 1 FROM {} WHERE {} LIMIT 1",
5919 base_table(table_name),
5920 clauses.join(" AND ")
5921 );
5922 self.conn
5923 .query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
5924 .is_ok()
5925 }
5926
5927 pub fn upgrading(&self) -> bool {
5929 self.upgrading
5930 }
5931
5932 pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
5938 let new_schema = parse_schema_json(schema_json)?;
5939 let marker: Option<i32> = self
5940 .get_meta(LOCAL_SCHEMA_VERSION_KEY)
5941 .and_then(|v| v.parse().ok());
5942 self.schema = new_schema;
5943 self.create_bookkeeping_tables()?;
5944 if marker != Some(self.schema.version) {
5945 self.run_schema_reset()?;
5946 }
5947 self.prune_unknown_subscriptions()?;
5948 self.enqueue_startup_sync_if_needed()?;
5952 Ok(())
5953 }
5954
5955 fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
5956 let startup_work = !self.stopped
5957 && (!self.outbox.is_empty()
5958 || !self.load_pending_evictions()?.is_empty()
5959 || self.subs.iter().any(|sub| sub.state == SubState::Active));
5960 if startup_work {
5961 self.sync_needed = true;
5962 self.sync_intent_queue.push_back(SyncIntent::Interactive);
5963 }
5964 Ok(())
5965 }
5966
5967 fn clear_satisfied_persisted_schema_floor(&mut self) {
5974 let satisfied = self
5975 .schema_floor
5976 .as_ref()
5977 .and_then(|floor| floor.required_schema_version)
5978 .is_some_and(|required| self.schema.version >= required);
5979 if !satisfied {
5980 return;
5981 }
5982 self.schema_floor = None;
5983 self.stopped = false;
5984 self.delete_meta(SCHEMA_FLOOR_KEY);
5985 }
5986
5987 fn run_schema_reset(&mut self) -> Result<(), String> {
5993 self.begin_observation("syncular_schema_reset")?;
5994 let mut batch = ChangeAccumulator::default();
5995 let result = self.run_schema_reset_observed(&mut batch, true);
5996 if let Err(error) = result {
5997 self.rollback_observation("syncular_schema_reset");
5998 return Err(error);
5999 }
6000 if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
6001 self.rollback_observation("syncular_schema_reset");
6002 return Err(error);
6003 }
6004 Ok(())
6005 }
6006
6007 fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
6008 let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
6009 self.begin_observation("syncular_log_epoch_reset")?;
6010 let mut batch = ChangeAccumulator::default();
6011 let result = self.run_schema_reset_observed(&mut batch, false);
6012 if let Err(error) = result {
6013 self.rollback_observation("syncular_log_epoch_reset");
6014 return Err(error);
6015 }
6016 self.set_meta(LOG_EPOCH_KEY, log_epoch);
6017 self.sync_needed = true;
6018 batch.status = true;
6019 if let Err(error) = self.finish_observation("syncular_log_epoch_reset", batch) {
6020 self.rollback_observation("syncular_log_epoch_reset");
6021 return Err(error);
6022 }
6023 self.sync_intent_queue.push_back(SyncIntent::Interactive);
6024 Ok(resets)
6025 }
6026
6027 fn run_schema_reset_observed(
6028 &mut self,
6029 batch: &mut ChangeAccumulator,
6030 drop_incompatible: bool,
6031 ) -> Result<(), String> {
6032 self.upgrading = true;
6033 batch.status = true;
6034 for table in &self.schema.tables {
6035 batch.table(&table.name);
6036 }
6037 for (base_key, unit, table) in self.load_registered_window_units() {
6038 batch.window(&base_key, &table, &unit);
6039 }
6040 self.insert_sql.borrow_mut().clear();
6042 self.overlay_dirty.set(true);
6043 let virtual_tables: Vec<String> = {
6051 let mut stmt = self
6052 .conn
6053 .prepare(
6054 "SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
6055 )
6056 .map_err(|e| e.to_string())?;
6057 let rows = stmt
6058 .query_map([], |row| row.get::<_, String>(0))
6059 .map_err(|e| e.to_string())?;
6060 rows.filter_map(Result::ok)
6061 .filter(|name| is_synced_table_name(name))
6062 .collect()
6063 };
6064 for name in virtual_tables {
6065 self.conn
6066 .execute_batch(&format!(
6067 "DROP TABLE IF EXISTS {}",
6068 quote_ident(&format!("_syncular_fts_{name}"))
6069 ))
6070 .map_err(|e| e.to_string())?;
6071 self.conn
6072 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
6073 .map_err(|e| e.to_string())?;
6074 }
6075 let existing: Vec<String> = {
6076 let mut stmt = self
6077 .conn
6078 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
6079 .map_err(|e| e.to_string())?;
6080 let rows = stmt
6081 .query_map([], |row| row.get::<_, String>(0))
6082 .map_err(|e| e.to_string())?;
6083 rows.filter_map(Result::ok)
6084 .filter(|name| is_synced_table_name(name))
6085 .collect()
6086 };
6087 for name in &existing {
6088 if let Some(table) = name.strip_prefix("_syncular_base_") {
6089 batch.table(table);
6090 } else if !name.starts_with("_syncular_") {
6091 batch.table(name);
6092 }
6093 }
6094 for name in existing {
6095 self.conn
6096 .execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
6097 .map_err(|e| e.to_string())?;
6098 }
6099 self.create_synced_tables()?;
6101 for sub in &mut self.subs {
6103 sub.cursor = -1;
6104 sub.bootstrap_state = None;
6105 sub.effective = None;
6106 sub.state = SubState::Active;
6107 sub.reason_code = None;
6108 sub.synced_once = false;
6109 }
6110 let subs = self.subs.clone();
6111 for sub in &subs {
6112 self.persist_sub(sub)?;
6113 }
6114 self.stopped = false;
6116 self.schema_floor = None;
6117 self.delete_meta(SCHEMA_FLOOR_KEY);
6118 self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
6120 if drop_incompatible && self.drop_incompatible_outbox()? {
6124 batch.rejections = true;
6125 batch.status = true;
6126 batch.outcomes = true;
6127 }
6128 self.rebuild_overlay();
6130 Ok(())
6131 }
6132
6133 fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
6137 let schema = &self.schema;
6138 let incompatible = self
6139 .outbox
6140 .iter()
6141 .filter(|commit| {
6142 commit.ops.iter().any(|op| {
6143 if !op.upsert {
6144 return false;
6145 }
6146 match schema.table(&op.table) {
6147 None => true,
6148 Some(table) => op.values.as_ref().is_some_and(|values| {
6149 values
6150 .keys()
6151 .any(|key| !table.columns.iter().any(|c| &c.name == key))
6152 }),
6153 }
6154 })
6155 })
6156 .cloned()
6157 .collect::<Vec<_>>();
6158 if incompatible.is_empty() {
6159 return Ok(false);
6160 }
6161 let mut rejections = Vec::new();
6162 for commit in &incompatible {
6163 let results = commit
6164 .ops
6165 .iter()
6166 .enumerate()
6167 .map(|(op_index, operation)| {
6168 let rejection = RejectionRecord {
6169 client_commit_id: commit.client_commit_id.clone(),
6170 op_index: op_index as i32,
6171 code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
6172 message: "the persisted commit cannot encode under the current schema"
6173 .to_owned(),
6174 retryable: false,
6175 details: None,
6176 operation: Some(CommitOperation::from(operation)),
6177 };
6178 rejections.push(rejection.clone());
6179 CommitOperationOutcome::Error { rejection }
6180 })
6181 .collect::<Vec<_>>();
6182 self.persist_commit_outcome(
6183 &commit.client_commit_id,
6184 CommitOutcomeStatus::Rejected,
6185 &results,
6186 Some(&commit.ops),
6187 )?;
6188 self.delete_outbox_persisted(&commit.client_commit_id)?;
6189 }
6190 self.prune_commit_outcomes()?;
6191 let incompatible_ids = incompatible
6192 .iter()
6193 .map(|commit| commit.client_commit_id.as_str())
6194 .collect::<BTreeSet<_>>();
6195 self.outbox
6196 .retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
6197 self.rejections.extend(rejections);
6198 Ok(true)
6199 }
6200
6201 fn create_synced_tables(&self) -> Result<(), String> {
6204 for table in &self.schema.tables {
6205 for (full, index_prefix) in [
6209 (base_table(&table.name), "_syncular_base_"),
6210 (visible_table(&table.name), ""),
6211 ] {
6212 let mut cols: Vec<String> =
6213 table.columns.iter().map(|c| quote_ident(&c.name)).collect();
6214 cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
6215 let sql = format!(
6216 "CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
6217 cols.join(", "),
6218 quote_ident(&table.primary_key)
6219 );
6220 self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
6221 for index in &table.indexes {
6225 let unique = if index.unique { "UNIQUE " } else { "" };
6226 let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
6227 let cols_sql = index
6228 .columns
6229 .iter()
6230 .map(|c| quote_ident(c))
6231 .collect::<Vec<_>>()
6232 .join(", ");
6233 let index_sql = format!(
6234 "CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
6235 );
6236 self.conn
6237 .execute(&index_sql, [])
6238 .map_err(|e| e.to_string())?;
6239 }
6240 }
6241 }
6242 for table in &self.schema.tables {
6245 for index in &table.fts_indexes {
6246 self.create_fts_projection(table, index)?;
6247 }
6248 }
6249 Ok(())
6250 }
6251
6252 fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
6253 let count: i64 = self
6254 .conn
6255 .query_row(
6256 "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
6257 rusqlite::params![index.name],
6258 |row| row.get(0),
6259 )
6260 .map_err(|e| e.to_string())?;
6261 Ok(count > 0)
6262 }
6263
6264 fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
6265 for suffix in ["ai", "ad", "au"] {
6266 self.conn
6267 .execute(
6268 &format!(
6269 "DROP TRIGGER IF EXISTS {}",
6270 quote_ident(&format!("{}_{suffix}", index.name))
6271 ),
6272 [],
6273 )
6274 .map_err(|e| e.to_string())?;
6275 }
6276 Ok(())
6277 }
6278
6279 fn create_fts_triggers(
6280 &self,
6281 table: &TableSchema,
6282 index: &FtsIndexSchema,
6283 ) -> Result<(), String> {
6284 let fts = quote_ident(&index.name);
6285 let source = visible_table(&table.name);
6286 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
6287 let pk = quote_ident(&table.primary_key);
6288 let projection_columns = std::iter::once(source_id.clone())
6289 .chain(index.columns.iter().map(|column| quote_ident(column)))
6290 .collect::<Vec<_>>()
6291 .join(", ");
6292 let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
6293 .chain(
6294 index
6295 .columns
6296 .iter()
6297 .map(|column| format!("new.{}", quote_ident(column))),
6298 )
6299 .collect::<Vec<_>>()
6300 .join(", ");
6301 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
6302 let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
6303 let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
6304 let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
6305 let bi = quote_ident(&format!("{}_bi", index.name));
6306 let ai = quote_ident(&format!("{}_ai", index.name));
6307 let ad = quote_ident(&format!("{}_ad", index.name));
6308 let au = quote_ident(&format!("{}_au", index.name));
6309 let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
6310 let sql = format!(
6311 "DROP TRIGGER IF EXISTS {bi};
6312 DROP TRIGGER IF EXISTS {ai};
6313 DROP TRIGGER IF EXISTS {ad};
6314 DROP TRIGGER IF EXISTS {au};
6315 CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
6316 CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
6317 CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
6318 CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
6319 );
6320 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
6321 }
6322
6323 fn rebuild_fts_projection(
6324 &self,
6325 table: &TableSchema,
6326 index: &FtsIndexSchema,
6327 ) -> Result<(), String> {
6328 let fts = quote_ident(&index.name);
6329 let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
6330 let pk = quote_ident(&table.primary_key);
6331 let indexed_columns = index
6332 .columns
6333 .iter()
6334 .map(|column| quote_ident(column))
6335 .collect::<Vec<_>>();
6336 let projection_columns = std::iter::once(source_id.clone())
6337 .chain(indexed_columns.iter().cloned())
6338 .collect::<Vec<_>>()
6339 .join(", ");
6340 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
6341 let sql = format!(
6342 "DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
6343 columns = indexed_columns.join(", "),
6344 source = visible_table(&table.name),
6345 );
6346 self.conn.execute_batch(&sql).map_err(|e| e.to_string())
6347 }
6348
6349 fn create_fts_projection(
6350 &self,
6351 table: &TableSchema,
6352 index: &FtsIndexSchema,
6353 ) -> Result<(), String> {
6354 self.begin_observation("syncular_fts_schema")?;
6355 let result = (|| {
6356 let existed = self.fts_projection_exists(index)?;
6357 let tokenizer = index.tokenize.replace('\'', "''");
6358 let columns = index
6359 .columns
6360 .iter()
6361 .map(|column| quote_ident(column))
6362 .collect::<Vec<_>>()
6363 .join(", ");
6364 let sql = format!(
6365 "CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
6366 fts = quote_ident(&index.name),
6367 source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
6368 );
6369 self.conn.execute(&sql, []).map_err(|error| {
6370 format!(
6371 "cannot create local FTS5 projection {:?}: {error}",
6372 index.name
6373 )
6374 })?;
6375 let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
6376 let mapped: bool = self
6377 .conn
6378 .query_row(
6379 "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
6380 [format!("_syncular_fts_{}", index.name)],
6381 |row| row.get(0),
6382 )
6383 .map_err(|e| e.to_string())?;
6384 self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|e| e.to_string())?;
6385 if !mapped && existed {
6386 self.conn
6387 .execute_batch(&format!(
6388 "INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
6389 quote_ident(FTS_SOURCE_ID_COLUMN),
6390 quote_ident(&index.name)
6391 ))
6392 .map_err(|e| e.to_string())?;
6393 }
6394 self.create_fts_triggers(table, index)?;
6395 if !existed {
6396 self.rebuild_fts_projection(table, index)?;
6397 }
6398 Ok(())
6399 })();
6400 if let Err(error) = result {
6401 self.rollback_observation("syncular_fts_schema");
6402 return Err(error);
6403 }
6404 if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
6405 self.rollback_observation("syncular_fts_schema");
6406 return Err(error.to_string());
6407 }
6408 Ok(())
6409 }
6410
6411 fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
6414 let state = serde_json::json!({
6415 "requested": scope_map_to_json(&sub.requested),
6416 "params": sub.params,
6417 "cursor": sub.cursor,
6418 "bootstrapState": sub.bootstrap_state,
6419 "status": sub.state.name(),
6420 "reasonCode": sub.reason_code,
6421 "effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
6422 "syncedOnce": sub.synced_once,
6423 });
6424 self.conn.execute(
6425 "INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
6426 rusqlite::params![sub.id, sub.table, state.to_string()],
6427 ).map(|_| ()).map_err(|error| error.to_string())
6428 }
6429
6430 fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
6431 let ops: Vec<Value> = commit
6432 .ops
6433 .iter()
6434 .map(|op| {
6435 serde_json::json!({
6436 "op": if op.upsert { "upsert" } else { "delete" },
6437 "table": op.table,
6438 "rowId": op.row_id,
6439 "baseVersion": op.base_version,
6440 "values": op.values.clone().map(Value::Object),
6441 })
6442 })
6443 .collect();
6444 self.conn
6445 .execute(
6446 "INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
6447 rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
6448 )
6449 .map_err(|error| error.to_string())?;
6450 for operation in &commit.ops {
6451 let Some(values) = &operation.values else {
6452 continue;
6453 };
6454 let Some(table) = self.schema.table(&operation.table) else {
6455 continue;
6456 };
6457 for column in table
6458 .columns
6459 .iter()
6460 .filter(|column| column.ty == ColumnType::BlobRef)
6461 {
6462 let Some(Value::String(raw)) = values.get(&column.name) else {
6463 continue;
6464 };
6465 let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
6466 continue;
6467 };
6468 let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
6469 continue;
6470 };
6471 self.conn
6472 .execute(
6473 "INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
6474 SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
6475 rusqlite::params![commit.client_commit_id, blob_id],
6476 )
6477 .map_err(|error| error.to_string())?;
6478 }
6479 }
6480 Ok(())
6481 }
6482
6483 fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
6484 self.conn
6485 .execute(
6486 "DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
6487 rusqlite::params![client_commit_id],
6488 )
6489 .map_err(|error| error.to_string())?;
6490 self.conn
6491 .execute(
6492 "DELETE FROM _syncular_outbox WHERE commit_id = ?1",
6493 rusqlite::params![client_commit_id],
6494 )
6495 .map(|_| ())
6496 .map_err(|error| error.to_string())
6497 }
6498
6499 fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
6500 match status {
6501 CommitOutcomeStatus::Applied => "applied",
6502 CommitOutcomeStatus::Cached => "cached",
6503 CommitOutcomeStatus::Conflict => "conflict",
6504 CommitOutcomeStatus::Rejected => "rejected",
6505 }
6506 }
6507
6508 fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
6509 match resolution {
6510 CommitOutcomeResolution::Active => "active",
6511 CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
6512 CommitOutcomeResolution::Superseded => "superseded",
6513 CommitOutcomeResolution::Dismissed => "dismissed",
6514 }
6515 }
6516
6517 fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
6518 match value {
6519 "applied" => Ok(CommitOutcomeStatus::Applied),
6520 "cached" => Ok(CommitOutcomeStatus::Cached),
6521 "conflict" => Ok(CommitOutcomeStatus::Conflict),
6522 "rejected" => Ok(CommitOutcomeStatus::Rejected),
6523 _ => Err(format!("invalid persisted commit outcome status {value:?}")),
6524 }
6525 }
6526
6527 fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
6528 match value {
6529 "active" => Ok(CommitOutcomeResolution::Active),
6530 "resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
6531 "superseded" => Ok(CommitOutcomeResolution::Superseded),
6532 "dismissed" => Ok(CommitOutcomeResolution::Dismissed),
6533 _ => Err(format!(
6534 "invalid persisted commit outcome resolution {value:?}"
6535 )),
6536 }
6537 }
6538
6539 fn persist_commit_outcome(
6540 &self,
6541 client_commit_id: &str,
6542 status: CommitOutcomeStatus,
6543 results: &[CommitOperationOutcome],
6544 operations: Option<&[OutboxOp]>,
6545 ) -> Result<(), String> {
6546 let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
6547 let operations_json = operations
6548 .map(|items| {
6549 serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
6550 })
6551 .transpose()
6552 .map_err(|error| error.to_string())?;
6553 self.conn
6554 .execute(
6555 "INSERT INTO _syncular_commit_outcomes (
6556 client_commit_id, status, recorded_at_ms, results_json,
6557 operations_json, resolution
6558 ) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
6559 rusqlite::params![
6560 client_commit_id,
6561 Self::outcome_status_name(status),
6562 self.clock_now_ms(),
6563 results_json,
6564 operations_json
6565 ],
6566 )
6567 .map(|_| ())
6568 .map_err(|error| error.to_string())
6569 }
6570
6571 fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
6572 let StoredCommitOutcomeRow {
6573 sequence,
6574 client_commit_id,
6575 status,
6576 recorded_at_ms,
6577 results_json,
6578 operations_json,
6579 resolution,
6580 resolved_at_ms,
6581 replacement_client_commit_id,
6582 } = row;
6583 Ok(CommitOutcome {
6584 sequence,
6585 client_commit_id,
6586 status: Self::parse_outcome_status(&status)?,
6587 recorded_at_ms,
6588 results: serde_json::from_str(&results_json)
6589 .map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
6590 operations: operations_json
6591 .map(|value| {
6592 serde_json::from_str(&value).map_err(|error| {
6593 format!("invalid persisted commit outcome operations: {error}")
6594 })
6595 })
6596 .transpose()?,
6597 resolution: Self::parse_outcome_resolution(&resolution)?,
6598 resolved_at_ms,
6599 replacement_client_commit_id,
6600 })
6601 }
6602
6603 pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
6604 let row = self
6605 .conn
6606 .query_row(
6607 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
6608 resolution, resolved_at_ms, replacement_client_commit_id
6609 FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
6610 rusqlite::params![client_commit_id],
6611 |row| {
6612 Ok(StoredCommitOutcomeRow {
6613 sequence: row.get(0)?,
6614 client_commit_id: row.get(1)?,
6615 status: row.get(2)?,
6616 recorded_at_ms: row.get(3)?,
6617 results_json: row.get(4)?,
6618 operations_json: row.get(5)?,
6619 resolution: row.get(6)?,
6620 resolved_at_ms: row.get(7)?,
6621 replacement_client_commit_id: row.get(8)?,
6622 })
6623 },
6624 )
6625 .optional()
6626 .map_err(|error| error.to_string())?;
6627 row.map(Self::outcome_from_row).transpose()
6628 }
6629
6630 pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
6631 if query.limit == Some(0) {
6632 return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
6633 }
6634 let mut sql = String::from(
6635 "SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
6636 resolution, resolved_at_ms, replacement_client_commit_id
6637 FROM _syncular_commit_outcomes",
6638 );
6639 if query.active_only {
6640 sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
6641 }
6642 sql.push_str(" ORDER BY seq DESC");
6643 if let Some(limit) = query.limit {
6644 sql.push_str(&format!(" LIMIT {limit}"));
6645 }
6646 let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
6647 let rows = stmt
6648 .query_map([], |row| {
6649 Ok(StoredCommitOutcomeRow {
6650 sequence: row.get(0)?,
6651 client_commit_id: row.get(1)?,
6652 status: row.get(2)?,
6653 recorded_at_ms: row.get(3)?,
6654 results_json: row.get(4)?,
6655 operations_json: row.get(5)?,
6656 resolution: row.get(6)?,
6657 resolved_at_ms: row.get(7)?,
6658 replacement_client_commit_id: row.get(8)?,
6659 })
6660 })
6661 .map_err(|error| error.to_string())?;
6662 let mut outcomes = Vec::new();
6663 for row in rows {
6664 outcomes.push(Self::outcome_from_row(
6665 row.map_err(|error| error.to_string())?,
6666 )?);
6667 }
6668 Ok(outcomes)
6669 }
6670
6671 fn prune_commit_outcomes(&self) -> Result<(), String> {
6672 #[cfg(test)]
6673 self.outcome_prune_count
6674 .set(self.outcome_prune_count.get() + 1);
6675 let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
6676 let count = self
6677 .conn
6678 .query_row(
6679 "SELECT COUNT(*) FROM _syncular_commit_outcomes",
6680 [],
6681 |row| row.get::<_, i64>(0),
6682 )
6683 .map_err(|error| error.to_string())? as usize;
6684 let excess = count.saturating_sub(max_entries);
6685 if excess == 0 {
6686 return Ok(());
6687 }
6688 let mut stmt = self
6689 .conn
6690 .prepare(
6691 "SELECT seq FROM _syncular_commit_outcomes
6692 WHERE status IN ('applied', 'cached') OR resolution != 'active'
6693 ORDER BY seq ASC LIMIT ?1",
6694 )
6695 .map_err(|error| error.to_string())?;
6696 let rows = stmt
6697 .query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
6698 .map_err(|error| error.to_string())?;
6699 let sequences = rows
6700 .collect::<Result<Vec<_>, _>>()
6701 .map_err(|error| error.to_string())?;
6702 drop(stmt);
6703 for sequence in sequences {
6704 self.conn
6705 .execute(
6706 "DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
6707 rusqlite::params![sequence],
6708 )
6709 .map_err(|error| error.to_string())?;
6710 }
6711 Ok(())
6712 }
6713
6714 pub fn subscribe(
6717 &mut self,
6718 id: String,
6719 table: String,
6720 scopes: Vec<(String, Vec<String>)>,
6721 params: Option<String>,
6722 ) -> Result<(), String> {
6723 if self.schema.table(&table).is_none() {
6724 return Err(format!("unknown table {table:?}"));
6725 }
6726 if let Some(existing) = self.subs.iter().find(|subscription| subscription.id == id) {
6727 let same_intent = existing.table == table
6728 && canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
6729 && existing.params == params;
6730 if same_intent {
6731 return Ok(());
6732 }
6733 return Err(
6734 "client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
6735 .to_owned(),
6736 );
6737 }
6738 let sub = Subscription {
6739 id: id.clone(),
6740 table,
6741 requested: scopes,
6742 params,
6743 cursor: -1,
6744 bootstrap_state: None,
6745 state: SubState::Active,
6746 reason_code: None,
6747 effective: None,
6748 synced_once: false,
6749 };
6750 self.persist_sub(&sub)?;
6751 self.subs.push(sub);
6752 Ok(())
6753 }
6754
6755 pub fn unsubscribe(&mut self, id: &str) {
6756 self.subs.retain(|s| s.id != id);
6757 let _ = self.conn.execute(
6758 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
6759 rusqlite::params![id],
6760 );
6761 }
6762
6763 pub fn set_window(
6771 &mut self,
6772 base: &WindowBase,
6773 units: &[String],
6774 ) -> Result<CommandEffects, String> {
6775 let table = self
6776 .schema
6777 .table(&base.table)
6778 .ok_or_else(|| format!("unknown table {:?}", base.table))?;
6779 if table.scope_column(&base.variable).is_none() {
6780 return Err(format!(
6781 "setWindow: table {:?} has no scope variable {:?} (§4.8)",
6782 base.table, base.variable
6783 ));
6784 }
6785 let base_key = window_base_key(base);
6786 let wanted: std::collections::HashSet<&String> = units.iter().collect();
6787 let live = self.load_window_units(&base_key);
6788 self.begin_observation("syncular_window")?;
6789 let mut batch = ChangeAccumulator::default();
6790 let mut changed = false;
6791
6792 let prior_subs = self.subs.clone();
6793 let widened = (|| {
6794 for unit in units {
6796 if live.iter().any(|(u, _)| u == unit) {
6797 continue;
6798 }
6799 let sub_id = derive_sub_id(base, unit);
6800 self.delete_pending_evict(&sub_id)?;
6801 self.insert_window_unit(&base_key, unit, &sub_id)?;
6802 self.subscribe(
6803 sub_id,
6804 base.table.clone(),
6805 unit_scopes(base, unit),
6806 base.params.clone(),
6807 )?;
6808 batch.window(&base_key, &base.table, unit);
6809 changed = true;
6810 }
6811
6812 self.finish_observation("syncular_window", batch)
6813 })();
6814 if let Err(error) = widened {
6815 self.rollback_observation("syncular_window");
6816 self.subs = prior_subs;
6817 return Err(error);
6818 }
6819 for (unit, sub_id) in live {
6820 if wanted.contains(&unit) {
6821 continue;
6822 }
6823 self.rebuild_overlay_if_dirty();
6824 let prior_subs = self.subs.clone();
6825 self.begin_observation("syncular_window")?;
6826 let effective = self
6827 .subs
6828 .iter()
6829 .find(|sub| sub.id == sub_id)
6830 .and_then(|sub| sub.effective.clone())
6831 .unwrap_or_else(|| unit_scopes(base, &unit));
6832 let mut batch = ChangeAccumulator::default();
6833 self.record_scope_map(&mut batch, &base.table, &effective);
6834 batch.window(&base_key, &base.table, &unit);
6835 let result = self.evict_unit(&base_key, base, &unit, &sub_id);
6836 let remaining = match result {
6837 Ok(remaining) => remaining,
6838 Err(error) => {
6839 self.rollback_observation("syncular_window");
6840 self.subs = prior_subs;
6841 return Err(error);
6842 }
6843 };
6844 if let Err(error) = self.finish_observation("syncular_window", batch) {
6845 self.rollback_observation("syncular_window");
6846 self.subs = prior_subs;
6847 return Err(error);
6848 }
6849 if remaining {
6850 self.drain_pending_evictions()?;
6851 }
6852 changed = true;
6853 }
6854 Ok(if changed {
6855 CommandEffects::interactive()
6856 } else {
6857 CommandEffects::none()
6858 })
6859 }
6860
6861 pub fn window_state(&self, base: &WindowBase) -> WindowState {
6866 let mut units = Vec::new();
6867 let mut pending = Vec::new();
6868 for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
6869 let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
6870 Some(sub) => {
6871 sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
6872 }
6873 None => true,
6874 };
6875 if is_pending {
6876 pending.push(unit.clone());
6877 }
6878 units.push(unit);
6879 }
6880 WindowState { units, pending }
6881 }
6882
6883 fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
6884 let mut stmt = match self
6885 .conn
6886 .prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
6887 {
6888 Ok(stmt) => stmt,
6889 Err(_) => return Vec::new(),
6890 };
6891 let rows = stmt.query_map(rusqlite::params![base_key], |row| {
6892 Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
6893 });
6894 match rows {
6895 Ok(rows) => rows.filter_map(Result::ok).collect(),
6896 Err(_) => Vec::new(),
6897 }
6898 }
6899
6900 fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
6901 let mut stmt = match self.conn.prepare(
6902 "SELECT windows.base, windows.unit, subscriptions.tbl
6903 FROM _syncular_windows AS windows
6904 JOIN _syncular_subscriptions AS subscriptions
6905 ON subscriptions.id = windows.sub_id
6906 ORDER BY windows.base, windows.unit",
6907 ) {
6908 Ok(stmt) => stmt,
6909 Err(_) => return Vec::new(),
6910 };
6911 let rows = stmt.query_map([], |row| {
6912 Ok((
6913 row.get::<_, String>(0)?,
6914 row.get::<_, String>(1)?,
6915 row.get::<_, String>(2)?,
6916 ))
6917 });
6918 match rows {
6919 Ok(rows) => rows.filter_map(Result::ok).collect(),
6920 Err(_) => Vec::new(),
6921 }
6922 }
6923
6924 fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
6925 self.conn
6926 .query_row(
6927 "SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
6928 rusqlite::params![sub_id],
6929 |row| Ok((row.get(0)?, row.get(1)?)),
6930 )
6931 .ok()
6932 }
6933
6934 fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
6935 self.conn
6936 .execute(
6937 "INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
6938 rusqlite::params![base_key, unit, sub_id],
6939 )
6940 .map(|_| ())
6941 .map_err(|error| error.to_string())
6942 }
6943
6944 fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
6945 self.conn
6946 .execute(
6947 "DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
6948 rusqlite::params![base_key, unit],
6949 )
6950 .map(|_| ())
6951 .map_err(|error| error.to_string())
6952 }
6953
6954 fn evict_unit(
6961 &mut self,
6962 base_key: &str,
6963 base: &WindowBase,
6964 unit: &str,
6965 sub_id: &str,
6966 ) -> Result<bool, String> {
6967 let effective = self
6968 .subs
6969 .iter()
6970 .find(|s| s.id == sub_id)
6971 .and_then(|s| s.effective.clone())
6972 .unwrap_or_else(|| unit_scopes(base, unit));
6973 let pinned = self.pinned_row_ids(&base.table);
6974 let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
6975 self.delete_window_unit(base_key, unit)?;
6976 self.conn
6977 .execute(
6978 "DELETE FROM _syncular_subscriptions WHERE id = ?1",
6979 [sub_id],
6980 )
6981 .map_err(|error| error.to_string())?;
6982 self.subs.retain(|sub| sub.id != sub_id);
6983 if deferred {
6984 self.save_pending_evict(sub_id, &base.table, &effective)?;
6985 } else {
6986 self.delete_pending_evict(sub_id)?;
6987 }
6988 Ok(remaining)
6989 }
6990
6991 fn evict_scope_rows(
6995 &self,
6996 table_name: &str,
6997 effective: &[(String, Vec<String>)],
6998 pinned: &std::collections::HashSet<String>,
6999 ) -> Result<(bool, bool), String> {
7000 if effective.is_empty() {
7001 return Ok((false, false));
7002 }
7003 let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
7004 let mut clauses = Vec::new();
7005 let mut params: Vec<SqlValue> = Vec::new();
7006 for (variable, values) in effective {
7007 let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
7008 if values.is_empty() {
7009 return Ok((false, false));
7010 }
7011 let holes = values
7012 .iter()
7013 .map(|v| {
7014 params.push(SqlValue::Text(v.clone()));
7015 "?"
7016 })
7017 .collect::<Vec<_>>()
7018 .join(", ");
7019 clauses.push(format!("{} IN ({holes})", quote_ident(column)));
7020 }
7021 let scope = clauses.join(" AND ");
7022 let scope_params = params.len();
7023 let pk = quote_ident(&table.primary_key);
7024 if !pinned.is_empty() {
7025 let holes = pinned
7026 .iter()
7027 .map(|id| {
7028 params.push(SqlValue::Text(id.clone()));
7029 "?"
7030 })
7031 .collect::<Vec<_>>()
7032 .join(", ");
7033 clauses.push(format!("{pk} NOT IN ({holes})"));
7034 }
7035 let predicate = clauses.join(" AND ");
7036 let base = base_table(table_name);
7037 let visible = visible_table(table_name);
7038 let mut ids: Vec<SqlValue> = Vec::new();
7039 for source in [&base, &visible] {
7040 let mut stmt = self
7041 .conn
7042 .prepare_cached(&format!(
7043 "SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
7044 ))
7045 .map_err(|e| e.to_string())?;
7046 ids = stmt
7047 .query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
7048 .map_err(|e| e.to_string())?
7049 .collect::<Result<_, _>>()
7050 .map_err(|e| e.to_string())?;
7051 if !ids.is_empty() {
7052 break;
7053 }
7054 }
7055 if !ids.is_empty() {
7056 let holes = vec!["?"; ids.len()].join(", ");
7057 for source in [&base, &visible] {
7058 self.conn
7059 .execute(
7060 &format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
7061 rusqlite::params_from_iter(&ids),
7062 )
7063 .map_err(|e| e.to_string())?;
7064 }
7065 }
7066 let mut remaining = false;
7067 let mut deferred = false;
7068 for source in [&base, &visible] {
7069 remaining |= self
7070 .conn
7071 .query_row(
7072 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
7073 rusqlite::params_from_iter(¶ms),
7074 |row| row.get::<_, bool>(0),
7075 )
7076 .map_err(|e| e.to_string())?;
7077 deferred |= self
7078 .conn
7079 .query_row(
7080 &format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
7081 rusqlite::params_from_iter(¶ms[..scope_params]),
7082 |row| row.get::<_, bool>(0),
7083 )
7084 .map_err(|e| e.to_string())?;
7085 }
7086 Ok((remaining, deferred))
7087 }
7088
7089 fn drain_pending_evictions(&mut self) -> Result<(), String> {
7091 let pending = self.load_pending_evictions()?;
7092 if pending.is_empty() {
7093 return Ok(());
7094 }
7095 self.rebuild_overlay_if_dirty();
7096 for (sub_id, table_name, effective) in pending {
7097 if self.schema.table(&table_name).is_none() {
7098 self.delete_pending_evict(&sub_id)?;
7099 continue;
7100 }
7101 loop {
7102 std::thread::yield_now();
7103 self.begin_observation("syncular_evict")?;
7104 let pinned = self.pinned_row_ids(&table_name);
7105 let result = self.evict_scope_rows(&table_name, &effective, &pinned);
7106 let (remaining, deferred) = match result {
7107 Ok(result) => result,
7108 Err(error) => {
7109 self.rollback_observation("syncular_evict");
7110 return Err(error);
7111 }
7112 };
7113 if !deferred {
7114 if let Err(error) = self.delete_pending_evict(&sub_id) {
7115 self.rollback_observation("syncular_evict");
7116 return Err(error);
7117 }
7118 }
7119 let mut batch = ChangeAccumulator::default();
7120 self.record_scope_map(&mut batch, &table_name, &effective);
7121 if let Err(error) = self.finish_observation("syncular_evict", batch) {
7122 self.rollback_observation("syncular_evict");
7123 return Err(error);
7124 }
7125 if !remaining {
7126 break;
7127 }
7128 }
7129 }
7130 Ok(())
7131 }
7132
7133 fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
7136 let mut pinned = std::collections::HashSet::new();
7137 for commit in &self.outbox {
7138 for op in &commit.ops {
7139 if op.table == table {
7140 pinned.insert(op.row_id.clone());
7141 }
7142 }
7143 }
7144 pinned
7145 }
7146
7147 fn save_pending_evict(
7148 &self,
7149 sub_id: &str,
7150 table: &str,
7151 effective: &[(String, Vec<String>)],
7152 ) -> Result<(), String> {
7153 self.conn.execute(
7154 "INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
7155 rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
7156 ).map(|_| ()).map_err(|error| error.to_string())
7157 }
7158
7159 fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
7160 self.conn
7161 .execute(
7162 "DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
7163 rusqlite::params![sub_id],
7164 )
7165 .map(|_| ())
7166 .map_err(|error| error.to_string())
7167 }
7168
7169 fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
7170 let mut stmt = self
7171 .conn
7172 .prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
7173 .map_err(|error| error.to_string())?;
7174 let rows = stmt
7175 .query_map([], |row| {
7176 Ok((
7177 row.get::<_, String>(0)?,
7178 row.get::<_, String>(1)?,
7179 row.get::<_, String>(2)?,
7180 ))
7181 })
7182 .map_err(|error| error.to_string())?;
7183 rows.map(|row| {
7184 let (sub_id, table, json) = row.map_err(|error| error.to_string())?;
7185 let value: Value = serde_json::from_str(&json)
7186 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
7187 let effective = json_to_scope_map(&value)
7188 .map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
7189 Ok((sub_id, table, effective))
7190 })
7191 .collect()
7192 }
7193
7194 pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
7196 if mutations.is_empty() {
7197 return Err("a commit must contain at least one operation (§6.1)".to_owned());
7198 }
7199 let mut ops = Vec::with_capacity(mutations.len());
7200 for mutation in mutations {
7201 match mutation {
7202 Mutation::Upsert {
7203 table,
7204 values,
7205 base_version,
7206 } => {
7207 let schema_table = self
7208 .schema
7209 .table(&table)
7210 .ok_or_else(|| format!("unknown table {table:?}"))?;
7211 let values = normalize_values_casing(schema_table, values)?;
7215 let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
7216 let values = full_row_values(schema_table, values)?;
7219 self.validate_author_encode(schema_table, &row_id, &values)?;
7225 ops.push(OutboxOp {
7226 upsert: true,
7227 table,
7228 row_id,
7229 base_version,
7230 values: Some(values),
7231 });
7232 }
7233 Mutation::Delete {
7234 table,
7235 row_id,
7236 base_version,
7237 } => {
7238 if self.schema.table(&table).is_none() {
7239 return Err(format!("unknown table {table:?}"));
7240 }
7241 ops.push(OutboxOp {
7242 upsert: false,
7243 table,
7244 row_id,
7245 base_version,
7246 values: None,
7247 });
7248 }
7249 }
7250 }
7251 self.record_outbox_commit(ops)
7252 }
7253
7254 fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
7255 let commit = OutboxCommit {
7256 client_commit_id: uuid::Uuid::new_v4().to_string(),
7257 ops,
7258 };
7259 self.begin_observation("syncular_mutation")?;
7260 let mut batch = ChangeAccumulator::default();
7261 for op in &commit.ops {
7262 let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
7263 if let Some(values) = &op.values {
7264 if let Some(table) = self.schema.table(&op.table) {
7265 for scope in &table.scope_variables {
7266 if let Some(Value::String(value)) = values.get(&scope.column) {
7267 batch.scope(&op.table, format!("{}:{value}", scope.prefix));
7268 precise = true;
7269 }
7270 }
7271 }
7272 }
7273 if !precise {
7274 batch.table(&op.table);
7275 }
7276 }
7277 if let Err(error) = self.persist_outbox_insert(&commit) {
7278 self.rollback_observation("syncular_mutation");
7279 return Err(error);
7280 }
7281 let was_dirty = self.overlay_dirty.get();
7282 if !was_dirty {
7285 self.apply_outbox_ops(&commit.ops);
7286 }
7287 let id = commit.client_commit_id.clone();
7288 self.outbox.push(commit);
7289 if was_dirty {
7290 self.rebuild_overlay();
7291 }
7292 batch.status = true;
7293 if let Err(error) = self.finish_observation("syncular_mutation", batch) {
7294 self.rollback_observation("syncular_mutation");
7295 self.outbox.pop();
7296 self.overlay_dirty.set(was_dirty);
7297 return Err(error);
7298 }
7299 Ok(id)
7300 }
7301
7302 pub fn patch(
7308 &mut self,
7309 table: &str,
7310 row_id: &str,
7311 partial: Map<String, Value>,
7312 base_version: Option<i64>,
7313 ) -> Result<String, String> {
7314 let schema_table = self
7315 .schema
7316 .table(table)
7317 .ok_or_else(|| format!("unknown table {table:?}"))?;
7318 let partial = normalize_values_casing(schema_table, partial)?;
7319 if let Some(pk) = partial.get(&schema_table.primary_key) {
7320 if pk.as_str() != Some(row_id) {
7321 return Err(format!(
7322 "sync.invalid_request: table {table:?}: patch cannot change the primary key"
7323 ));
7324 }
7325 }
7326 for scope in &schema_table.scope_variables {
7329 if partial.contains_key(&scope.column) {
7330 return Err(format!(
7331 "sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
7332 scope.column
7333 ));
7334 }
7335 }
7336 let mut values = partial;
7337 values.insert(
7338 schema_table.primary_key.clone(),
7339 Value::from(row_id.to_owned()),
7340 );
7341 self.validate_author_encode(schema_table, row_id, &values)?;
7347 self.record_outbox_commit(vec![OutboxOp {
7348 upsert: true,
7349 table: table.to_owned(),
7350 row_id: row_id.to_owned(),
7351 base_version,
7352 values: Some(values),
7353 }])
7354 }
7355
7356 fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
7361 let selector = self.encryption.key_id_columns.get(&table.name)?;
7362 let index = table
7363 .columns
7364 .iter()
7365 .position(|column| &column.name == selector)?;
7366 let sql = format!(
7367 "SELECT {} FROM {} WHERE {}",
7368 quote_ident(selector),
7369 visible_table(&table.name),
7370 row_id_predicate(table)
7371 );
7372 let key: Option<String> = self
7373 .conn
7374 .query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
7375 .ok()?;
7376 let key = key.filter(|key| !key.is_empty())?;
7377 let mut fallback: Row = vec![None; table.columns.len()];
7378 fallback[index] = Some(ColumnValue::String(key));
7379 Some(fallback)
7380 }
7381
7382 fn validate_author_encode(
7387 &self,
7388 table: &TableSchema,
7389 row_id: &str,
7390 values: &Map<String, Value>,
7391 ) -> Result<(), String> {
7392 match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
7393 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
7394 result => result.map(|_| ()),
7395 }
7396 }
7397
7398 fn drop_unencodable_outbox(&mut self) -> Result<bool, String> {
7404 if !self
7405 .schema
7406 .tables
7407 .iter()
7408 .any(|table| table.has_encrypted_columns())
7409 {
7410 return Ok(false);
7411 }
7412 let failures = self
7413 .outbox
7414 .iter()
7415 .filter_map(|commit| {
7416 commit
7417 .ops
7418 .iter()
7419 .find_map(|op| {
7420 let values = op.values.as_ref().filter(|_| op.upsert)?;
7421 let table = self.schema.table(&op.table)?;
7422 let fallback = self.stored_key_fallback(table, &op.row_id);
7423 match encode_sparse_row_json(
7424 table,
7425 &op.row_id,
7426 values,
7427 &self.encryption,
7428 fallback.as_ref(),
7429 ) {
7430 Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Some(error),
7431 _ => None,
7432 }
7433 })
7434 .map(|error| (commit.clone(), error))
7435 })
7436 .collect::<Vec<_>>();
7437 if failures.is_empty() {
7438 return Ok(false);
7439 }
7440 for (commit, message) in failures {
7441 let rejection = RejectionRecord {
7442 client_commit_id: commit.client_commit_id.clone(),
7443 op_index: 0,
7444 code: ENCRYPT_FAILED_CODE.to_owned(),
7445 message,
7446 retryable: false,
7447 details: None,
7448 operation: commit.ops.first().map(CommitOperation::from),
7449 };
7450 self.persist_commit_outcome(
7451 &commit.client_commit_id,
7452 CommitOutcomeStatus::Rejected,
7453 &[CommitOperationOutcome::Error {
7454 rejection: rejection.clone(),
7455 }],
7456 Some(&commit.ops),
7457 )?;
7458 self.delete_outbox_persisted(&commit.client_commit_id)?;
7459 self.outbox
7460 .retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
7461 self.rejections.push(rejection);
7462 }
7463 self.prune_commit_outcomes()?;
7464 self.overlay_dirty.set(true);
7465 self.rebuild_overlay();
7466 Ok(true)
7467 }
7468
7469 pub fn pending_commit_ids(&self) -> Vec<String> {
7470 self.outbox
7471 .iter()
7472 .map(|c| c.client_commit_id.clone())
7473 .collect()
7474 }
7475
7476 pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
7480 let mut payloads = Vec::new();
7481 for commit in &self.outbox {
7482 for op in &commit.ops {
7483 let Some(values) = op.values.as_ref().filter(|_| op.upsert) else {
7484 continue;
7485 };
7486 let Some(table) = self.schema.table(&op.table) else {
7487 continue;
7488 };
7489 let fallback = self.stored_key_fallback(table, &op.row_id);
7490 if let Ok(payload) = encode_sparse_row_json(
7491 table,
7492 &op.row_id,
7493 values,
7494 &self.encryption,
7495 fallback.as_ref(),
7496 ) {
7497 payloads.push(payload);
7498 }
7499 }
7500 }
7501 payloads
7502 }
7503
7504 pub fn conflicts(&self) -> &[ConflictRecord] {
7505 &self.conflicts
7506 }
7507
7508 pub fn rejections(&self) -> &[RejectionRecord] {
7509 &self.rejections
7510 }
7511
7512 pub fn resolve_commit_outcome(
7513 &mut self,
7514 input: ResolveCommitOutcomeInput,
7515 ) -> Result<CommitOutcome, String> {
7516 let current = self
7517 .commit_outcome(&input.client_commit_id)?
7518 .ok_or_else(|| {
7519 format!(
7520 "sync.outcome_not_found: no durable outcome exists for {:?}",
7521 input.client_commit_id
7522 )
7523 })?;
7524 if current.resolution != CommitOutcomeResolution::Active {
7525 return Ok(current);
7526 }
7527 if input.resolution == CommitOutcomeResolution::Active {
7528 return Err("sync.invalid_request: resolution must leave active state".to_owned());
7529 }
7530 match input.resolution {
7531 CommitOutcomeResolution::Superseded => {
7532 let replacement = input
7533 .replacement_client_commit_id
7534 .as_deref()
7535 .filter(|value| !value.is_empty() && *value != input.client_commit_id)
7536 .ok_or_else(|| {
7537 "sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
7538 .to_owned()
7539 })?;
7540 let _ = replacement;
7541 }
7542 _ if input.replacement_client_commit_id.is_some() => {
7543 return Err(
7544 "sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
7545 .to_owned(),
7546 );
7547 }
7548 _ => {}
7549 }
7550 let allowed = match current.status {
7551 CommitOutcomeStatus::Conflict => matches!(
7552 input.resolution,
7553 CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
7554 ),
7555 CommitOutcomeStatus::Rejected => {
7556 input.resolution == CommitOutcomeResolution::Superseded
7557 }
7558 CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
7559 input.resolution == CommitOutcomeResolution::Dismissed
7560 }
7561 };
7562 if !allowed {
7563 return Err(format!(
7564 "sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
7565 input.resolution, current.status
7566 ));
7567 }
7568
7569 self.begin_observation("syncular_outcome_resolution")?;
7570 let result = (|| {
7571 self.conn
7572 .execute(
7573 "UPDATE _syncular_commit_outcomes
7574 SET resolution = ?1, resolved_at_ms = ?2,
7575 replacement_client_commit_id = ?3
7576 WHERE client_commit_id = ?4 AND resolution = 'active'",
7577 rusqlite::params![
7578 Self::outcome_resolution_name(input.resolution),
7579 self.clock_now_ms(),
7580 input.replacement_client_commit_id,
7581 input.client_commit_id
7582 ],
7583 )
7584 .map_err(|error| error.to_string())?;
7585 let resolved = self
7586 .commit_outcome(¤t.client_commit_id)?
7587 .ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
7588 self.prune_commit_outcomes()?;
7589 Ok(resolved)
7590 })();
7591 let resolved = match result {
7592 Ok(outcome) => outcome,
7593 Err(error) => {
7594 self.rollback_observation("syncular_outcome_resolution");
7595 return Err(error);
7596 }
7597 };
7598 self.conflicts
7599 .retain(|record| record.client_commit_id != current.client_commit_id);
7600 self.rejections
7601 .retain(|record| record.client_commit_id != current.client_commit_id);
7602 let batch = ChangeAccumulator {
7603 conflicts: current.status == CommitOutcomeStatus::Conflict,
7604 rejections: current.status == CommitOutcomeStatus::Rejected,
7605 outcomes: true,
7606 ..ChangeAccumulator::default()
7607 };
7608 if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
7609 self.rollback_observation("syncular_outcome_resolution");
7610 return Err(error);
7611 }
7612 Ok(resolved)
7613 }
7614
7615 pub fn schema_floor(&self) -> Option<&SchemaFloor> {
7616 self.schema_floor.as_ref()
7617 }
7618
7619 pub fn lease_state(&self) -> Option<&LeaseState> {
7621 self.lease_state.as_ref()
7622 }
7623
7624 fn record_lease_error(&mut self, code: &str) {
7627 if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
7628 return;
7629 }
7630 let mut next = self.lease_state.clone().unwrap_or_default();
7631 next.error_code = Some(code.to_owned());
7632 self.set_lease_state(Some(next));
7633 }
7634
7635 fn set_lease_state(&mut self, next: Option<LeaseState>) {
7636 if self.lease_state == next {
7637 return;
7638 }
7639 if self.begin_observation("syncular_lease").is_err() {
7640 return;
7641 }
7642 self.lease_state = next;
7643 if let Some(lease) = &self.lease_state {
7644 if let Ok(json) = serde_json::to_string(lease) {
7645 self.set_meta(LEASE_STATE_KEY, &json);
7646 }
7647 } else {
7648 self.delete_meta(LEASE_STATE_KEY);
7649 }
7650 let batch = ChangeAccumulator {
7651 status: true,
7652 ..ChangeAccumulator::default()
7653 };
7654 if self.finish_observation("syncular_lease", batch).is_err() {
7655 self.rollback_observation("syncular_lease");
7656 }
7657 }
7658
7659 fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
7660 if self.schema_floor == next {
7661 return;
7662 }
7663 if self.begin_observation("syncular_schema_floor").is_err() {
7664 return;
7665 }
7666 self.schema_floor = next;
7667 self.stopped = self.schema_floor.is_some();
7668 if let Some(floor) = &self.schema_floor {
7669 if let Ok(json) = serde_json::to_string(floor) {
7670 self.set_meta(SCHEMA_FLOOR_KEY, &json);
7671 }
7672 } else {
7673 self.delete_meta(SCHEMA_FLOOR_KEY);
7674 }
7675 let batch = ChangeAccumulator {
7676 status: true,
7677 ..ChangeAccumulator::default()
7678 };
7679 if self
7680 .finish_observation("syncular_schema_floor", batch)
7681 .is_err()
7682 {
7683 self.rollback_observation("syncular_schema_floor");
7684 }
7685 }
7686
7687 fn set_upgrading(&mut self, value: bool) {
7688 if self.upgrading == value {
7689 return;
7690 }
7691 if self.begin_observation("syncular_upgrading").is_err() {
7692 return;
7693 }
7694 self.upgrading = value;
7695 let batch = ChangeAccumulator {
7696 status: true,
7697 ..ChangeAccumulator::default()
7698 };
7699 if self
7700 .finish_observation("syncular_upgrading", batch)
7701 .is_err()
7702 {
7703 self.rollback_observation("syncular_upgrading");
7704 }
7705 }
7706
7707 pub fn sync_needed(&self) -> bool {
7708 self.sync_needed
7709 }
7710
7711 pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
7712 let sub = self.subs.iter().find(|s| s.id == id)?;
7713 Some(SubscriptionStateView {
7714 id: sub.id.clone(),
7715 table: sub.table.clone(),
7716 status: sub.state.name().to_owned(),
7717 cursor: sub.cursor,
7718 has_resume_token: sub.bootstrap_state.is_some(),
7719 effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
7720 reason_code: sub.reason_code.clone(),
7721 })
7722 }
7723
7724 pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
7725 let schema_table = self
7726 .schema
7727 .table(table)
7728 .ok_or_else(|| format!("unknown table {table:?}"))?;
7729 let sql = format!(
7730 "SELECT * FROM {} ORDER BY {} ASC",
7731 visible_table(table),
7732 quote_ident(&schema_table.primary_key)
7733 );
7734 let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
7735 let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
7736 let mut out = Vec::new();
7737 while let Some(row) = rows.next().map_err(|e| e.to_string())? {
7738 let mut values = Map::new();
7739 for (i, column) in schema_table.columns.iter().enumerate() {
7740 let value = row.get_ref(i).map_err(|e| e.to_string())?;
7741 values.insert(column.name.clone(), sql_ref_to_json(column, value));
7742 }
7743 let version: i64 = row
7744 .get(schema_table.columns.len())
7745 .map_err(|e| e.to_string())?;
7746 let row_id = match values.get(&schema_table.primary_key) {
7747 Some(Value::String(s)) => s.clone(),
7748 Some(Value::Number(n)) => n.to_string(),
7749 Some(Value::Bool(b)) => b.to_string(),
7750 other => format!("{}", other.cloned().unwrap_or(Value::Null)),
7751 };
7752 out.push(RowState {
7753 row_id,
7754 version,
7755 values,
7756 });
7757 }
7758 Ok(out)
7759 }
7760
7761 #[cfg(feature = "crdt-yjs")]
7777 fn crdt_column_bytes(
7778 &self,
7779 table: &str,
7780 row_id: &str,
7781 column: &str,
7782 ) -> Result<Option<Vec<u8>>, String> {
7783 let schema_table = self
7784 .schema
7785 .table(table)
7786 .ok_or_else(|| format!("unknown table {table:?}"))?;
7787 let col = schema_table
7788 .columns
7789 .iter()
7790 .find(|c| c.name == column)
7791 .ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
7792 if col.ty != ColumnType::Crdt {
7793 return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
7794 }
7795 let sql = format!(
7796 "SELECT {} FROM {} WHERE {}",
7797 quote_ident(column),
7798 visible_table(table),
7799 row_id_predicate(schema_table)
7800 );
7801 let bytes: Option<Vec<u8>> = self
7802 .conn
7803 .prepare_cached(&sql)
7804 .map_err(|error| error.to_string())?
7805 .query_row(rusqlite::params![row_id], |row| {
7806 row.get::<_, Option<Vec<u8>>>(0)
7807 })
7808 .map_err(|e| match e {
7809 rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
7810 other => other.to_string(),
7811 })?;
7812 Ok(bytes)
7813 }
7814
7815 #[cfg(feature = "crdt-yjs")]
7819 pub fn crdt_text(
7820 &self,
7821 table: &str,
7822 row_id: &str,
7823 column: &str,
7824 name: &str,
7825 ) -> Result<String, String> {
7826 let bytes = self
7827 .crdt_column_bytes(table, row_id, column)?
7828 .unwrap_or_default();
7829 crate::crdt::text(&bytes, name)
7830 }
7831
7832 #[cfg(feature = "crdt-yjs")]
7836 pub fn crdt_insert_text(
7837 &mut self,
7838 table: &str,
7839 row_id: &str,
7840 column: &str,
7841 name: &str,
7842 index: u32,
7843 value: &str,
7844 ) -> Result<String, String> {
7845 let current = self
7846 .crdt_column_bytes(table, row_id, column)?
7847 .unwrap_or_default();
7848 let update = crate::crdt::insert_text(¤t, name, index, value)?;
7849 self.crdt_push_update(table, row_id, column, &update)
7850 }
7851
7852 #[cfg(feature = "crdt-yjs")]
7856 pub fn crdt_delete_text(
7857 &mut self,
7858 table: &str,
7859 row_id: &str,
7860 column: &str,
7861 name: &str,
7862 index: u32,
7863 len: u32,
7864 ) -> Result<String, String> {
7865 let current = self
7866 .crdt_column_bytes(table, row_id, column)?
7867 .unwrap_or_default();
7868 let update = crate::crdt::delete_text(¤t, name, index, len)?;
7869 self.crdt_push_update(table, row_id, column, &update)
7870 }
7871
7872 #[cfg(feature = "crdt-yjs")]
7877 pub fn crdt_apply_update(
7878 &mut self,
7879 table: &str,
7880 row_id: &str,
7881 column: &str,
7882 update: &[u8],
7883 ) -> Result<String, String> {
7884 let current = self
7885 .crdt_column_bytes(table, row_id, column)?
7886 .unwrap_or_default();
7887 let next = crate::crdt::apply_update(¤t, update)?;
7888 self.crdt_push_update(table, row_id, column, &next)
7889 }
7890
7891 #[cfg(feature = "crdt-yjs")]
7898 fn crdt_push_update(
7899 &mut self,
7900 table: &str,
7901 row_id: &str,
7902 column: &str,
7903 crdt_bytes: &[u8],
7904 ) -> Result<String, String> {
7905 let mut bytes_obj = Map::new();
7906 bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
7907 let mut partial = Map::new();
7908 partial.insert(column.to_owned(), Value::Object(bytes_obj));
7909 self.patch(table, row_id, partial, None)
7910 }
7911
7912 pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
7927 query_connection(&self.conn, sql, params)
7928 }
7929
7930 #[cfg(feature = "bench-internals")]
7932 #[doc(hidden)]
7933 pub fn benchmark_connection(&mut self) -> &mut Connection {
7934 &mut self.conn
7935 }
7936
7937 #[cfg(feature = "bench-internals")]
7939 #[doc(hidden)]
7940 pub fn benchmark_phases(
7941 &self,
7942 enabled: Option<bool>,
7943 reset: bool,
7944 ) -> Result<Option<Value>, String> {
7945 self.benchmark_phases.configure(enabled, reset)?;
7946 Ok(self.benchmark_phases.snapshot())
7947 }
7948
7949 pub fn query_snapshot(
7951 &mut self,
7952 sql: &str,
7953 params: &[QueryValue],
7954 coverage: &[WindowCoverage],
7955 ) -> Result<QuerySnapshot, String> {
7956 snapshot_connection(&self.conn, sql, params, coverage)
7957 }
7958
7959 fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
7962 #[cfg(feature = "bench-internals")]
7963 let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
7964 let log_epoch = self.get_meta(LOG_EPOCH_KEY);
7965 let mut frames = vec![Frame::ReqHeader {
7966 client_id: self.client_id.clone(),
7967 schema_version: self.schema.version,
7968 log_epoch: log_epoch.clone(),
7969 }];
7970 let mut pushed_ids = Vec::new();
7971 let mut ops_in_request = 0usize;
7972 let mut deferred_commits = 0usize;
7973 #[cfg(feature = "bench-internals")]
7974 let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
7975 for (index, commit) in self
7976 .outbox
7977 .iter()
7978 .take(if log_epoch.is_some() {
7979 self.outbox.len()
7980 } else {
7981 0
7982 })
7983 .enumerate()
7984 {
7985 if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
7990 deferred_commits = self.outbox.len() - index;
7991 break;
7992 }
7993 ops_in_request += commit.ops.len();
7994 let operations = commit
7995 .ops
7996 .iter()
7997 .map(|op| {
7998 let payload = op.values.as_ref().and_then(|values| {
7999 let table = self.schema.table(&op.table)?;
8000 let fallback = self.stored_key_fallback(table, &op.row_id);
8009 encode_sparse_row_json(
8010 table,
8011 &op.row_id,
8012 values,
8013 &self.encryption,
8014 fallback.as_ref(),
8015 )
8016 .ok()
8017 });
8018 ssp2::model::Operation {
8019 table: op.table.clone(),
8020 row_id: op.row_id.clone(),
8021 op: if op.upsert { Op::Upsert } else { Op::Delete },
8022 base_version: op.base_version,
8023 payload,
8024 }
8025 })
8026 .collect();
8027 frames.push(Frame::PushCommit {
8028 client_commit_id: commit.client_commit_id.clone(),
8029 operations,
8030 });
8031 pushed_ids.push(commit.client_commit_id.clone());
8032 }
8033 #[cfg(feature = "bench-internals")]
8034 drop(outbox_phase);
8035 let accept = self.limits.accept.unwrap_or(if url_capable {
8038 DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
8039 } else {
8040 DEFAULT_ACCEPT
8041 });
8042 frames.push(Frame::PullHeader {
8043 limit_commits: self.limits.limit_commits.unwrap_or(0),
8044 limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
8045 max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
8046 accept,
8047 });
8048 let mut fresh = Vec::new();
8049 for sub in &self.subs {
8050 if sub.state != SubState::Active {
8051 continue;
8052 }
8053 let mut scopes = sub.requested.clone();
8054 sort_scope_map(&mut scopes);
8055 frames.push(Frame::Subscription {
8056 id: sub.id.clone(),
8057 table: sub.table.clone(),
8058 scopes,
8059 params: sub.params.clone().map(RawJson),
8060 cursor: sub.cursor,
8061 bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
8062 });
8063 fresh.push((
8064 sub.id.clone(),
8065 sub.cursor < 0 && sub.bootstrap_state.is_none(),
8066 ));
8067 }
8068 let message = Message {
8069 wire_version: WIRE_VERSION,
8070 msg_kind: MsgKind::Request,
8071 frames,
8072 };
8073 (
8074 message,
8075 RequestMeta {
8076 pushed_ids,
8077 fresh,
8078 accept,
8079 deferred_commits,
8080 },
8081 )
8082 }
8083
8084 pub fn progress(&self) -> ProgressObserver {
8088 self.progress.clone()
8089 }
8090
8091 pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
8092 self.progress.start();
8093 let started_at_ms = self.clock_now_ms();
8094 let outcome = self.sync_inner(transport);
8095 self.progress.update(|p| match &outcome {
8096 SyncOutcome::Ok(report) if report.failed.is_empty() => {
8097 p.state = ProgressState::Complete
8098 }
8099 SyncOutcome::Ok(_) => {
8100 p.state = ProgressState::Failed;
8101 p.error_code = Some("sync.scope_revoked".into());
8102 }
8103 SyncOutcome::Failed { error_code, .. } => {
8104 p.state = ProgressState::Failed;
8105 p.error_code = Some(Self::diagnostic_code(error_code));
8106 }
8107 });
8108 let completed_at_ms = self.clock_now_ms();
8109 self.last_round = Some(match &outcome {
8110 SyncOutcome::Ok(report) => DiagnosticLastRound {
8111 status: "succeeded".to_owned(),
8112 started_at_ms,
8113 completed_at_ms,
8114 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
8115 counters: Some(DiagnosticRoundCounters {
8116 pushed: report.pushed,
8117 applied: report.applied.len(),
8118 rejected: report.rejected.len(),
8119 retryable: report.retryable.len(),
8120 conflicts: report.conflicts,
8121 commits_applied: report.commits_applied,
8122 segment_rows_applied: report.segment_rows_applied,
8123 bootstrapping: report.bootstrapping.len(),
8124 resets: report.resets.len(),
8125 revoked: report.revoked.len(),
8126 failed: report.failed.len(),
8127 deferred_commits: report.deferred_commits,
8128 }),
8129 error_code: None,
8130 },
8131 SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
8132 status: "failed".to_owned(),
8133 started_at_ms,
8134 completed_at_ms,
8135 duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
8136 counters: None,
8137 error_code: Some(Self::diagnostic_code(error_code)),
8138 },
8139 });
8140 outcome
8141 }
8142
8143 fn sync_inner(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
8144 if self.stopped {
8145 return SyncOutcome::Ok(SyncReport {
8148 schema_floor: self.schema_floor.clone(),
8149 ..SyncReport::default()
8150 });
8151 }
8152 self.set_sync_needed(false, false);
8155 if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
8158 if let Err(TransportError { code, message }) = self.flush_blob_uploads(transport) {
8159 if Self::retryable_transport_code(&code) {
8160 self.schedule_background_retry();
8161 }
8162 return SyncOutcome::Failed {
8163 error_code: code,
8164 message,
8165 };
8166 }
8167 }
8168 if let Err(message) = self.drain_pending_evictions() {
8169 return SyncOutcome::Failed {
8170 error_code: "storage.failed".into(),
8171 message,
8172 };
8173 }
8174 if let Err(message) = self.drop_unencodable_outbox() {
8178 return SyncOutcome::Failed {
8179 error_code: "storage.failed".into(),
8180 message,
8181 };
8182 }
8183 let (message, meta) = self.build_request(transport.supports_url_fetch());
8184 let request_bytes = {
8185 #[cfg(feature = "bench-internals")]
8186 let _phase = self.benchmark_phases.start(Phase::RequestEncode);
8187 encode_message(&message)
8188 };
8189 let round = if self.realtime_connected {
8193 transport.realtime_sync(&request_bytes)
8194 } else {
8195 transport.sync(&request_bytes)
8196 };
8197 let response_bytes = match round {
8198 Ok(bytes) => bytes,
8199 Err(TransportError { code, message }) => {
8200 self.record_lease_error(&code);
8203 if Self::retryable_transport_code(&code) {
8204 self.schedule_background_retry();
8205 }
8206 return SyncOutcome::Failed {
8207 error_code: code,
8208 message,
8209 };
8210 }
8211 };
8212 #[cfg(feature = "bench-internals")]
8213 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
8214 let response = match decode_message(&response_bytes) {
8215 Ok(message) => message,
8216 Err(error) => {
8217 return SyncOutcome::Failed {
8220 error_code: error.code.as_str().to_owned(),
8221 message: error.detail,
8222 };
8223 }
8224 };
8225 #[cfg(feature = "bench-internals")]
8226 drop(decode_phase);
8227 if response.msg_kind != MsgKind::Response {
8228 return SyncOutcome::Failed {
8229 error_code: "sync.invalid_request".to_owned(),
8230 message: "expected a response message".to_owned(),
8231 };
8232 }
8233 let mut outcome = self.process_response(transport, response, &meta);
8234 if let SyncOutcome::Ok(report) = &mut outcome {
8235 report.deferred_commits = meta.deferred_commits;
8236 }
8237 match &outcome {
8238 SyncOutcome::Ok(_) => self.reset_background_retry(),
8239 SyncOutcome::Failed { error_code, .. }
8240 if Self::retryable_transport_code(error_code) =>
8241 {
8242 self.schedule_background_retry();
8243 }
8244 SyncOutcome::Failed { .. } => {}
8245 }
8246 if meta.deferred_commits > 0 {
8247 self.set_sync_needed(true, true);
8250 }
8251 outcome
8252 }
8253
8254 pub fn sync_until_idle(
8255 &mut self,
8256 transport: &mut dyn Transport,
8257 max_rounds: Option<u32>,
8258 ) -> SyncOutcome {
8259 let rounds = max_rounds.unwrap_or(20).max(1);
8260 let mut aggregate = SyncReport::default();
8261 let mut spent = 0;
8262 while spent < rounds {
8263 spent += 1;
8264 let before = self
8265 .subs
8266 .iter()
8267 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
8268 .collect::<Vec<_>>();
8269 match self.sync(transport) {
8270 SyncOutcome::Failed {
8271 error_code,
8272 message,
8273 } => {
8274 return SyncOutcome::Failed {
8275 error_code,
8276 message,
8277 };
8278 }
8279 SyncOutcome::Ok(report) => {
8280 if max_rounds.is_none()
8281 && report.segment_rows_applied > 0
8282 && !report.bootstrapping.is_empty()
8283 && before
8284 != self
8285 .subs
8286 .iter()
8287 .map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
8288 .collect::<Vec<_>>()
8289 {
8290 spent = 0;
8291 }
8292 aggregate.pushed += report.pushed;
8293 aggregate.applied.extend(report.applied.iter().cloned());
8294 aggregate.rejected.extend(report.rejected.iter().cloned());
8295 aggregate.retryable.extend(report.retryable.iter().cloned());
8296 aggregate.conflicts += report.conflicts;
8297 aggregate.commits_applied += report.commits_applied;
8298 aggregate.segment_rows_applied += report.segment_rows_applied;
8299 aggregate.bootstrapping = report.bootstrapping.clone();
8300 aggregate.resets.extend(report.resets.iter().cloned());
8301 aggregate.revoked.extend(report.revoked.iter().cloned());
8302 aggregate.failed.extend(report.failed.iter().cloned());
8303 aggregate.deferred_commits = report.deferred_commits;
8304 if report.schema_floor.is_some() {
8305 aggregate.schema_floor = report.schema_floor.clone();
8306 }
8307 let more = !report.bootstrapping.is_empty()
8313 || report.commits_applied > 0
8314 || report.segment_rows_applied > 0
8315 || !report.resets.is_empty()
8316 || self.sync_needed;
8317 if !more {
8318 return SyncOutcome::Ok(aggregate);
8319 }
8320 }
8321 }
8322 }
8323 SyncOutcome::Failed {
8324 error_code: "sync.invalid_request".into(),
8325 message: "sync did not reach idle within the round budget".into(),
8326 }
8327 }
8328
8329 fn process_response(
8330 &mut self,
8331 transport: &mut dyn Transport,
8332 response: Message,
8333 meta: &RequestMeta,
8334 ) -> SyncOutcome {
8335 #[cfg(feature = "bench-internals")]
8336 let _phase = self.benchmark_phases.start(Phase::ResponseApply);
8337 let mut report = SyncReport {
8338 pushed: meta.pushed_ids.len() as u32,
8339 ..SyncReport::default()
8340 };
8341 let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
8342 HashMap::new();
8343 let pushed_ids = meta
8344 .pushed_ids
8345 .iter()
8346 .map(String::as_str)
8347 .collect::<HashSet<_>>();
8348 let mut last_final_push_result_id: Option<String> = None;
8349 for frame in &response.frames {
8350 match frame {
8351 Frame::PushResultDetails {
8352 client_commit_id,
8353 entries,
8354 } => {
8355 let details = rejection_details_by_commit
8356 .entry(client_commit_id.clone())
8357 .or_default();
8358 for entry in entries {
8359 let parsed = match RejectionDetails::parse(&entry.details.0) {
8360 Ok(value) => value,
8361 Err(message) => {
8362 return SyncOutcome::Failed {
8363 error_code: "sync.invalid_request".to_owned(),
8364 message,
8365 };
8366 }
8367 };
8368 details.insert(entry.op_index, parsed);
8369 }
8370 }
8371 Frame::PushResult {
8372 client_commit_id,
8373 status,
8374 results,
8375 ..
8376 } if pushed_ids.contains(client_commit_id.as_str())
8377 && Self::push_result_is_final(*status, results) =>
8378 {
8379 last_final_push_result_id = Some(client_commit_id.clone());
8380 }
8381 _ => {}
8382 }
8383 }
8384 let mut frames = response.frames.into_iter().peekable();
8385 if response.wire_version < 2 {
8386 return SyncOutcome::Failed {
8387 error_code: "client.invalid_host_response".to_owned(),
8388 message: "server response does not carry wire version 2 log-epoch state".to_owned(),
8389 };
8390 }
8391 match frames.next() {
8392 Some(Frame::RespHeader {
8393 required_schema_version,
8394 latest_schema_version,
8395 log_epoch,
8396 reset_required,
8397 }) => {
8398 let Some(log_epoch) = log_epoch else {
8399 return SyncOutcome::Failed {
8400 error_code: "client.invalid_host_response".to_owned(),
8401 message: "response header omits logEpoch".to_owned(),
8402 };
8403 };
8404 let Some(reset_required) = reset_required else {
8405 return SyncOutcome::Failed {
8406 error_code: "client.invalid_host_response".to_owned(),
8407 message: "response header omits resetRequired".to_owned(),
8408 };
8409 };
8410 if let Some(required) = required_schema_version {
8411 let floor = SchemaFloor {
8414 required_schema_version: Some(required),
8415 latest_schema_version,
8416 };
8417 self.set_schema_floor(Some(floor.clone()));
8418 report.schema_floor = Some(floor);
8419 return SyncOutcome::Ok(report);
8420 }
8421 if reset_required {
8422 if frames.next().is_some() {
8423 return SyncOutcome::Failed {
8424 error_code: "client.invalid_host_response".to_owned(),
8425 message: "log-epoch reset response contains body frames".to_owned(),
8426 };
8427 }
8428 match self.run_log_epoch_reset(&log_epoch) {
8429 Ok(resets) => {
8430 report.resets = resets;
8431 return SyncOutcome::Ok(report);
8432 }
8433 Err(message) => {
8434 return SyncOutcome::Failed {
8435 error_code: "sync.local_corrupt".to_owned(),
8436 message,
8437 };
8438 }
8439 }
8440 }
8441 if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
8442 return SyncOutcome::Failed {
8443 error_code: "client.invalid_host_response".to_owned(),
8444 message: "server changed logEpoch without requiring a reset".to_owned(),
8445 };
8446 }
8447 }
8448 _ => {
8449 return SyncOutcome::Failed {
8450 error_code: "sync.invalid_request".to_owned(),
8451 message: "response does not start with RESP_HEADER".to_owned(),
8452 };
8453 }
8454 }
8455
8456 let mut failure: Option<(String, String)> = None;
8457 while let Some(frame) = frames.next() {
8458 match frame {
8459 frame @ Frame::PushResult { .. } => {
8460 let successful = matches!(
8461 &frame,
8462 Frame::PushResult {
8463 status: PushStatus::Applied | PushStatus::Cached,
8464 ..
8465 }
8466 );
8467 let mut results = vec![frame];
8468 if successful {
8469 while matches!(
8470 frames.peek(),
8471 Some(Frame::PushResult {
8472 status: PushStatus::Applied | PushStatus::Cached,
8473 ..
8474 })
8475 ) {
8476 if let Some(next) = frames.next() {
8477 results.push(next);
8478 }
8479 }
8480 }
8481 if self
8482 .handle_push_results(
8483 &results,
8484 &pushed_ids,
8485 &rejection_details_by_commit,
8486 &mut report,
8487 last_final_push_result_id.as_deref(),
8488 )
8489 .is_err()
8490 {
8491 failure = Some((
8492 "client.outcome_persistence_failed".to_owned(),
8493 "local commit outcome could not be persisted".to_owned(),
8494 ));
8495 break;
8496 }
8497 }
8498 Frame::PushResultDetails { .. } => {}
8499 Frame::SubStart {
8500 id,
8501 status,
8502 reason_code,
8503 effective_scopes,
8504 bootstrap: _,
8505 } => {
8506 let mut body = Vec::new();
8507 let mut sub_end: Option<(i64, Option<String>)> = None;
8508 for inner in frames.by_ref() {
8509 match inner {
8510 Frame::SubEnd {
8511 next_cursor,
8512 bootstrap_state,
8513 } => {
8514 sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
8515 break;
8516 }
8517 Frame::Unknown { .. } => {}
8518 other => body.push(other),
8519 }
8520 }
8521 if let Err(SectionError::Abort(code, message)) = self.process_section(
8522 transport,
8523 &id,
8524 status,
8525 &reason_code,
8526 effective_scopes,
8527 body,
8528 sub_end,
8529 meta,
8530 &mut report,
8531 ) {
8532 failure = Some((code, message));
8533 break;
8534 }
8535 }
8536 Frame::Lease {
8537 lease_id,
8538 expires_at_ms,
8539 } => {
8540 self.set_lease_state(Some(LeaseState {
8543 lease_id: Some(lease_id),
8544 expires_at_ms: Some(expires_at_ms),
8545 error_code: None,
8546 }));
8547 }
8548 Frame::Error { code, message, .. } => {
8549 failure = Some((code, message));
8552 break;
8553 }
8554 Frame::Unknown { .. } => {}
8555 _ => {
8556 failure = Some((
8557 "sync.invalid_request".to_owned(),
8558 "unexpected frame in response".to_owned(),
8559 ));
8560 break;
8561 }
8562 }
8563 }
8564
8565 if self.overlay_dirty.get() {
8570 self.rebuild_overlay();
8571 }
8572 if let Some((error_code, message)) = failure {
8573 return SyncOutcome::Failed {
8574 error_code,
8575 message,
8576 };
8577 }
8578 if let Err(message) = self.drain_pending_evictions() {
8581 return SyncOutcome::Failed {
8582 error_code: "storage.failed".into(),
8583 message,
8584 };
8585 }
8586 self.ack_after_pull(transport);
8587 if self.upgrading && report.bootstrapping.is_empty() {
8590 self.set_upgrading(false);
8591 }
8592 SyncOutcome::Ok(report)
8593 }
8594
8595 fn handle_push_results(
8598 &mut self,
8599 frames: &[Frame],
8600 pushed_ids: &HashSet<&str>,
8601 rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
8602 report: &mut SyncReport,
8603 last_final_push_result_id: Option<&str>,
8604 ) -> Result<(), String> {
8605 self.begin_observation("syncular_push_results")?;
8606 let conflict_count = self.conflicts.len();
8607 let rejection_count = self.rejections.len();
8608 let applied_count = report.applied.len();
8609 let rejected_count = report.rejected.len();
8610 let retryable_count = report.retryable.len();
8611 let was_dirty = self.overlay_dirty.get();
8612 let mut removed = Vec::new();
8613 let mut batch = ChangeAccumulator::default();
8614 let persisted = (|| {
8615 for frame in frames {
8616 let Frame::PushResult {
8617 client_commit_id,
8618 status,
8619 results,
8620 ..
8621 } = frame
8622 else {
8623 return Err("unexpected acknowledgement frame".to_owned());
8624 };
8625 if !pushed_ids.contains(client_commit_id.as_str()) {
8626 continue;
8627 }
8628 let Some(index) = self
8629 .outbox
8630 .iter()
8631 .position(|commit| commit.client_commit_id == *client_commit_id)
8632 else {
8633 continue;
8634 };
8635 let status = *status;
8636 let rejection_details = rejection_details_by_commit.get(client_commit_id);
8637 let operations = self.outbox[index].ops.clone();
8638 match status {
8639 PushStatus::Applied | PushStatus::Cached => {
8640 let journal_results = results
8643 .iter()
8644 .map(|result| {
8645 let op_index = match result {
8646 OpResult::Applied { op_index }
8647 | OpResult::Conflict { op_index, .. }
8648 | OpResult::Error { op_index, .. } => *op_index,
8649 };
8650 CommitOperationOutcome::Applied { op_index }
8651 })
8652 .collect::<Vec<_>>();
8653 let outcome_status = if status == PushStatus::Applied {
8654 CommitOutcomeStatus::Applied
8655 } else {
8656 CommitOutcomeStatus::Cached
8657 };
8658 let persisted = self
8659 .persist_commit_outcome(
8660 client_commit_id,
8661 outcome_status,
8662 &journal_results,
8663 None,
8664 )
8665 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
8666 persisted?;
8667 self.overlay_dirty.set(true);
8668 batch.status = true;
8669 batch.outcomes = true;
8670 }
8671 PushStatus::Rejected => {
8672 if results.iter().any(|result| {
8673 matches!(
8674 result,
8675 OpResult::Error {
8676 code,
8677 retryable: true,
8678 ..
8679 } if code == "sync.idempotency_cache_miss"
8680 )
8681 }) {
8682 report.retryable.push(client_commit_id.to_owned());
8685 continue;
8686 }
8687
8688 let mut journal_results = Vec::with_capacity(results.len());
8689 let mut conflicts = Vec::new();
8690 let mut rejections = Vec::new();
8691 for result in results {
8692 match result {
8693 OpResult::Applied { op_index } => {
8694 journal_results.push(CommitOperationOutcome::Applied {
8695 op_index: *op_index,
8696 });
8697 }
8698 OpResult::Conflict {
8699 op_index,
8700 code,
8701 message,
8702 server_version,
8703 server_row,
8704 conflict_columns: conflict_columns_bitmap,
8705 } => {
8706 let operation = operations
8707 .get(*op_index as usize)
8708 .map(CommitOperation::from);
8709 let (table, row_id) = operation
8710 .as_ref()
8711 .map(|op| (op.table.clone(), op.row_id.clone()))
8712 .unwrap_or_default();
8713 let server_row_json = self
8714 .schema
8715 .table(&table)
8716 .and_then(|t| {
8717 decode_row_bytes(t, server_row, &self.encryption)
8718 .ok()
8719 .map(|row| (t, row))
8720 })
8721 .map(|(t, row)| {
8722 let mut map = Map::new();
8723 for (i, column) in t.columns.iter().enumerate() {
8724 map.insert(
8725 column.name.clone(),
8726 column_value_to_json(
8727 row.get(i).unwrap_or(&None),
8728 ),
8729 );
8730 }
8731 map
8732 })
8733 .unwrap_or_default();
8734 let conflict_columns = self
8737 .schema
8738 .table(&table)
8739 .map(|t| {
8740 t.columns
8741 .iter()
8742 .enumerate()
8743 .filter(|(index, _)| {
8744 conflict_columns_bitmap
8745 .get(index / 8)
8746 .is_some_and(|byte| {
8747 byte & (1 << (index % 8)) != 0
8748 })
8749 })
8750 .map(|(_, column)| column.name.clone())
8751 .collect::<Vec<_>>()
8752 })
8753 .unwrap_or_default();
8754 let conflict = ConflictRecord {
8755 client_commit_id: client_commit_id.to_owned(),
8756 op_index: *op_index,
8757 table,
8758 row_id,
8759 code: code.clone(),
8760 message: message.clone(),
8761 server_version: *server_version,
8762 server_row: server_row_json,
8763 conflict_columns,
8764 operation,
8765 };
8766 journal_results.push(CommitOperationOutcome::Conflict {
8767 conflict: conflict.clone(),
8768 });
8769 conflicts.push(conflict);
8770 }
8771 OpResult::Error {
8772 op_index,
8773 code,
8774 message,
8775 retryable,
8776 } => {
8777 let rejection = RejectionRecord {
8778 client_commit_id: client_commit_id.to_owned(),
8779 op_index: *op_index,
8780 code: code.clone(),
8781 message: message.clone(),
8782 retryable: *retryable,
8783 details: rejection_details
8784 .and_then(|details| details.get(op_index))
8785 .cloned(),
8786 operation: operations
8787 .get(*op_index as usize)
8788 .map(CommitOperation::from),
8789 };
8790 journal_results.push(CommitOperationOutcome::Error {
8791 rejection: rejection.clone(),
8792 });
8793 rejections.push(rejection);
8794 }
8795 }
8796 }
8797 let outcome_status = if conflicts.is_empty() {
8798 CommitOutcomeStatus::Rejected
8799 } else {
8800 CommitOutcomeStatus::Conflict
8801 };
8802 let persisted = self
8803 .persist_commit_outcome(
8804 client_commit_id,
8805 outcome_status,
8806 &journal_results,
8807 Some(&operations),
8808 )
8809 .and_then(|()| self.delete_outbox_persisted(client_commit_id));
8810 persisted?;
8811 batch.conflicts = !conflicts.is_empty();
8812 batch.rejections = !rejections.is_empty();
8813 batch.status = true;
8814 batch.outcomes = true;
8815 self.conflicts.extend(conflicts);
8816 self.rejections.extend(rejections);
8817 self.overlay_dirty.set(true);
8818 }
8819 }
8820 removed.push((index, self.outbox.remove(index)));
8821 if batch.status {
8822 for operation in &operations {
8823 if !self.record_row_scopes(
8824 &mut batch,
8825 &operation.table,
8826 &operation.row_id,
8827 false,
8828 ) {
8829 batch.table(&operation.table);
8830 }
8831 }
8832 }
8833 match status {
8834 PushStatus::Applied | PushStatus::Cached => {
8835 report.applied.push(client_commit_id.to_owned())
8836 }
8837 PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
8838 }
8839 }
8840 if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
8841 self.prune_commit_outcomes()?;
8842 }
8843 self.finish_observation("syncular_push_results", batch)
8844 })();
8845 if let Err(error) = persisted {
8846 self.rollback_observation("syncular_push_results");
8847 for (index, commit) in removed.into_iter().rev() {
8848 self.outbox.insert(index, commit);
8849 }
8850 self.conflicts.truncate(conflict_count);
8851 self.rejections.truncate(rejection_count);
8852 self.overlay_dirty.set(was_dirty);
8853 report.applied.truncate(applied_count);
8854 report.rejected.truncate(rejected_count);
8855 report.retryable.truncate(retryable_count);
8856 return Err(error);
8857 }
8858 report.conflicts += (self.conflicts.len() - conflict_count) as u32;
8859 Ok(())
8860 }
8861
8862 fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
8863 status != PushStatus::Rejected
8864 || !results.iter().any(|result| {
8865 matches!(
8866 result,
8867 OpResult::Error {
8868 code,
8869 retryable: true,
8870 ..
8871 } if code == "sync.idempotency_cache_miss"
8872 )
8873 })
8874 }
8875
8876 #[allow(clippy::too_many_arguments)]
8879 fn process_section(
8880 &mut self,
8881 transport: &mut dyn Transport,
8882 id: &str,
8883 status: SubStatus,
8884 reason_code: &str,
8885 effective_scopes: Vec<(String, Vec<String>)>,
8886 body: Vec<Frame>,
8887 sub_end: Option<(i64, Option<String>)>,
8888 meta: &RequestMeta,
8889 report: &mut SyncReport,
8890 ) -> Result<(), SectionError> {
8891 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
8892 return Ok(()); };
8894 if status != SubStatus::Active && sub_end.is_none() {
8895 return Err(SectionError::Abort(
8896 "sync.invalid_request".into(),
8897 "subscription section without SUB_END".into(),
8898 ));
8899 }
8900 match status {
8901 SubStatus::Revoked => {
8902 self.begin_observation("syncular_revocation")
8903 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
8904 let previous = self.subs[sub_index].clone();
8905 let was_dirty = self.overlay_dirty.get();
8906 let previous_outbox = self.outbox.clone();
8907 let rejection_count = self.rejections.len();
8908 let revoked_count = report.revoked.len();
8909 let failed_count = report.failed.len();
8910 let result =
8911 (|| {
8912 let mut batch = ChangeAccumulator::default();
8913 let registered = self.window_unit_by_sub_id(id);
8914 let (table, effective) = {
8916 let sub = &self.subs[sub_index];
8917 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
8918 };
8919 let purged = self.purge_scope_rows(&table, &effective);
8920 match purged {
8921 Ok(()) => {
8922 self.record_scope_map(&mut batch, &table, &effective);
8923 let sub = &mut self.subs[sub_index];
8924 sub.state = SubState::Revoked;
8925 sub.reason_code = Some(if reason_code.is_empty() {
8926 "sync.scope_revoked".to_owned()
8927 } else {
8928 reason_code.to_owned()
8929 });
8930 report.revoked.push(id.to_owned());
8931 let doomed_effective = effective;
8932 let sub_table = table;
8933 self.persist_sub(&self.subs[sub_index].clone()).map_err(
8934 |message| SectionError::Abort("storage.failed".into(), message),
8935 )?;
8936 let dropped = self
8937 .drop_doomed_outbox(&sub_table, &doomed_effective)
8938 .map_err(|message| {
8939 SectionError::Abort("storage.failed".to_owned(), message)
8940 })?;
8941 if dropped {
8942 batch.status = true;
8943 batch.rejections = true;
8944 batch.outcomes = true;
8945 }
8946 self.delete_unreferenced_cached_blobs();
8949 }
8950 Err(()) => {
8951 let sub = &mut self.subs[sub_index];
8954 sub.state = SubState::Failed;
8955 sub.reason_code = Some("sync.scope_revoked".to_owned());
8956 report.failed.push(id.to_owned());
8957 self.persist_sub(&self.subs[sub_index].clone()).map_err(
8958 |message| SectionError::Abort("storage.failed".into(), message),
8959 )?;
8960 }
8961 }
8962 if let Some((base_key, unit)) = registered {
8963 batch.window(&base_key, &self.subs[sub_index].table, &unit);
8964 }
8965 self.rebuild_overlay_if_dirty();
8966 self.finish_observation("syncular_revocation", batch)
8967 .map_err(|message| {
8968 SectionError::Abort("storage.failed".to_owned(), message)
8969 })?;
8970 Ok(())
8971 })();
8972 if result.is_err() {
8973 self.rollback_observation("syncular_revocation");
8974 self.subs[sub_index] = previous;
8975 self.overlay_dirty.set(was_dirty);
8976 self.outbox = previous_outbox;
8977 self.rejections.truncate(rejection_count);
8978 report.revoked.truncate(revoked_count);
8979 report.failed.truncate(failed_count);
8980 }
8981 result
8982 }
8983 SubStatus::Reset => {
8984 self.begin_observation("syncular_reset")
8985 .map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
8986 let previous = self.subs[sub_index].clone();
8987 let was_dirty = self.overlay_dirty.get();
8988 let reset_count = report.resets.len();
8989 let result = (|| {
8990 let mut batch = ChangeAccumulator::default();
8991 let registered = self.window_unit_by_sub_id(id);
8992 let sub = &mut self.subs[sub_index];
8995 sub.cursor = -1;
8996 sub.bootstrap_state = None;
8997 report.resets.push(id.to_owned());
8998 self.persist_sub(&self.subs[sub_index].clone())
8999 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9000 if let Some((base_key, unit)) = registered {
9001 batch.window(&base_key, &self.subs[sub_index].table, &unit);
9002 }
9003 self.finish_observation("syncular_reset", batch)
9004 .map_err(|message| {
9005 SectionError::Abort("storage.failed".to_owned(), message)
9006 })?;
9007 Ok(())
9008 })();
9009 if result.is_err() {
9010 self.rollback_observation("syncular_reset");
9011 self.subs[sub_index] = previous;
9012 self.overlay_dirty.set(was_dirty);
9013 report.resets.truncate(reset_count);
9014 }
9015 result
9016 }
9017 SubStatus::Active => {
9018 let fresh = meta
9019 .fresh
9020 .iter()
9021 .find(|(fid, _)| fid == id)
9022 .map(|(_, fresh)| *fresh)
9023 .unwrap_or(false);
9024 let previous = self.subs[sub_index].clone();
9025 self.subs[sub_index].effective = Some(effective_scopes);
9026 let outcome = self
9027 .apply_section_body(transport, sub_index, body, fresh, meta, report)
9028 .and_then(|()| {
9029 let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
9030 SectionError::Abort(
9031 "sync.invalid_request".into(),
9032 "subscription section without SUB_END".into(),
9033 )
9034 })?;
9035 self.apply_sub_end(sub_index, cursor, bootstrap_state)
9036 .map_err(|message| {
9037 SectionError::Abort("storage.failed".into(), message)
9038 })
9039 });
9040 match outcome {
9041 Ok(()) => {
9042 if self.subs[sub_index].bootstrap_state.is_some() {
9043 report.bootstrapping.push(id.to_owned());
9044 }
9045 Ok(())
9046 }
9047 Err(SectionError::FailClosed) => {
9048 self.subs[sub_index] = previous.clone();
9049 self.begin_observation("syncular_section_failure")
9050 .map_err(|message| {
9051 SectionError::Abort("storage.failed".into(), message)
9052 })?;
9053 let persisted = (|| {
9054 let mut batch = ChangeAccumulator::default();
9055 if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
9056 batch.window(&base_key, &self.subs[sub_index].table, &unit);
9057 }
9058 self.subs[sub_index].state = SubState::Failed;
9059 self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
9060 self.persist_sub(&self.subs[sub_index])?;
9061 self.finish_observation("syncular_section_failure", batch)
9062 })();
9063 if let Err(message) = persisted {
9064 self.rollback_observation("syncular_section_failure");
9065 self.subs[sub_index] = previous;
9066 return Err(SectionError::Abort("storage.failed".into(), message));
9067 }
9068 report.failed.push(id.to_owned());
9069 Ok(())
9070 }
9071 Err(error) => {
9072 self.subs[sub_index] = previous;
9075 Err(error)
9076 }
9077 }
9078 }
9079 }
9080 }
9081
9082 fn apply_sub_end(
9083 &mut self,
9084 sub_index: usize,
9085 cursor: i64,
9086 bootstrap_state: Option<String>,
9087 ) -> Result<(), String> {
9088 #[cfg(feature = "bench-internals")]
9089 let _phase = self.benchmark_phases.start(Phase::CursorPersist);
9090 let previous = self.subs[sub_index].clone();
9091 self.begin_observation("syncular_sub_end")?;
9092 let persisted = (|| {
9093 let mut batch = ChangeAccumulator::default();
9094 let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
9095 && cursor >= 0
9096 && bootstrap_state.is_none();
9097 if completed {
9098 if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
9099 batch.window(&base_key, &previous.table, &unit);
9100 }
9101 }
9102 let sub = &mut self.subs[sub_index];
9103 sub.cursor = cursor;
9104 sub.bootstrap_state = bootstrap_state;
9105 sub.synced_once = true;
9106 self.persist_sub(&self.subs[sub_index])?;
9107 self.finish_observation("syncular_sub_end", batch)
9108 })();
9109 if persisted.is_err() {
9110 self.rollback_observation("syncular_sub_end");
9111 self.subs[sub_index] = previous;
9112 }
9113 persisted
9114 }
9115
9116 fn apply_section_body(
9117 &mut self,
9118 transport: &mut dyn Transport,
9119 sub_index: usize,
9120 body: Vec<Frame>,
9121 fresh: bool,
9122 meta: &RequestMeta,
9123 report: &mut SyncReport,
9124 ) -> Result<(), SectionError> {
9125 let mut saw_segment = false;
9126 for frame in body {
9127 match frame {
9128 Frame::Commit {
9129 tables, changes, ..
9130 } => {
9131 self.apply_commit_frame(&tables, &changes)
9132 .map_err(|(c, m)| SectionError::Abort(c, m))?;
9133 report.commits_applied += 1;
9134 }
9135 Frame::SegmentInline { payload } => {
9136 let segment = decode_rows_segment(&payload)
9137 .map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
9138 self.progress.update(|p| {
9139 p.phase = ProgressPhase::Import;
9140 p.subscription_id = Some(self.subs[sub_index].id.clone());
9141 p.table = Some(self.subs[sub_index].table.clone());
9142 p.segment_id = None;
9143 p.bytes_received = payload.len() as u64;
9144 p.bytes_total = Some(payload.len() as u64);
9145 p.rows_processed = 0;
9146 p.rows_total =
9147 Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
9148 });
9149 let first = !saw_segment;
9150 saw_segment = true;
9151 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
9152 report.segment_rows_applied += applied;
9153 }
9154 Frame::SegmentRef {
9155 segment_id,
9156 byte_length,
9157 media_type,
9158 table,
9159 row_count,
9160 as_of_commit_seq,
9161 scope_digest,
9162 row_cursor,
9163 next_row_cursor,
9164 url,
9165 url_expires_at_ms,
9166 ..
9167 } => {
9168 let advertised = match media_type {
9171 MediaType::Rows => {
9172 meta.accept & ACCEPT_EXTERNAL_ROWS != 0
9173 || meta.accept & ACCEPT_INLINE_ROWS != 0
9174 }
9175 MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
9176 };
9177 if !advertised {
9178 return Err(SectionError::Abort(
9179 "sync.invalid_request".to_owned(),
9180 format!(
9181 "SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
9182 media_type.name()
9183 ),
9184 ));
9185 }
9186 self.progress.update(|p| {
9187 p.phase = ProgressPhase::Download;
9188 p.subscription_id = Some(self.subs[sub_index].id.clone());
9189 p.table = Some(table.clone());
9190 p.segment_id = Some(segment_id.clone());
9191 p.bytes_received = 0;
9192 p.bytes_total = u64::try_from(byte_length).ok();
9193 p.rows_processed = 0;
9194 p.rows_total = u64::try_from(row_count).ok();
9195 });
9196 let observer = self.progress.clone();
9197 let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
9198 let bytes = if let Some(url) = url {
9199 if meta.accept & ACCEPT_SIGNED_URLS == 0 {
9204 return Err(SectionError::Abort(
9205 "sync.invalid_request".to_owned(),
9206 "SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
9207 .to_owned(),
9208 ));
9209 }
9210 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
9212 return Err(SectionError::Abort(
9213 "sync.segment_expired".to_owned(),
9214 format!(
9215 "signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
9216 ),
9217 ));
9218 }
9219 transport
9220 .fetch_url(&url, &mut on_progress)
9221 .map_err(|e| SectionError::Abort(e.code, e.message))?
9222 } else {
9223 let requested_scopes_json =
9224 canonical_scope_json(&self.subs[sub_index].requested);
9225 transport
9226 .download_segment(
9227 &SegmentRequest {
9228 segment_id: segment_id.clone(),
9229 table,
9230 requested_scopes_json,
9231 },
9232 &mut on_progress,
9233 )
9234 .map_err(|e| SectionError::Abort(e.code, e.message))?
9235 };
9236 on_progress(bytes.len() as u64);
9238 let digest = Sha256::digest(&bytes);
9239 let expected = segment_id
9240 .strip_prefix("sha256:")
9241 .unwrap_or(segment_id.as_str());
9242 if bytes_to_hex(&digest) != expected {
9243 return Err(SectionError::Abort(
9244 "sync.invalid_request".to_owned(),
9245 "segment bytes do not match the content address (§5.1)".to_owned(),
9246 ));
9247 }
9248 self.progress.update(|p| p.phase = ProgressPhase::Import);
9249 if media_type == MediaType::Sqlite {
9250 if row_cursor.is_some() || next_row_cursor.is_some() {
9253 return Err(SectionError::Abort(
9254 "sync.invalid_request".to_owned(),
9255 "sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
9256 .to_owned(),
9257 ));
9258 }
9259 let first = !saw_segment;
9260 saw_segment = true;
9261 let applied = self.apply_sqlite_segment(
9262 sub_index,
9263 &bytes,
9264 fresh && first,
9265 row_count,
9266 as_of_commit_seq,
9267 &scope_digest,
9268 )?;
9269 report.segment_rows_applied += applied;
9270 } else {
9271 let segment = decode_rows_segment(&bytes).map_err(|e| {
9272 SectionError::Abort(e.code.as_str().to_owned(), e.detail)
9273 })?;
9274 let first = row_cursor.is_none();
9275 saw_segment = true;
9276 let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
9277 report.segment_rows_applied += applied;
9278 }
9279 }
9280 Frame::Unknown { .. } => {}
9281 Frame::Error { code, message, .. } => {
9282 return Err(SectionError::Abort(code, message))
9283 }
9284 _ => {
9285 return Err(SectionError::Abort(
9286 "sync.invalid_request".to_owned(),
9287 "unexpected frame inside a subscription section".to_owned(),
9288 ));
9289 }
9290 }
9291 }
9292 Ok(())
9293 }
9294
9295 fn apply_commit_frame(
9297 &mut self,
9298 tables: &[String],
9299 changes: &[ssp2::model::Change],
9300 ) -> Result<(), (String, String)> {
9301 #[cfg(feature = "bench-internals")]
9302 let _phase = self.benchmark_phases.start(Phase::CommitApply);
9303 let was_dirty = self.overlay_dirty.get();
9304 self.begin_observation("syncular_commit")
9305 .map_err(|message| ("storage.failed".into(), message))?;
9306 let applied = (|| {
9307 let mut batch = ChangeAccumulator::default();
9308 self.record_commit_changes(&mut batch, tables, changes);
9309 self.apply_commit_changes(tables, changes)?;
9310 self.rebuild_overlay_if_dirty();
9311 self.finish_observation("syncular_commit", batch)
9312 .map_err(|message| ("storage.failed".into(), message))
9313 })();
9314 if applied.is_err() {
9315 self.rollback_observation("syncular_commit");
9316 self.overlay_dirty.set(was_dirty);
9317 }
9318 applied
9319 }
9320
9321 fn apply_commit_changes(
9322 &mut self,
9323 tables: &[String],
9324 changes: &[ssp2::model::Change],
9325 ) -> Result<(), (String, String)> {
9326 let mirror_visible = !self.overlay_dirty.get()
9331 && (self.outbox.is_empty()
9332 || changes.iter().all(|change| {
9333 tables
9334 .get(change.table_index as usize)
9335 .and_then(|name| self.schema.table(name))
9336 .is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
9337 }));
9338 for change in changes {
9339 let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
9340 (
9341 "sync.invalid_request".to_owned(),
9342 "change tableIndex out of range".to_owned(),
9343 )
9344 })?;
9345 let table = self.schema.table(table_name).ok_or_else(|| {
9346 (
9347 "sync.schema_mismatch".to_owned(),
9348 format!("change targets unknown table {table_name:?}"),
9349 )
9350 })?;
9351 match change.op {
9352 Op::Upsert => {
9353 let payload = change.row.as_ref().ok_or_else(|| {
9354 (
9355 "sync.invalid_request".to_owned(),
9356 "upsert change without row payload".to_owned(),
9357 )
9358 })?;
9359 let row = {
9361 #[cfg(feature = "bench-internals")]
9362 let _phase = self.benchmark_phases.start(Phase::RowDecode);
9363 decode_row_bytes(table, payload, &self.encryption)
9364 .map_err(|m| ("sync.invalid_request".to_owned(), m))?
9365 };
9366 let version = change.row_version.unwrap_or(0);
9367 let table_name = table.name.clone();
9368 self.write_base_row(&table_name, &row, version)
9369 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
9370 if mirror_visible {
9371 self.write_row(&visible_table(&table_name), &table_name, &row, version)
9372 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
9373 }
9374 }
9375 Op::Delete => {
9376 self.delete_base_row(table_name, &change.row_id)
9377 .map_err(|m| ("sync.invalid_request".to_owned(), m))?;
9378 if mirror_visible {
9379 let sql = format!(
9380 "DELETE FROM {} WHERE {}",
9381 visible_table(table_name),
9382 row_id_predicate(table)
9383 );
9384 self.conn
9385 .prepare_cached(&sql)
9386 .and_then(|mut statement| {
9387 statement.execute(rusqlite::params![change.row_id])
9388 })
9389 .map_err(|error| {
9390 ("sync.invalid_request".to_owned(), error.to_string())
9391 })?;
9392 }
9393 }
9394 }
9395 }
9396 if mirror_visible {
9397 if !self.outbox.is_empty() {
9398 let changed_rows: HashSet<(&str, &str)> = changes
9399 .iter()
9400 .map(|change| {
9401 (
9402 tables[change.table_index as usize].as_str(),
9403 change.row_id.as_str(),
9404 )
9405 })
9406 .collect();
9407 self.apply_outbox_ops(
9408 self.outbox
9409 .iter()
9410 .flat_map(|commit| &commit.ops)
9411 .filter(|op| {
9412 changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
9413 }),
9414 );
9415 }
9416 self.overlay_dirty.set(false);
9417 }
9418 Ok(())
9419 }
9420
9421 fn apply_segment(
9426 &mut self,
9427 sub_index: usize,
9428 segment: &RowsSegment,
9429 first_fresh_page: bool,
9430 ) -> Result<u32, SectionError> {
9431 let (sub_table, effective) = {
9432 let sub = &self.subs[sub_index];
9433 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
9434 };
9435 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
9436 SectionError::Abort(
9437 "sync.schema_mismatch".to_owned(),
9438 format!("subscription table {sub_table:?} is not in the client schema"),
9439 )
9440 })?;
9441 let matches = segment.table == table.name
9446 && segment.schema_version == self.schema.version
9447 && segment.columns.len() == table.wire_columns.len()
9448 && segment
9449 .columns
9450 .iter()
9451 .zip(table.wire_columns.iter())
9452 .all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
9453 if !matches {
9454 return Err(SectionError::Abort(
9455 "sync.schema_mismatch".to_owned(),
9456 "segment column table does not match the generated schema (§5.2)".to_owned(),
9457 ));
9458 }
9459 let mut applied = 0u32;
9460 let empty = Vec::new();
9461 let blocks = if segment.blocks.is_empty() {
9462 std::slice::from_ref(&empty)
9463 } else {
9464 &segment.blocks
9465 };
9466 for (index, block) in blocks.iter().enumerate() {
9467 let clear = first_fresh_page && index == 0;
9468 if !clear && block.is_empty() {
9469 continue;
9470 }
9471 let was_dirty = self.overlay_dirty.get();
9472 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
9473 && self
9474 .outbox
9475 .iter()
9476 .flat_map(|commit| &commit.ops)
9477 .any(|op| op.table == table.name);
9478 let mirror_visible = !reconcile_pending;
9479 self.begin_observation("syncular_segment_block")
9480 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9481 let outcome = (|| {
9482 self.rebuild_overlay_if_dirty();
9483 let mut changed_keys = Vec::new();
9484 let mut batch = ChangeAccumulator::default();
9485 if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
9486 batch.table(&table.name);
9487 }
9488 if clear {
9489 self.purge_scope_rows(&table.name, &effective)
9490 .map_err(|()| SectionError::FailClosed)?;
9491 }
9492 for (row_index, row) in block.iter().enumerate() {
9493 let decrypted;
9494 let values = if table.has_encrypted_columns() {
9495 let mut values = row.values.clone();
9496 crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
9497 .map_err(|message| {
9498 SectionError::Abort("client.decrypt_failed".into(), message)
9499 })?;
9500 decrypted = values;
9501 &decrypted
9502 } else {
9503 &row.values
9504 };
9505 self.write_base_row(&table.name, values, row.server_version)
9506 .map_err(|message| {
9507 SectionError::Abort("sync.invalid_request".into(), message)
9508 })?;
9509 if reconcile_pending {
9510 changed_keys.push(
9511 owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
9512 |message| SectionError::Abort("storage.failed".into(), message),
9513 )?,
9514 );
9515 }
9516 if mirror_visible {
9517 self.write_row(
9518 &visible_table(&table.name),
9519 &table.name,
9520 values,
9521 row.server_version,
9522 )
9523 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9524 }
9525 let processed = u64::from(applied) + row_index as u64 + 1;
9526 if processed.is_multiple_of(1024) {
9527 self.progress.update(|p| p.rows_processed = processed);
9528 }
9529 }
9530 if reconcile_pending {
9531 self.reconcile_imported_rows(&table, &mut changed_keys)
9532 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9533 }
9534 self.apply_outbox_ops(
9535 self.outbox
9536 .iter()
9537 .flat_map(|commit| &commit.ops)
9538 .filter(|op| op.table == table.name),
9539 );
9540 self.overlay_dirty.set(false);
9541 self.finish_observation("syncular_segment_block", batch)
9542 .map_err(|message| SectionError::Abort("storage.failed".into(), message))
9543 })();
9544 if let Err(error) = outcome {
9545 self.rollback_observation("syncular_segment_block");
9546 self.overlay_dirty.set(was_dirty);
9547 return Err(error);
9548 }
9549 applied += block.len() as u32;
9550 self.progress
9551 .update(|p| p.rows_processed = u64::from(applied));
9552 std::thread::yield_now();
9553 }
9554 self.progress
9555 .update(|p| p.rows_processed = u64::from(applied));
9556 Ok(applied)
9557 }
9558
9559 fn apply_sqlite_segment(
9567 &mut self,
9568 sub_index: usize,
9569 bytes: &[u8],
9570 first_fresh_page: bool,
9571 row_count: i64,
9572 as_of_commit_seq: i64,
9573 scope_digest: &str,
9574 ) -> Result<u32, SectionError> {
9575 let invalid = |detail: &str| {
9576 SectionError::Abort(
9577 "sync.invalid_request".to_owned(),
9578 format!("sqlite segment rejected: {detail} (§5.3)"),
9579 )
9580 };
9581 let (sub_table, effective) = {
9582 let sub = &self.subs[sub_index];
9583 (sub.table.clone(), sub.effective.clone().unwrap_or_default())
9584 };
9585 let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
9586 SectionError::Abort(
9587 "sync.schema_mismatch".to_owned(),
9588 format!("subscription table {sub_table:?} is not in the client schema"),
9589 )
9590 })?;
9591
9592 let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
9593 std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
9594 let img = match rusqlite::Connection::open_with_flags(
9595 &path,
9596 rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
9597 ) {
9598 Ok(conn) => conn,
9599 Err(_) => {
9600 let _ = std::fs::remove_file(&path);
9601 return Err(invalid("bytes do not open as a SQLite database"));
9602 }
9603 };
9604 let outcome = self.apply_sqlite_image(
9605 &img,
9606 &table,
9607 first_fresh_page,
9608 &effective,
9609 row_count,
9610 as_of_commit_seq,
9611 scope_digest,
9612 );
9613 drop(img);
9614 let _ = std::fs::remove_file(&path);
9615 outcome
9616 }
9617
9618 #[allow(clippy::too_many_arguments)]
9619 fn apply_sqlite_image(
9620 &mut self,
9621 img: &rusqlite::Connection,
9622 table: &crate::schema::TableSchema,
9623 first_fresh_page: bool,
9624 effective: &[(String, Vec<String>)],
9625 row_count: i64,
9626 as_of_commit_seq: i64,
9627 scope_digest: &str,
9628 ) -> Result<u32, SectionError> {
9629 let invalid = |detail: String| {
9630 SectionError::Abort(
9631 "sync.invalid_request".to_owned(),
9632 format!("sqlite segment rejected: {detail} (§5.3)"),
9633 )
9634 };
9635
9636 type MetaRow = (i64, String, i64, i64, String, i64, i64);
9639 let meta: MetaRow = img
9640 .query_row(
9641 "SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
9642 \"scopeDigest\", \"rowCount\",
9643 (SELECT count(*) FROM _syncular_segment)
9644 FROM _syncular_segment",
9645 [],
9646 |row| {
9647 Ok((
9648 row.get(0)?,
9649 row.get(1)?,
9650 row.get(2)?,
9651 row.get(3)?,
9652 row.get(4)?,
9653 row.get(5)?,
9654 row.get(6)?,
9655 ))
9656 },
9657 )
9658 .map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
9659 let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
9660 if meta_count != 1 {
9661 return Err(invalid(format!(
9662 "_syncular_segment must contain exactly one row, found {meta_count}"
9663 )));
9664 }
9665 if format != 1 {
9666 return Err(invalid(format!("format {format}")));
9667 }
9668 if meta_table != table.name {
9669 return Err(invalid(format!("image table {meta_table:?}")));
9670 }
9671 if schema_version != i64::from(self.schema.version) {
9672 return Err(invalid(format!("schemaVersion {schema_version}")));
9673 }
9674 if pin != as_of_commit_seq {
9675 return Err(invalid(format!("asOfCommitSeq {pin}")));
9676 }
9677 if digest != scope_digest {
9678 return Err(invalid("scopeDigest mismatch".to_owned()));
9679 }
9680 if meta_rows != row_count {
9681 return Err(invalid(format!("rowCount {meta_rows}")));
9682 }
9683
9684 let mut names: Vec<String> = Vec::new();
9686 let mut primary_keys: Vec<String> = Vec::new();
9687 {
9688 let mut stmt = img
9689 .prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
9690 .map_err(|_| invalid("image data table missing".to_owned()))?;
9691 let mut rows = stmt
9692 .query([])
9693 .map_err(|_| invalid("image data table unreadable".to_owned()))?;
9694 while let Some(row) = rows
9695 .next()
9696 .map_err(|_| invalid("image data table unreadable".to_owned()))?
9697 {
9698 if row.get::<_, i64>(5).map_err(|e| invalid(e.to_string()))? > 0 {
9699 primary_keys.push(row.get(1).map_err(|e| invalid(e.to_string()))?);
9700 }
9701 names.push(
9702 row.get::<_, String>(1)
9703 .map_err(|_| invalid("image data table unreadable".to_owned()))?,
9704 );
9705 }
9706 }
9707 let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
9708 expected.push("_syncular_version");
9709 if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
9710 return Err(SectionError::Abort(
9711 "sync.schema_mismatch".to_owned(),
9712 "sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
9713 ));
9714 }
9715
9716 if primary_keys != vec![table.primary_key.clone()] {
9717 return Err(invalid(
9718 "image primary key does not match generated schema".into(),
9719 ));
9720 }
9721 let actual_count: i64 = img
9722 .query_row(
9723 &format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
9724 [],
9725 |row| row.get(0),
9726 )
9727 .map_err(|e| invalid(e.to_string()))?;
9728 if actual_count != row_count {
9729 return Err(invalid("image row count does not match descriptor".into()));
9730 }
9731 let insert = self.insert_row_sql(&base_table(&table.name), table);
9732 let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
9733 let column_list = names
9734 .iter()
9735 .map(|n| quote_ident(n))
9736 .collect::<Vec<_>>()
9737 .join(", ");
9738 let mut stmt = img
9739 .prepare(&format!(
9740 "SELECT {column_list} FROM {} ORDER BY {}",
9741 quote_ident(&table.name),
9742 quote_ident(&table.primary_key)
9743 ))
9744 .map_err(|e| invalid(e.to_string()))?;
9745 let mut rows = stmt.query([]).map_err(|e| invalid(e.to_string()))?;
9746 let mut applied = 0u32;
9747 loop {
9748 let clear = applied == 0 && first_fresh_page;
9749 let was_dirty = self.overlay_dirty.get();
9750 let reconcile_pending = table.indexes.iter().any(|index| index.unique)
9751 && self
9752 .outbox
9753 .iter()
9754 .flat_map(|commit| &commit.ops)
9755 .any(|op| op.table == table.name);
9756 let mirror_visible = !reconcile_pending;
9757 self.begin_observation("syncular_image_chunk")
9758 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9759 let result = (|| {
9760 self.rebuild_overlay_if_dirty();
9761 let mut changed_keys = Vec::new();
9762 let mut batch = ChangeAccumulator::default();
9763 if clear {
9764 if self.scoped_rows_exist(&table.name, effective) {
9765 batch.table(&table.name);
9766 }
9767 self.purge_scope_rows(&table.name, effective)
9768 .map_err(|()| SectionError::FailClosed)?;
9769 }
9770 let mut processed = 0u32;
9771 {
9772 let mut ins = self
9773 .conn
9774 .prepare_cached(&insert)
9775 .map_err(|e| invalid(e.to_string()))?;
9776 let mut mirror = if mirror_visible {
9777 Some(
9778 self.conn
9779 .prepare_cached(&visible_insert)
9780 .map_err(|e| invalid(e.to_string()))?,
9781 )
9782 } else {
9783 None
9784 };
9785 for _ in 0..1024 {
9786 let Some(row) = rows.next().map_err(|e| invalid(e.to_string()))? else {
9787 break;
9788 };
9789 for (i, column) in table.columns.iter().enumerate() {
9790 let cell = row.get_ref(i).map_err(|e| invalid(e.to_string()))?;
9791 let param = image_cell_param(column, cell).map_err(&invalid)?;
9792 ins.raw_bind_parameter(i + 1, ¶m)
9793 .map_err(|e| invalid(e.to_string()))?;
9794 if let Some(mirror) = &mut mirror {
9795 mirror
9796 .raw_bind_parameter(i + 1, ¶m)
9797 .map_err(|e| invalid(e.to_string()))?;
9798 }
9799 }
9800 let version_index = table.columns.len();
9801 let version: i64 =
9802 row.get(version_index).map_err(|e| invalid(e.to_string()))?;
9803 if version < 1 {
9804 return Err(invalid("image row version must be positive".into()));
9805 }
9806 ins.raw_bind_parameter(version_index + 1, version)
9807 .map_err(|e| invalid(e.to_string()))?;
9808 ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
9809 if let Some(mirror) = &mut mirror {
9810 mirror
9811 .raw_bind_parameter(version_index + 1, version)
9812 .map_err(|e| invalid(e.to_string()))?;
9813 mirror.raw_execute().map_err(|e| invalid(e.to_string()))?;
9814 }
9815 if reconcile_pending {
9816 changed_keys.push(
9817 row.get::<_, SqlValue>(table.pk_index)
9818 .map_err(|error| invalid(error.to_string()))?,
9819 );
9820 }
9821 processed += 1;
9822 }
9823 }
9824 if processed > 0 {
9825 batch.table(&table.name);
9826 self.overlay_dirty.set(true);
9827 }
9828 if reconcile_pending {
9829 self.reconcile_imported_rows(table, &mut changed_keys)
9830 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9831 }
9832 self.apply_outbox_ops(
9833 self.outbox
9834 .iter()
9835 .flat_map(|commit| &commit.ops)
9836 .filter(|op| op.table == table.name),
9837 );
9838 self.overlay_dirty.set(false);
9839 self.finish_observation("syncular_image_chunk", batch)
9840 .map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
9841 Ok(processed)
9842 })();
9843 let processed = match result {
9844 Ok(processed) => processed,
9845 Err(error) => {
9846 self.rollback_observation("syncular_image_chunk");
9847 self.overlay_dirty.set(was_dirty);
9848 return Err(error);
9849 }
9850 };
9851 applied += processed;
9852 self.progress
9853 .update(|p| p.rows_processed = u64::from(applied));
9854 std::thread::yield_now();
9855 if i64::from(applied) == actual_count {
9856 break;
9857 }
9858 }
9859 Ok(applied)
9860 }
9861
9862 fn reconcile_imported_rows(
9866 &self,
9867 table: &TableSchema,
9868 keys: &mut Vec<SqlValue>,
9869 ) -> Result<(), String> {
9870 let visible = visible_table(&table.name);
9871 let base = base_table(&table.name);
9872 let pk = quote_ident(&table.primary_key);
9873 let predicate = row_id_predicate(table);
9874 let mut lookup = self.conn.prepare_cached(&format!(
9875 "SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
9876 )).map_err(|error| error.to_string())?;
9877 let mut deleted = HashSet::new();
9878 for op in self
9879 .outbox
9880 .iter()
9881 .flat_map(|commit| &commit.ops)
9882 .filter(|op| op.table == table.name)
9883 {
9884 if op.upsert {
9885 if let Some(values) = &op.values {
9886 let key = json_to_column_value(
9887 &table.columns[table.pk_index],
9888 values.get(&table.primary_key),
9889 )?;
9890 keys.push(owned_sql_value(RowParam::Cell(&key))?);
9891 }
9892 } else if deleted.insert(&op.row_id) {
9893 keys.extend(
9894 lookup
9895 .query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
9896 .map_err(|error| error.to_string())?
9897 .collect::<Result<Vec<_>, _>>()
9898 .map_err(|error| error.to_string())?,
9899 );
9900 }
9901 }
9902 for chunk in keys.chunks(400) {
9905 let holes = vec!["?"; chunk.len()].join(",");
9906 self.conn
9907 .prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
9908 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
9909 .map_err(|error| error.to_string())?;
9910 }
9911 let updates = table
9912 .columns
9913 .iter()
9914 .map(|column| column.name.as_str())
9915 .chain(std::iter::once("_syncular_version"))
9916 .filter(|name| *name != table.primary_key)
9917 .map(|name| {
9918 let name = quote_ident(name);
9919 format!("{name}=excluded.{name}")
9920 })
9921 .collect::<Vec<_>>()
9922 .join(",");
9923 for chunk in keys.chunks(400) {
9924 let holes = vec!["?"; chunk.len()].join(",");
9925 self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
9926 .and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
9927 .map_err(|error| error.to_string())?;
9928 }
9929 Ok(())
9930 }
9931
9932 fn compile_local_data_purge(
9935 &self,
9936 input: &LocalDataPurgeInput,
9937 ) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
9938 let invalid = |message: String| format!("sync.invalid_request: {message}");
9939 if input.purge_id.is_empty()
9940 || input.purge_id.len() > 128
9941 || !is_local_operation_code_like(&input.purge_id)
9942 {
9943 return Err(invalid(
9944 "local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
9945 ));
9946 }
9947 if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
9948 return Err(invalid(format!(
9949 "local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
9950 )));
9951 }
9952
9953 let mut deduplicated: BTreeMap<
9954 String,
9955 (CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
9956 > = BTreeMap::new();
9957 for target in &input.targets {
9958 let table = self.schema.table(&target.table).ok_or_else(|| {
9959 invalid(format!(
9960 "local purge names unknown table {:?}",
9961 target.table
9962 ))
9963 })?;
9964 if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
9965 return Err(invalid(format!(
9966 "local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
9967 target.table
9968 )));
9969 }
9970 let mut selectors = Vec::with_capacity(target.selectors.len());
9971 let mut canonical_selectors = BTreeMap::new();
9972 for (column_name, raw_values) in &target.selectors {
9973 let Some((column_index, column)) = table
9974 .columns
9975 .iter()
9976 .enumerate()
9977 .find(|(_, column)| column.name == *column_name)
9978 else {
9979 return Err(invalid(format!(
9980 "local purge target {:?} names unknown column {:?}",
9981 target.table, column_name
9982 )));
9983 };
9984 let encrypted = table
9985 .encrypted_columns
9986 .iter()
9987 .any(|candidate| candidate.index == column_index);
9988 if column.ty != ColumnType::String || encrypted {
9989 return Err(invalid(format!(
9990 "local purge selector {:?}.{:?} must be a plaintext string column",
9991 target.table, column_name
9992 )));
9993 }
9994 if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
9995 return Err(invalid(format!(
9996 "local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
9997 target.table, column_name
9998 )));
9999 }
10000 let mut values = raw_values.clone();
10001 values.sort();
10002 values.dedup();
10003 if values.iter().any(|value| {
10004 value.is_empty()
10005 || value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
10006 || !is_local_operation_code_like(value)
10007 }) {
10008 return Err(invalid(format!(
10009 "local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
10010 )));
10011 }
10012 selectors.push((column_name.clone(), values.clone()));
10013 canonical_selectors.insert(column_name.clone(), values);
10014 }
10015 let canonical = LocalDataPurgeTarget {
10016 table: target.table.clone(),
10017 selectors: canonical_selectors,
10018 };
10019 let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10020 deduplicated.insert(
10021 key,
10022 (
10023 CompiledLocalDataPurgeTarget {
10024 table: target.table.clone(),
10025 selectors,
10026 },
10027 canonical,
10028 ),
10029 );
10030 }
10031 let targets = deduplicated
10032 .values()
10033 .map(|(compiled, _)| compiled.clone())
10034 .collect::<Vec<_>>();
10035 let canonical = deduplicated
10036 .values()
10037 .map(|(_, target)| target.clone())
10038 .collect::<Vec<_>>();
10039 let canonical_plan =
10040 serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
10041 Ok((targets, canonical_plan))
10042 }
10043
10044 fn local_purge_base_row_ids(
10045 &self,
10046 targets: &[CompiledLocalDataPurgeTarget],
10047 ) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
10048 let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
10049 for target in targets {
10050 let table = self
10051 .schema
10052 .table(&target.table)
10053 .ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
10054 let mut clauses = Vec::with_capacity(target.selectors.len());
10055 let mut params = Vec::new();
10056 for (column, values) in &target.selectors {
10057 clauses.push(format!(
10058 "{} IN ({})",
10059 quote_ident(column),
10060 values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
10061 ));
10062 params.extend(values.iter().cloned().map(SqlValue::Text));
10063 }
10064 let sql = format!(
10065 "SELECT CAST({} AS TEXT) FROM {} WHERE {}",
10066 quote_ident(&table.primary_key),
10067 base_table(&target.table),
10068 clauses.join(" AND ")
10069 );
10070 let mut statement = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
10071 let rows = statement
10072 .query_map(rusqlite::params_from_iter(params), |row| {
10073 row.get::<_, String>(0)
10074 })
10075 .map_err(|error| error.to_string())?;
10076 let ids = by_table.entry(target.table.clone()).or_default();
10077 for row in rows {
10078 ids.insert(row.map_err(|error| error.to_string())?);
10079 }
10080 }
10081 Ok(by_table)
10082 }
10083
10084 fn local_purge_values_match(
10085 target: &CompiledLocalDataPurgeTarget,
10086 values: &Map<String, Value>,
10087 ) -> bool {
10088 target.selectors.iter().all(|(column, allowed)| {
10089 values
10090 .get(column)
10091 .and_then(Value::as_str)
10092 .is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
10093 })
10094 }
10095
10096 pub fn purge_local_data(
10100 &mut self,
10101 input: &LocalDataPurgeInput,
10102 ) -> Result<LocalDataPurgeResult, String> {
10103 let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
10104 let meta_key = format!("localPurge:{}", input.purge_id);
10105 let applied_plan = self
10106 .conn
10107 .query_row(
10108 "SELECT value FROM _syncular_meta WHERE key = ?1",
10109 rusqlite::params![meta_key],
10110 |row| row.get::<_, String>(0),
10111 )
10112 .optional()
10113 .map_err(|error| error.to_string())?;
10114 if let Some(applied_plan) = applied_plan {
10115 if applied_plan != canonical_plan {
10116 return Err(format!(
10117 "sync.invalid_request: local purge id {:?} was already used with a different plan",
10118 input.purge_id
10119 ));
10120 }
10121 return Ok(LocalDataPurgeResult {
10122 already_applied: true,
10123 purged_rows: 0,
10124 dropped_commits: 0,
10125 });
10126 }
10127
10128 let prior_outbox = self.outbox.clone();
10129 let prior_rejection_count = self.rejections.len();
10130 let prior_overlay_dirty = self.overlay_dirty.get();
10131 self.begin_observation("syncular_local_purge")?;
10132 let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
10133 let row_ids = self.local_purge_base_row_ids(&targets)?;
10134 let doomed = self
10135 .outbox
10136 .iter()
10137 .filter(|commit| {
10138 commit.ops.iter().any(|operation| {
10139 let matching_targets = targets
10140 .iter()
10141 .filter(|target| target.table == operation.table)
10142 .collect::<Vec<_>>();
10143 if matching_targets.is_empty() {
10144 return false;
10145 }
10146 if row_ids
10147 .get(&operation.table)
10148 .is_some_and(|ids| ids.contains(&operation.row_id))
10149 {
10150 return true;
10151 }
10152 operation.values.as_ref().is_some_and(|values| {
10153 matching_targets
10154 .iter()
10155 .any(|target| Self::local_purge_values_match(target, values))
10156 })
10157 })
10158 })
10159 .cloned()
10160 .collect::<Vec<_>>();
10161 let doomed_ids = doomed
10162 .iter()
10163 .map(|commit| commit.client_commit_id.clone())
10164 .collect::<BTreeSet<_>>();
10165 let mut batch = ChangeAccumulator::default();
10166 let mut rejections = Vec::new();
10167 for commit in &doomed {
10168 for operation in &commit.ops {
10169 batch.table(&operation.table);
10170 }
10171 let results = commit
10172 .ops
10173 .iter()
10174 .enumerate()
10175 .map(|(op_index, operation)| {
10176 let rejection = RejectionRecord {
10177 client_commit_id: commit.client_commit_id.clone(),
10178 op_index: op_index as i32,
10179 code: "client.local_data_purged".to_owned(),
10180 message: "the commit was dropped by an application-authorized local data purge".to_owned(),
10181 retryable: false,
10182 details: None,
10183 operation: Some(CommitOperation::from(operation)),
10184 };
10185 rejections.push(rejection.clone());
10186 CommitOperationOutcome::Error { rejection }
10187 })
10188 .collect::<Vec<_>>();
10189 self.persist_commit_outcome(
10190 &commit.client_commit_id,
10191 CommitOutcomeStatus::Rejected,
10192 &results,
10193 Some(&commit.ops),
10194 )?;
10195 self.delete_outbox_persisted(&commit.client_commit_id)?;
10196 }
10197 if !doomed.is_empty() {
10198 self.outbox
10199 .retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
10200 self.rejections.extend(rejections);
10201 self.prune_commit_outcomes()?;
10202 self.overlay_dirty.set(true);
10203 batch.status = true;
10204 batch.rejections = true;
10205 batch.outcomes = true;
10206 }
10207
10208 let mut purged_rows = 0usize;
10209 for (table, ids) in &row_ids {
10210 if ids.is_empty() {
10211 continue;
10212 }
10213 batch.table(table);
10214 for row_id in ids {
10215 self.delete_base_row(table, row_id)?;
10216 purged_rows += 1;
10217 }
10218 }
10219 self.rebuild_overlay_if_dirty();
10220 self.delete_unreferenced_cached_blobs();
10221 self.conn
10222 .execute(
10223 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
10224 rusqlite::params![meta_key, canonical_plan],
10225 )
10226 .map_err(|error| error.to_string())?;
10227 Ok((
10228 batch,
10229 LocalDataPurgeResult {
10230 already_applied: false,
10231 purged_rows,
10232 dropped_commits: doomed.len(),
10233 },
10234 ))
10235 })();
10236 let (batch, result) = match applied {
10237 Ok(value) => value,
10238 Err(error) => {
10239 self.rollback_observation("syncular_local_purge");
10240 self.outbox = prior_outbox;
10241 self.rejections.truncate(prior_rejection_count);
10242 self.overlay_dirty.set(prior_overlay_dirty);
10243 return Err(error);
10244 }
10245 };
10246 if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
10247 self.rollback_observation("syncular_local_purge");
10248 self.outbox = prior_outbox;
10249 self.rejections.truncate(prior_rejection_count);
10250 self.overlay_dirty.set(prior_overlay_dirty);
10251 return Err(error);
10252 }
10253 Ok(result)
10254 }
10255
10256 pub fn rebootstrap_local_data(
10262 &mut self,
10263 input: &LocalDataRebootstrapInput,
10264 ) -> Result<LocalDataRebootstrapResult, String> {
10265 if input.rebootstrap_id.is_empty()
10266 || input.rebootstrap_id.len() > 128
10267 || !is_local_operation_code_like(&input.rebootstrap_id)
10268 {
10269 return Err(
10270 "sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
10271 .to_owned(),
10272 );
10273 }
10274 let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
10275 if let Some(persisted) = self.get_meta_strict(&meta_key)? {
10276 let (retained_commits, reset_subscriptions) =
10277 decode_local_rebootstrap_receipt(&persisted)?;
10278 return Ok(LocalDataRebootstrapResult {
10279 already_applied: true,
10280 retained_commits,
10281 reset_subscriptions,
10282 });
10283 }
10284 if self.stopped || self.schema_floor.is_some() {
10285 return Err(
10286 "sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
10287 .to_owned(),
10288 );
10289 }
10290
10291 let retained_commits = self.outbox.len();
10292 let reset_subscriptions = self.subs.len();
10293 let prior_subs = self.subs.clone();
10294 let prior_upgrading = self.upgrading;
10295 let prior_stopped = self.stopped;
10296 let prior_schema_floor = self.schema_floor.clone();
10297 let prior_overlay_dirty = self.overlay_dirty.get();
10298 let prior_sync_needed = self.sync_needed;
10299 let prior_sync_intents = self.sync_intent_queue.clone();
10300 let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
10301
10302 self.begin_observation("syncular_local_rebootstrap")?;
10303 let mut batch = ChangeAccumulator::default();
10304 let applied = (|| -> Result<(), String> {
10305 self.run_schema_reset_observed(&mut batch, false)?;
10306 self.conn
10307 .execute(
10308 "INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
10309 rusqlite::params![meta_key, receipt],
10310 )
10311 .map_err(|error| error.to_string())?;
10312 self.sync_needed = true;
10313 self.sync_intent_queue.push_back(SyncIntent::Interactive);
10314 batch.status = true;
10315 Ok(())
10316 })();
10317 if let Err(error) = applied {
10318 self.rollback_observation("syncular_local_rebootstrap");
10319 self.subs = prior_subs;
10320 self.upgrading = prior_upgrading;
10321 self.stopped = prior_stopped;
10322 self.schema_floor = prior_schema_floor;
10323 self.overlay_dirty.set(prior_overlay_dirty);
10324 self.sync_needed = prior_sync_needed;
10325 self.sync_intent_queue = prior_sync_intents;
10326 return Err(error);
10327 }
10328 if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
10329 self.rollback_observation("syncular_local_rebootstrap");
10330 self.subs = prior_subs;
10331 self.upgrading = prior_upgrading;
10332 self.stopped = prior_stopped;
10333 self.schema_floor = prior_schema_floor;
10334 self.overlay_dirty.set(prior_overlay_dirty);
10335 self.sync_needed = prior_sync_needed;
10336 self.sync_intent_queue = prior_sync_intents;
10337 return Err(error);
10338 }
10339
10340 Ok(LocalDataRebootstrapResult {
10341 already_applied: false,
10342 retained_commits,
10343 reset_subscriptions,
10344 })
10345 }
10346
10347 fn purge_scope_rows(
10352 &mut self,
10353 table_name: &str,
10354 effective: &[(String, Vec<String>)],
10355 ) -> Result<(), ()> {
10356 if effective.is_empty() {
10357 return Ok(());
10358 }
10359 let table = self.schema.table(table_name).ok_or(())?.clone();
10360 let mut clauses = Vec::new();
10361 let mut params: Vec<SqlValue> = Vec::new();
10362 for (variable, values) in effective {
10363 let column = table.scope_column(variable).ok_or(())?;
10364 let placeholders: Vec<String> = values
10365 .iter()
10366 .map(|v| {
10367 params.push(SqlValue::Text(v.clone()));
10368 "?".to_owned()
10369 })
10370 .collect();
10371 clauses.push(format!(
10372 "{} IN ({})",
10373 quote_ident(column),
10374 placeholders.join(", ")
10375 ));
10376 }
10377 let sql = format!(
10378 "DELETE FROM {} WHERE {}",
10379 base_table(table_name),
10380 clauses.join(" AND ")
10381 );
10382 self.overlay_dirty.set(true);
10383 self.conn
10384 .execute(&sql, rusqlite::params_from_iter(¶ms))
10385 .map_err(|_| ())?;
10386 self.conn
10387 .execute(
10388 &format!(
10389 "DELETE FROM {} WHERE {}",
10390 visible_table(table_name),
10391 clauses.join(" AND ")
10392 ),
10393 rusqlite::params_from_iter(¶ms),
10394 )
10395 .map_err(|_| ())?;
10396 Ok(())
10397 }
10398
10399 fn drop_doomed_outbox(
10402 &mut self,
10403 table_name: &str,
10404 effective: &[(String, Vec<String>)],
10405 ) -> Result<bool, String> {
10406 if effective.is_empty() {
10407 return Ok(false);
10408 }
10409 let Some(table) = self.schema.table(table_name).cloned() else {
10410 return Ok(false);
10411 };
10412 let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
10413 for (variable, values) in effective {
10414 match table.scope_column(variable) {
10415 Some(column) => mappings.push((column, values)),
10416 None => return Ok(false), }
10418 }
10419 let doomed: Vec<OutboxCommit> = self
10420 .outbox
10421 .iter()
10422 .filter(|commit| {
10423 commit.ops.iter().any(|op| {
10424 op.upsert
10425 && op.table == table_name
10426 && op.values.as_ref().is_some_and(|values| {
10427 mappings
10428 .iter()
10429 .all(|(column, allowed)| match values.get(*column) {
10430 Some(Value::String(s)) => allowed.contains(s),
10431 Some(Value::Number(n)) => allowed.contains(&n.to_string()),
10432 _ => false,
10433 })
10434 })
10435 })
10436 })
10437 .cloned()
10438 .collect();
10439 if doomed.is_empty() {
10440 return Ok(false);
10441 }
10442 let mut rejections = Vec::new();
10443 for commit in &doomed {
10444 let results = commit
10445 .ops
10446 .iter()
10447 .enumerate()
10448 .map(|(op_index, operation)| {
10449 let rejection = RejectionRecord {
10450 client_commit_id: commit.client_commit_id.clone(),
10451 op_index: op_index as i32,
10452 code: "sync.scope_revoked".to_owned(),
10453 message: "the commit was dropped because its effective scope was revoked"
10454 .to_owned(),
10455 retryable: false,
10456 details: None,
10457 operation: Some(CommitOperation::from(operation)),
10458 };
10459 rejections.push(rejection.clone());
10460 CommitOperationOutcome::Error { rejection }
10461 })
10462 .collect::<Vec<_>>();
10463 self.persist_commit_outcome(
10464 &commit.client_commit_id,
10465 CommitOutcomeStatus::Rejected,
10466 &results,
10467 Some(&commit.ops),
10468 )?;
10469 self.delete_outbox_persisted(&commit.client_commit_id)?;
10470 }
10471 self.prune_commit_outcomes()?;
10472 let doomed_ids = doomed
10473 .iter()
10474 .map(|commit| commit.client_commit_id.as_str())
10475 .collect::<BTreeSet<_>>();
10476 self.outbox
10477 .retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
10478 self.rejections.extend(rejections);
10479 self.overlay_dirty.set(true);
10480 Ok(true)
10481 }
10482
10483 pub fn upload_blob(
10490 &mut self,
10491 bytes: &[u8],
10492 media_type: Option<String>,
10493 name: Option<String>,
10494 ) -> Result<Value, String> {
10495 let blob_id = blob_id_for(bytes);
10496 let now = self.clock_now_ms();
10497 let tx = self.conn.transaction().map_err(|e| e.to_string())?;
10498 tx.execute(
10499 "INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
10500 ON CONFLICT(blob_id) DO NOTHING",
10501 rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
10502 )
10503 .map_err(|e| e.to_string())?;
10504 tx.execute(
10505 "INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
10506 rusqlite::params![blob_id, media_type, now],
10507 )
10508 .map_err(|e| e.to_string())?;
10509 tx.commit().map_err(|e| e.to_string())?;
10510 self.enforce_blob_cache_cap();
10511 let mut obj = Map::new();
10512 obj.insert("blobId".to_owned(), Value::from(blob_id));
10513 obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
10514 if let Some(mt) = media_type {
10515 obj.insert("mediaType".to_owned(), Value::from(mt));
10516 }
10517 if let Some(n) = name {
10518 obj.insert("name".to_owned(), Value::from(n));
10519 }
10520 Ok(Value::Object(obj))
10521 }
10522
10523 pub fn fetch_blob_bytes(
10527 &mut self,
10528 transport: &mut dyn Transport,
10529 blob_id_or_ref: &str,
10530 ) -> Result<FetchedBlob, (String, String)> {
10531 let simple = |m: String| ("client.failed".to_owned(), m);
10532 let blob_id = if blob_id_or_ref.starts_with("sha256:") {
10533 blob_id_or_ref.to_owned()
10534 } else {
10535 let value: Value = serde_json::from_str(blob_id_or_ref)
10536 .map_err(|_| simple("blob ref is not JSON".to_owned()))?;
10537 value
10538 .get("blobId")
10539 .and_then(Value::as_str)
10540 .ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
10541 .to_owned()
10542 };
10543 if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
10544 return Ok(cached);
10545 }
10546 #[cfg(feature = "bench-internals")]
10552 let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
10553 let bytes = match transport
10554 .blob_download(&blob_id)
10555 .map_err(|e| (e.code, e.message))?
10556 {
10557 BlobDownload::Bytes(bytes) => bytes,
10558 BlobDownload::Url {
10559 url,
10560 url_expires_at_ms,
10561 } => {
10562 if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
10564 return Err((
10565 "sync.segment_expired".to_owned(),
10566 format!(
10567 "blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
10568 ),
10569 ));
10570 }
10571 transport
10572 .fetch_blob_url(&url)
10573 .map_err(|e| (e.code, e.message))?
10574 }
10575 };
10576 #[cfg(feature = "bench-internals")]
10577 drop(download_phase);
10578 #[cfg(feature = "bench-internals")]
10579 let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
10580 if blob_id_for(&bytes) != blob_id {
10582 return Err(simple(format!(
10583 "blob content address mismatch for {blob_id}"
10584 )));
10585 }
10586 #[cfg(feature = "bench-internals")]
10587 drop(validate_phase);
10588 #[cfg(feature = "bench-internals")]
10589 let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
10590 let now = self.clock_now_ms();
10591 self.conn
10592 .execute(
10593 "INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
10594 rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
10595 )
10596 .map_err(|e| simple(e.to_string()))?;
10597 self.conn
10598 .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
10599 .map_err(|e| simple(e.to_string()))?;
10600 self.enforce_blob_cache_cap();
10601 #[cfg(feature = "bench-internals")]
10602 drop(insert_phase);
10603 self.get_cached_blob_bytes(&blob_id)
10604 .map_err(simple)?
10605 .ok_or_else(|| simple("blob cache write failed".to_owned()))
10606 }
10607
10608 fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
10609 #[cfg(feature = "bench-internals")]
10610 let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
10611 let mut stmt = self
10612 .conn
10613 .prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
10614 .map_err(|e| e.to_string())?;
10615 let mut rows = stmt
10616 .query(rusqlite::params![blob_id])
10617 .map_err(|e| e.to_string())?;
10618 if let Some(row) = rows.next().map_err(|e| e.to_string())? {
10619 let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
10620 let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
10621 let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
10622 return Ok(Some(FetchedBlob {
10623 blob_id: blob_id.to_owned(),
10624 byte_length,
10625 bytes,
10626 media_type,
10627 }));
10628 }
10629 Ok(None)
10630 }
10631
10632 fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
10634 let pending: Vec<(String, Option<String>)> = {
10635 let mut stmt = self
10636 .conn
10637 .prepare(
10638 "SELECT blob_id, media_type FROM (
10639 SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
10640 UNION ALL
10641 SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
10642 FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
10643 WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
10644 ) ORDER BY created_at_ms, blob_id",
10645 )
10646 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
10647 let rows = stmt
10648 .query_map([], |row| {
10649 Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
10650 })
10651 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
10652 rows.collect::<Result<_, _>>()
10653 .map_err(|error| match error {
10654 rusqlite::Error::InvalidColumnType(..)
10655 | rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
10656 "sync.local_corrupt",
10657 "Pending blob upload metadata is invalid",
10658 ),
10659 _ => TransportError::new("client.failed", error.to_string()),
10660 })?
10661 };
10662 for (blob_id, media_type) in pending {
10663 let stored: Option<(SqlValue, SqlValue)> = self
10664 .conn
10665 .query_row(
10666 "SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
10667 rusqlite::params![blob_id],
10668 |row| Ok((row.get(0)?, row.get(1)?)),
10669 )
10670 .optional()
10671 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
10672 let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
10673 return Err(TransportError::new(
10674 "sync.local_corrupt",
10675 "Pending blob upload body is missing or corrupt",
10676 ));
10677 };
10678 if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
10679 return Err(TransportError::new(
10680 "sync.local_corrupt",
10681 "Pending blob upload body is missing or corrupt",
10682 ));
10683 }
10684 self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
10685 self.conn
10686 .execute(
10687 "DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
10688 rusqlite::params![blob_id],
10689 )
10690 .map_err(|e| TransportError::new("client.failed", e.to_string()))?;
10691 }
10692 Ok(())
10693 }
10694
10695 fn upload_one(
10702 &self,
10703 transport: &mut dyn Transport,
10704 blob_id: &str,
10705 bytes: &[u8],
10706 media_type: Option<&str>,
10707 ) -> Result<(), TransportError> {
10708 match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
10709 BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
10711 url,
10712 url_expires_at_ms,
10713 } => {
10714 let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
10715 if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
10716 return Ok(());
10717 }
10718 }
10720 BlobUploadGrant::None => {
10721 }
10723 }
10724 transport.blob_upload(blob_id, bytes, media_type)
10725 }
10726
10727 fn visible_blob_ids_sql(&self) -> String {
10728 let mut selects = Vec::new();
10729 for table in &self.schema.tables {
10730 for column in table
10731 .columns
10732 .iter()
10733 .filter(|column| column.ty == ColumnType::BlobRef)
10734 {
10735 let identifier = quote_ident(&column.name);
10736 selects.push(format!(
10737 "SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
10738 WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
10739 AND json_type({identifier}, '$.blobId') = 'text'",
10740 quote_ident(&table.name),
10741 ));
10742 }
10743 }
10744 if selects.is_empty() {
10745 "SELECT NULL AS blob_id WHERE 0".to_owned()
10746 } else {
10747 selects.join(" UNION ")
10748 }
10749 }
10750
10751 fn enforce_blob_cache_cap(&self) {
10754 let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
10755 return;
10756 };
10757 let mut total: i64 = self
10758 .conn
10759 .query_row(
10760 "SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
10761 [],
10762 |row| row.get(0),
10763 )
10764 .unwrap_or(0);
10765 if total <= max_bytes {
10766 return;
10767 }
10768 let query = format!(
10769 "SELECT blob_id, byte_length FROM _syncular_blobs
10770 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
10771 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
10772 AND blob_id NOT IN ({})
10773 ORDER BY created_at_ms ASC, blob_id ASC",
10774 self.visible_blob_ids_sql(),
10775 );
10776 let candidates: Vec<(String, i64)> = {
10777 let Ok(mut stmt) = self.conn.prepare(&query) else {
10778 return;
10779 };
10780 let Ok(rows) = stmt.query_map([], |row| {
10781 Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
10782 }) else {
10783 return;
10784 };
10785 rows.filter_map(Result::ok).collect()
10786 };
10787 for (blob_id, byte_length) in candidates {
10788 if total <= max_bytes {
10789 break;
10790 }
10791 let _ = self.conn.execute(
10792 "DELETE FROM _syncular_blobs WHERE blob_id = ?",
10793 rusqlite::params![blob_id],
10794 );
10795 total -= byte_length;
10796 }
10797 }
10798
10799 fn delete_unreferenced_cached_blobs(&mut self) {
10800 #[cfg(feature = "bench-internals")]
10801 let _phase = self.benchmark_phases.start(Phase::BlobRetention);
10802 if !self.schema_has_blobs() {
10803 return;
10804 }
10805 self.rebuild_overlay_if_dirty();
10806 let _ = self.conn.execute(
10807 &format!(
10808 "DELETE FROM _syncular_blobs
10809 WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
10810 AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
10811 AND blob_id NOT IN ({})",
10812 self.visible_blob_ids_sql(),
10813 ),
10814 [],
10815 );
10816 }
10817
10818 fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
10823 if let Some(sql) = self.insert_sql.borrow().get(full_table) {
10824 return sql.clone();
10825 }
10826 let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
10827 columns.push(quote_ident("_syncular_version"));
10828 let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
10829 let primary_key = quote_ident(&table.primary_key);
10830 let updates = columns
10831 .iter()
10832 .filter(|column| **column != primary_key)
10833 .map(|column| format!("{column}=excluded.{column}"))
10834 .collect::<Vec<_>>()
10835 .join(", ");
10836 let sql = format!(
10837 "INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
10838 columns.join(", "),
10839 placeholders.join(", ")
10840 );
10841 self.insert_sql
10842 .borrow_mut()
10843 .insert(full_table.to_owned(), sql.clone());
10844 sql
10845 }
10846
10847 fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
10848 self.overlay_dirty.set(true);
10849 self.write_row(&base_table(table_name), table_name, row, version)
10850 }
10851
10852 fn write_row(
10853 &self,
10854 full_table: &str,
10855 table_name: &str,
10856 row: &Row,
10857 version: i64,
10858 ) -> Result<(), String> {
10859 #[cfg(feature = "bench-internals")]
10860 let _phase = self.benchmark_phases.start(Phase::RowWrite);
10861 let table = self
10862 .schema
10863 .table(table_name)
10864 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
10865 let sql = self.insert_row_sql(full_table, table);
10866 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
10867 let params = row
10868 .iter()
10869 .map(RowParam::Cell)
10870 .chain(std::iter::once(RowParam::Version(version)));
10871 stmt.execute(rusqlite::params_from_iter(params))
10872 .map_err(|e| e.to_string())?;
10873 Ok(())
10874 }
10875
10876 fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
10877 let table = self
10878 .schema
10879 .table(table_name)
10880 .ok_or_else(|| format!("unknown table {table_name:?}"))?;
10881 self.overlay_dirty.set(true);
10882 let sql = format!(
10883 "DELETE FROM {} WHERE {}",
10884 base_table(table_name),
10885 row_id_predicate(table)
10886 );
10887 let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
10888 stmt.execute(rusqlite::params![row_id])
10889 .map_err(|e| e.to_string())?;
10890 Ok(())
10891 }
10892
10893 fn rebuild_overlay_if_dirty(&mut self) {
10896 if self.overlay_dirty.get() {
10897 self.rebuild_overlay();
10898 }
10899 }
10900
10901 fn rebuild_overlay(&mut self) {
10905 #[cfg(feature = "bench-internals")]
10906 let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
10907 #[cfg(test)]
10908 self.overlay_rebuild_count
10909 .set(self.overlay_rebuild_count.get() + 1);
10910 self.exec("SAVEPOINT syncular_overlay");
10911 for table in self.schema.tables.clone() {
10912 for index in &table.fts_indexes {
10913 let _ = self.drop_fts_triggers(index);
10914 }
10915 let visible = visible_table(&table.name);
10916 let base = base_table(&table.name);
10917 self.exec(&format!("DELETE FROM {visible}"));
10918 self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
10919 }
10920 for commit in &self.outbox {
10921 self.apply_outbox_ops(&commit.ops);
10922 }
10923 for table in self.schema.tables.clone() {
10924 for index in &table.fts_indexes {
10925 let _ = self.rebuild_fts_projection(&table, index);
10926 let _ = self.create_fts_triggers(&table, index);
10927 }
10928 }
10929 self.exec("RELEASE syncular_overlay");
10930 self.overlay_dirty.set(false);
10931 }
10932
10933 fn apply_outbox_ops<'a>(&self, ops: impl IntoIterator<Item = &'a OutboxOp>) {
10934 #[cfg(feature = "bench-internals")]
10935 let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
10936 for op in ops {
10937 #[cfg(feature = "bench-internals")]
10938 if let Some(phase) = &mut phase {
10939 phase.unit();
10940 }
10941 let Some(table) = self.schema.table(&op.table) else {
10942 continue;
10943 };
10944 if op.upsert {
10945 let Some(values) = op.values.as_ref() else {
10946 continue;
10947 };
10948 let full = table
10953 .columns
10954 .iter()
10955 .all(|column| values.contains_key(&column.name));
10956 let local = self.visible_row(table, &op.row_id);
10957 if local.is_none() && !full {
10958 continue;
10959 }
10960 let mut row: Row = Vec::with_capacity(table.columns.len());
10961 let mut ok = true;
10962 for (index, column) in table.columns.iter().enumerate() {
10963 let incoming = match values.get(&column.name) {
10964 Some(raw) => json_to_column_value(column, Some(raw)),
10965 None => Ok(local
10966 .as_ref()
10967 .and_then(|(values, _)| values.get(index).cloned())
10968 .flatten()),
10969 };
10970 match incoming {
10971 Ok(value) => row.push(value),
10972 Err(_) => {
10973 ok = false;
10974 break;
10975 }
10976 }
10977 }
10978 if ok {
10979 let version = local.as_ref().map_or(-1, |(_, version)| *version);
10980 let _ = self.write_row(&visible_table(&table.name), &table.name, &row, version);
10981 }
10982 } else {
10983 let sql = format!(
10984 "DELETE FROM {} WHERE {}",
10985 visible_table(&table.name),
10986 row_id_predicate(table)
10987 );
10988 let _ = self
10989 .conn
10990 .prepare_cached(&sql)
10991 .and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]));
10992 }
10993 }
10994 }
10995
10996 fn visible_row(&self, table: &TableSchema, row_id: &str) -> Option<(Row, i64)> {
10999 let sql = format!(
11000 "SELECT * FROM {} WHERE {}",
11001 visible_table(&table.name),
11002 row_id_predicate(table)
11003 );
11004 let mut stmt = self.conn.prepare_cached(&sql).ok()?;
11005 let mut rows = stmt.query(rusqlite::params![row_id]).ok()?;
11006 let row = rows.next().ok()??;
11007 let mut values = Vec::with_capacity(table.columns.len());
11008 for (index, column) in table.columns.iter().enumerate() {
11009 let raw = row.get_ref(index).ok()?;
11010 let json = sql_ref_to_json(column, raw);
11011 values.push(json_to_column_value(column, Some(&json)).ok()?);
11012 }
11013 let version: i64 = row.get(table.columns.len()).ok()?;
11014 Some((values, version))
11015 }
11016
11017 fn exec(&self, sql: &str) {
11018 let _ = self.conn.execute_batch(sql);
11019 }
11020
11021 pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
11024 if self.realtime_connected {
11025 return Ok(());
11026 }
11027 transport
11028 .realtime_connect_for_client(&self.client_id)
11029 .map_err(|e| format!("{}: {}", e.code, e.message))?;
11030 self.realtime_connected = true;
11031 Ok(())
11032 }
11033
11034 pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
11035 if !self.realtime_connected {
11036 return;
11037 }
11038 let _ = transport.realtime_close();
11039 self.realtime_connected = false;
11040 self.presence.clear(); }
11042
11043 pub fn set_presence(
11049 &mut self,
11050 transport: &mut dyn Transport,
11051 scope_key: &str,
11052 doc: Option<&Value>,
11053 ) -> Result<(), String> {
11054 if !self.realtime_connected {
11055 return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
11056 }
11057 let text = encode_presence_publish(scope_key, doc);
11058 transport
11059 .realtime_send(&text)
11060 .map_err(|e| format!("{}: {}", e.code, e.message))
11061 }
11062
11063 pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
11065 self.presence
11066 .get(scope_key)
11067 .map(|peers| peers.values().cloned().collect())
11068 .unwrap_or_default()
11069 }
11070
11071 fn apply_presence(
11073 &mut self,
11074 scope_key: String,
11075 kind: Option<PresenceKind>,
11076 actor_id: Option<String>,
11077 client_id: Option<String>,
11078 doc: Option<Value>,
11079 error: Option<String>,
11080 ) {
11081 if error.is_some() {
11084 return;
11085 }
11086 let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
11087 return;
11088 };
11089 let peer_key = format!("{actor_id} {client_id}");
11090 match kind {
11091 PresenceKind::Leave => {
11092 if let Some(peers) = self.presence.get_mut(&scope_key) {
11093 peers.remove(&peer_key);
11094 if peers.is_empty() {
11095 self.presence.remove(&scope_key);
11096 }
11097 }
11098 }
11099 _ => {
11100 let doc = match doc {
11101 Some(Value::Object(_)) => doc.unwrap(),
11102 _ => return,
11103 };
11104 self.presence.entry(scope_key).or_default().insert(
11105 peer_key,
11106 PresencePeer {
11107 actor_id,
11108 client_id,
11109 doc,
11110 },
11111 );
11112 }
11113 }
11114 }
11115
11116 pub fn on_realtime_text(&mut self, text: &str) {
11118 match parse_control(text) {
11119 Ok(ControlMessage::Hello { requires_sync, .. }) => {
11120 if requires_sync {
11121 self.set_sync_needed(true, true);
11123 }
11124 }
11125 Ok(ControlMessage::Presence {
11126 scope_key,
11127 kind,
11128 actor_id,
11129 client_id,
11130 doc,
11131 error,
11132 ..
11133 }) => {
11134 self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
11135 }
11136 Ok(ControlMessage::Wake { .. }) => {
11137 self.set_sync_needed(true, true);
11139 }
11140 _ => {}
11141 }
11142 }
11143
11144 pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
11147 if self.stopped {
11148 return;
11149 }
11150 #[cfg(feature = "bench-internals")]
11151 let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
11152 let message = match decode_message(bytes) {
11153 Ok(m) if m.msg_kind == MsgKind::Response => m,
11154 _ => {
11155 self.set_sync_needed(true, true);
11156 return;
11157 }
11158 };
11159 #[cfg(feature = "bench-internals")]
11160 drop(decode_phase);
11161 let mut frames = message.frames.into_iter();
11162 let mut applied_cursor: Option<i64> = None;
11163 let mut any_covered = false;
11164 let mut dropped = false;
11165 while let Some(frame) = frames.next() {
11166 let Frame::SubStart {
11167 id,
11168 status,
11169 effective_scopes,
11170 ..
11171 } = frame
11172 else {
11173 continue;
11174 };
11175 let mut body = Vec::new();
11176 let mut next_cursor: Option<i64> = None;
11177 for inner in frames.by_ref() {
11178 match inner {
11179 Frame::SubEnd {
11180 next_cursor: nc, ..
11181 } => {
11182 next_cursor = Some(nc);
11183 break;
11184 }
11185 Frame::Unknown { .. } => {}
11186 other => body.push(other),
11187 }
11188 }
11189 let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
11190 dropped = true;
11191 continue;
11192 };
11193 let sub = &self.subs[sub_index];
11194 if status != SubStatus::Active
11197 || sub.state != SubState::Active
11198 || sub.bootstrap_state.is_some()
11199 || !sub.synced_once
11200 {
11201 dropped = true;
11202 continue;
11203 }
11204 if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
11205 any_covered = true;
11207 continue;
11208 }
11209 let previous_effective = self.subs[sub_index].effective.clone();
11210 let previous_cursor = self.subs[sub_index].cursor;
11211 self.subs[sub_index].effective = Some(effective_scopes);
11212 let mut failed = false;
11213 for inner in body {
11214 match inner {
11215 Frame::Commit {
11216 tables, changes, ..
11217 } => {
11218 if self.apply_commit_frame(&tables, &changes).is_err() {
11219 failed = true;
11220 break;
11221 }
11222 }
11223 _ => {
11224 failed = true;
11225 break;
11226 }
11227 }
11228 }
11229 let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
11230 self.apply_sub_end(sub_index, cursor, None)
11231 .ok()
11232 .map(|()| cursor)
11233 });
11234 let Some(next_cursor) = committed else {
11235 self.subs[sub_index].effective = previous_effective;
11236 self.subs[sub_index].cursor = previous_cursor;
11237 dropped = true;
11238 continue;
11239 };
11240 applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
11241 }
11242 if let Some(cursor) = applied_cursor {
11243 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
11245 let _ = transport.realtime_send(&ack);
11246 } else if !any_covered || dropped {
11247 self.set_sync_needed(true, true);
11249 }
11250 }
11251
11252 fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
11256 if !self.realtime_connected {
11257 return;
11258 }
11259 let floor = self
11260 .subs
11261 .iter()
11262 .filter(|s| {
11263 s.state == SubState::Active
11264 && s.bootstrap_state.is_none()
11265 && s.synced_once
11266 && s.cursor >= 0
11267 })
11268 .map(|s| s.cursor)
11269 .min();
11270 if let Some(cursor) = floor {
11271 let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
11272 let _ = transport.realtime_send(&ack);
11273 }
11274 }
11275}