use crate::{ProgressObserver, ProgressPhase, ProgressState};
use std::cell::{Cell, RefCell};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
use rusqlite::types::{ToSqlOutput, Value as SqlValue, ValueRef};
use rusqlite::{Connection, OpenFlags, OptionalExtension};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use sha2::{Digest, Sha256};
use ssp2::decode::WIRE_VERSION;
use ssp2::model::{Frame, MediaType, Message, MsgKind, Op, OpResult, PushStatus, SubStatus};
use ssp2::primitives::RawJson;
use ssp2::segment::{decode_rows_segment, Column, ColumnType, ColumnValue, Row, RowsSegment};
use ssp2::{
decode_message, encode_message, encode_presence_publish, parse_control, ControlMessage,
PresenceKind,
};
use crate::api::{
ClientChangeBatch, ClientDiagnosticsHost, ClientDiagnosticsLease, ClientDiagnosticsReplica,
ClientDiagnosticsRequest, ClientDiagnosticsSchema, ClientDiagnosticsSnapshot,
ClientDiagnosticsStorage, ClientLimits, CommandEffects, CommitOperation,
CommitOperationOutcome, CommitOutcome, CommitOutcomeQuery, CommitOutcomeResolution,
CommitOutcomeStatus, ConflictRecord, CoverageSnapshot, DiagnosticLastChange,
DiagnosticLastRound, DiagnosticQueryFailure, DiagnosticRoundCounters, DiagnosticSubscription,
FetchedBlob, LeaseState, LocalDataPurgeInput, LocalDataPurgeResult, LocalDataPurgeTarget,
LocalDataRebootstrapInput, LocalDataRebootstrapResult, Mutation, PresencePeer,
PreviousVersionStatus, QueryOwner, QueryReadFailure, QueryRow, QuerySnapshot, QueryValue,
RejectionDetails, RejectionRecord, ResolveCommitOutcomeInput, RowState, SchemaFloor,
SubscriptionStateView, SyncIntent, SyncOutcome, SyncReport, SyncStatusSnapshot, TableChange,
WindowBase, WindowChange, WindowCoverage, WindowState, WindowUnitRef,
CLIENT_DIAGNOSTICS_VERSION, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS,
MAX_DIAGNOSTIC_QUERY_FAILURES,
};
use crate::api::{RealtimePolicy, RealtimeState, REALTIME_UNAVAILABLE_CODE};
#[cfg(feature = "bench-internals")]
use crate::bench::{Phase, Recorder};
use crate::previous_version::{
capture_previous_version_from_replica, first_incompatibility,
reconcile_previous_version_at_boot, set_local_schema_descriptor,
sweep_previous_version_container, PendingCommitAudit, PreviousVersionContextConfig,
PreviousVersionLifecycle, PreviousVersionReadSpec, PreviousVersionSnapshot,
};
use crate::schema::{parse_schema_json, ClientSchema, FtsIndexSchema, TableSchema};
use crate::transport::{BlobDownload, BlobUploadGrant, SegmentRequest, Transport, TransportError};
use crate::values::{
bytes_to_hex, canonical_scope_json, column_value_to_json, decode_row_bytes,
encode_sparse_row_json, full_row_values, json_to_column_value, json_to_scope_map,
normalize_values_casing, render_row_id_json, scope_map_to_json, sort_scope_map,
};
const DEFAULT_ACCEPT: u8 = 0b0111;
const ACCEPT_INLINE_ROWS: u8 = 1 << 0;
const ACCEPT_EXTERNAL_ROWS: u8 = 1 << 1;
const ACCEPT_SQLITE: u8 = 1 << 2;
const ACCEPT_SIGNED_URLS: u8 = 1 << 3;
const MAX_DIAGNOSTIC_DOMAINS: usize = 256;
pub(crate) const LOCAL_SCHEMA_VERSION_KEY: &str = "localSchemaVersion";
const LOCAL_REVISION_KEY: &str = "localRevision";
const CLIENT_ID_KEY: &str = "clientId";
const LEASE_STATE_KEY: &str = "leaseState";
const SCHEMA_FLOOR_KEY: &str = "schemaFloor";
const LOG_EPOCH_KEY: &str = "logEpoch";
const SECURITY_PREFLIGHT_PENDING_KEY: &str = "securityPreflightPending";
const LOCAL_REBOOTSTRAP_RECEIPT_VERSION: u8 = 2;
const MAX_JS_SAFE_INTEGER: u64 = 9_007_199_254_740_991;
const OUTBOX_INCOMPATIBLE_CODE: &str = "sync.outbox_incompatible";
const ENCRYPT_FAILED_CODE: &str = "client.encrypt_failed";
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct PersistedLocalDataRebootstrapReceipt {
version: u8,
retained_commits: u64,
reset_subscriptions: u64,
}
fn invalid_local_rebootstrap_receipt() -> String {
"sync.local_corrupt: persisted local rebootstrap receipt is invalid".to_owned()
}
fn encode_local_rebootstrap_receipt(
retained_commits: usize,
reset_subscriptions: usize,
) -> Result<String, String> {
let retained_commits =
u64::try_from(retained_commits).map_err(|_| invalid_local_rebootstrap_receipt())?;
let reset_subscriptions =
u64::try_from(reset_subscriptions).map_err(|_| invalid_local_rebootstrap_receipt())?;
if retained_commits > MAX_JS_SAFE_INTEGER || reset_subscriptions > MAX_JS_SAFE_INTEGER {
return Err(invalid_local_rebootstrap_receipt());
}
serde_json::to_string(&PersistedLocalDataRebootstrapReceipt {
version: LOCAL_REBOOTSTRAP_RECEIPT_VERSION,
retained_commits,
reset_subscriptions,
})
.map_err(|_| invalid_local_rebootstrap_receipt())
}
fn decode_local_rebootstrap_receipt(value: &str) -> Result<(usize, usize), String> {
if value == "v1" {
return Ok((0, 0));
}
let receipt: PersistedLocalDataRebootstrapReceipt =
serde_json::from_str(value).map_err(|_| invalid_local_rebootstrap_receipt())?;
if receipt.version != LOCAL_REBOOTSTRAP_RECEIPT_VERSION
|| receipt.retained_commits > MAX_JS_SAFE_INTEGER
|| receipt.reset_subscriptions > MAX_JS_SAFE_INTEGER
{
return Err(invalid_local_rebootstrap_receipt());
}
Ok((
usize::try_from(receipt.retained_commits)
.map_err(|_| invalid_local_rebootstrap_receipt())?,
usize::try_from(receipt.reset_subscriptions)
.map_err(|_| invalid_local_rebootstrap_receipt())?,
))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SubState {
Active,
Revoked,
Failed,
}
#[cfg(test)]
mod observation_tests {
use super::*;
use crate::native_transport::HostTransport;
use crate::values::encode_row_json;
use serde_json::json;
fn client() -> SyncClient {
SyncClient::new(
"retry-test".to_owned(),
&json!({
"version": 1,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}]
}),
ClientLimits::default(),
)
.expect("test client")
}
#[cfg(feature = "e2ee")]
#[test]
fn an_unresolvable_key_id_becomes_a_durable_rejection_at_the_push_seam() {
let schema = json!({
"version": 1,
"tables": [{
"name": "secrets",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "encryption_key_id", "type": "string", "nullable": true },
{ "name": "note", "type": "bytes", "nullable": true,
"encrypted": true, "declaredType": "string" }
],
"scopes": []
}]
});
let mut client = SyncClient::new("encrypt-reject".into(), &schema, ClientLimits::default())
.expect("test client");
let mut config = crate::values::EncryptionConfig::default();
config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
config
.key_id_columns
.insert("secrets".to_owned(), "encryption_key_id".to_owned());
client.set_encryption(config);
let commit_id = client
.patch(
"secrets",
"ghost",
Map::from_iter([("note".to_owned(), json!("x"))]),
None,
)
.expect("patch records without an author-time encrypt failure");
assert_eq!(client.pending_commit_ids(), vec![commit_id.clone()]);
assert!(client.drop_unencodable_outbox().unwrap());
assert!(client.pending_commit_ids().is_empty());
let rejections = client.rejections();
assert_eq!(rejections.len(), 1);
assert_eq!(rejections[0].code, "client.encrypt_failed");
assert_eq!(rejections[0].client_commit_id, commit_id);
assert_eq!(
client
.commit_outcome(&commit_id)
.unwrap()
.map(|outcome| outcome.status),
Some(CommitOutcomeStatus::Rejected)
);
}
#[test]
fn patch_with_a_scope_column_matches_the_stored_local_row() {
let mut client = client();
client.create_synced_tables().unwrap();
client
.conn
.execute(
"INSERT INTO tasks (id, project_id, _syncular_version) VALUES ('t1', 'p1', 1)",
[],
)
.unwrap();
client
.patch(
"tasks",
"t1",
Map::from_iter([
("project_id".to_owned(), json!("p1")),
("id".to_owned(), json!("t1")),
]),
None,
)
.expect("an equal scope column is a no-op");
let values = client.outbox[0].ops[0]
.values
.clone()
.expect("the patch records values");
assert_eq!(values.get("id"), Some(&json!("t1")));
assert!(!values.contains_key("project_id"));
let differing = client.patch(
"tasks",
"t1",
Map::from_iter([("project_id".to_owned(), json!("p2"))]),
None,
);
assert!(differing
.expect_err("a differing scope column is rejected")
.contains("patch cannot write scope column"));
let absent = client.patch(
"tasks",
"ghost",
Map::from_iter([("project_id".to_owned(), json!("p1"))]),
None,
);
assert!(absent
.expect_err("an absent local row is rejected")
.contains("patch cannot write scope column"));
assert_eq!(client.pending_commit_ids().len(), 1);
}
#[test]
fn patch_scope_column_parity_for_the_primary_key_and_coerced_values() {
let schema = json!({
"version": 1,
"tables": [
{
"name": "tenants",
"primaryKey": "tenant_id",
"columns": [
{ "name": "tenant_id", "type": "string", "nullable": false },
{ "name": "body", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "tenant:{tenant_id}" }]
},
{
"name": "buckets",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "bucket", "type": "float", "nullable": false },
{ "name": "body", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "bucket:{bucket}" }]
}
]
});
let mut client = SyncClient::new("scope-parity".into(), &schema, ClientLimits::default())
.expect("test client");
client.create_synced_tables().unwrap();
client
.patch(
"tenants",
"t1",
Map::from_iter([
("tenant_id".to_owned(), json!("t1")),
("body".to_owned(), json!("created")),
]),
None,
)
.expect("the primary-key scope column is proven equal by construction");
client
.conn
.execute(
"INSERT INTO buckets (id, bucket, body, _syncular_version) VALUES ('b1', 2.0, 'seed', 1)",
[],
)
.unwrap();
client
.patch(
"buckets",
"b1",
Map::from_iter([
("bucket".to_owned(), json!(2)),
("body".to_owned(), json!("coerced")),
]),
None,
)
.expect("the stored REAL value coerces from a supplied integer");
let values = client.outbox[1].ops[0]
.values
.clone()
.expect("the patch records values");
assert!(!values.contains_key("bucket"));
let differing = client.patch(
"buckets",
"b1",
Map::from_iter([("bucket".to_owned(), json!(3))]),
None,
);
assert!(differing
.expect_err("a differing scope value is rejected")
.contains("patch cannot write scope column"));
}
#[cfg(feature = "e2ee")]
#[test]
fn stored_key_fallback_resolves_an_integer_primary_key() {
let schema = json!({
"version": 1,
"tables": [{
"name": "patients",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "integer", "nullable": false },
{ "name": "encryption_key_id", "type": "string", "nullable": true },
{ "name": "note", "type": "bytes", "nullable": true,
"encrypted": true, "declaredType": "string" }
],
"scopes": []
}]
});
let mut client = SyncClient::new("int-key".into(), &schema, ClientLimits::default())
.expect("test client");
client.create_synced_tables().unwrap();
client
.conn
.execute(
"INSERT INTO patients (id, encryption_key_id, note, _syncular_version)
VALUES (5, 'k1', NULL, 1)",
[],
)
.unwrap();
let mut config = crate::values::EncryptionConfig::default();
config.keys.insert("k1".to_owned(), vec![0x2A; 32]);
config
.key_id_columns
.insert("patients".to_owned(), "encryption_key_id".to_owned());
client.set_encryption(config);
let table = client.schema.table("patients").unwrap();
let fallback = client
.stored_key_fallback(table, "5")
.expect("the stored integer key resolves the fallback");
assert_eq!(fallback[1], Some(ColumnValue::String("k1".to_owned())));
}
#[test]
fn blob_staging_failures_preserve_durable_state_and_allow_retry() {
let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
{"name":"id","type":"string","nullable":false},
{"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
for state in ["absent", "cached", "pinned"] {
let path = std::env::temp_dir().join(format!(
"syncular-blob-stage-{}.sqlite",
uuid::Uuid::new_v4()
));
let mut client = SyncClient::open_path(
"stage-test".into(),
&schema,
Default::default(),
path.to_str().unwrap(),
)
.unwrap();
let bytes = b"atomic stage";
if state != "absent" {
client
.upload_blob(bytes, Some("text/plain".into()), None)
.unwrap();
if state == "cached" {
client
.conn
.execute("DELETE FROM _syncular_blob_uploads", [])
.unwrap();
}
}
let bodies = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
client
.conn
.execute_batch(
"CREATE TRIGGER fail_stage_insert BEFORE INSERT ON _syncular_blobs
BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;
CREATE TRIGGER fail_stage_update BEFORE UPDATE ON _syncular_blobs
BEGIN SELECT RAISE(ABORT, 'injected stage failure'); END;",
)
.unwrap();
let error = client
.upload_blob(bytes, Some("application/octet-stream".into()), None)
.unwrap_err();
assert!(error.contains("injected stage failure"), "{state}: {error}");
assert!(client.conn.is_autocommit());
let immediate = client.query("SELECT * FROM _syncular_blobs", &[]).unwrap();
drop(client);
let mut reopened = SyncClient::open_path(
"stage-test".into(),
&schema,
Default::default(),
path.to_str().unwrap(),
)
.unwrap();
let durable = reopened
.query("SELECT * FROM _syncular_blobs", &[])
.unwrap();
reopened
.conn
.execute_batch("DROP TRIGGER fail_stage_insert; DROP TRIGGER fail_stage_update")
.unwrap();
let retried = reopened.upload_blob(bytes, None, None).unwrap();
assert_eq!(reopened.upload_blob(bytes, None, None).unwrap(), retried);
assert_eq!(
reopened
.query("SELECT blob_id FROM _syncular_blob_uploads", &[])
.unwrap(),
vec![Map::from_iter([(
"blob_id".into(),
Value::from(blob_id_for(bytes)),
)])]
);
drop(reopened);
std::fs::remove_file(&path).unwrap();
assert_eq!(durable, bodies, "{state}: reopened body state");
assert_eq!(immediate, bodies, "{state}: immediate body state");
}
}
#[test]
fn blob_upload_failures_preserve_pins_and_original_commits_after_file_reopen() {
let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
{"name":"id","type":"string","nullable":false},
{"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
for fault in [
"missing",
"body-type",
"length",
"hash",
"pin-metadata",
"read",
"pin-delete",
] {
let path = std::env::temp_dir().join(format!(
"syncular-blob-upload-{}.sqlite",
uuid::Uuid::new_v4()
));
let mut client = SyncClient::open_path(
"upload-test".into(),
&schema,
Default::default(),
path.to_str().unwrap(),
)
.unwrap();
let bytes = b"durable upload";
let ref_value = client
.upload_blob(bytes, Some("text/plain".into()), None)
.unwrap();
let commit = client
.mutate(vec![Mutation::Upsert {
table: "attachments".into(),
values: Map::from_iter([
("id".into(), Value::from("pending")),
(
"file".into(),
Value::from(format!(
"{{\"blobId\":\"{}\",\"byteLength\":{}}}",
blob_id_for(bytes),
bytes.len()
)),
),
]),
base_version: None,
}])
.unwrap();
client
.conn
.execute_batch("CREATE TABLE saved_bodies AS SELECT * FROM _syncular_blobs")
.unwrap();
client.conn.execute_batch(match fault {
"missing" => "DELETE FROM _syncular_blobs",
"body-type" => "UPDATE _syncular_blobs SET bytes = 'wrong type'",
"length" => "UPDATE _syncular_blobs SET byte_length = byte_length + 1",
"hash" => "UPDATE _syncular_blobs SET bytes = zeroblob(byte_length)",
"pin-metadata" => "UPDATE _syncular_blob_uploads SET media_type = x'ff'",
"read" => "ALTER TABLE _syncular_blobs RENAME TO unavailable_bodies",
_ => "CREATE TRIGGER fail_pin_delete BEFORE DELETE ON _syncular_blob_uploads BEGIN SELECT RAISE(ABORT, 'injected pin deletion failure'); END",
}).unwrap();
let state = if fault == "read" {
Vec::new()
} else {
client.query("SELECT * FROM _syncular_blobs", &[]).unwrap()
};
let mut transport = CountingRealtimeTransport::default();
let error = client.flush_blob_uploads(&mut transport).unwrap_err();
assert_eq!(
error.code,
if matches!(fault, "read" | "pin-delete") {
"client.failed"
} else {
"sync.local_corrupt"
},
"{fault}"
);
assert_eq!(
transport.blob_uploads.len(),
usize::from(fault == "pin-delete"),
"{fault}: corrupt bytes must never be uploaded"
);
assert_eq!(client.pending_commit_ids(), std::slice::from_ref(&commit));
if fault != "read" {
assert_eq!(
client.query("SELECT * FROM _syncular_blobs", &[]).unwrap(),
state
);
} else {
client
.conn
.execute_batch("ALTER TABLE unavailable_bodies RENAME TO _syncular_blobs")
.unwrap();
}
drop(client);
let mut reopened = SyncClient::open_path(
"upload-test".into(),
&schema,
Default::default(),
path.to_str().unwrap(),
)
.unwrap();
assert_eq!(reopened.pending_commit_ids(), std::slice::from_ref(&commit));
if fault != "read" {
assert_eq!(
reopened
.query("SELECT * FROM _syncular_blobs", &[])
.unwrap(),
state
);
}
if fault == "pin-delete" {
reopened
.conn
.execute_batch("DROP TRIGGER fail_pin_delete")
.unwrap();
}
if fault == "pin-metadata" {
reopened
.conn
.execute(
"UPDATE _syncular_blob_uploads SET media_type = 'text/plain'",
[],
)
.unwrap();
}
reopened
.conn
.execute_batch(
"DELETE FROM _syncular_blobs;
INSERT INTO _syncular_blobs SELECT * FROM saved_bodies",
)
.unwrap();
transport.blob_uploads.clear();
reopened.flush_blob_uploads(&mut transport).unwrap();
assert_eq!(
transport.blob_uploads,
[(blob_id_for(bytes), bytes.to_vec())]
);
assert_eq!(
reopened
.query("SELECT blob_id FROM _syncular_blob_uploads", &[])
.unwrap(),
Vec::<Map<String, Value>>::new()
);
assert_eq!(reopened.pending_commit_ids(), [commit]);
assert_eq!(
reopened
.get_cached_blob_bytes(ref_value["blobId"].as_str().unwrap())
.unwrap()
.unwrap()
.bytes,
bytes
);
drop(reopened);
std::fs::remove_file(path).unwrap();
}
}
#[test]
fn incoming_frame_transactions_preserve_prefix_cursor_and_revision_after_reopen() {
let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[
{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false}],
"scopes":[{"pattern":"project:{project_id}"}]}]});
for realtime in [false, true] {
for fault in [
"none",
"row",
"revision",
"commit",
"cursor",
"error",
"missing-end",
] {
if realtime && fault == "missing-end" {
continue;
}
let path = std::env::temp_dir()
.join(format!("syncular-frame-{}.sqlite", uuid::Uuid::new_v4()));
let mut client = SyncClient::open_path(
"frame-test".into(),
&schema,
Default::default(),
path.to_str().unwrap(),
)
.unwrap();
client.set_meta(LOG_EPOCH_KEY, "epoch-1");
client
.subscribe(
"tasks".into(),
"tasks".into(),
vec![("project_id".into(), vec!["p1".into()])],
None,
)
.unwrap();
client.subs[0].cursor = 0;
client.subs[0].synced_once = true;
client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
client.persist_sub(&client.subs[0]).unwrap();
client.realtime_state = RealtimeState::Connected;
client.realtime_reason_code = None;
client.drain_change_batches();
let mut transport = CountingRealtimeTransport::default();
let (_, meta) = client.build_request(false);
let mut response = Message {
wire_version: WIRE_VERSION,
msg_kind: MsgKind::Response,
frames: vec![
Frame::RespHeader {
required_schema_version: None,
latest_schema_version: None,
log_epoch: Some("epoch-1".into()),
reset_required: Some(false),
},
Frame::SubStart {
id: "tasks".into(),
status: SubStatus::Active,
reason_code: "".into(),
effective_scopes: vec![("project_id".into(), vec!["p1".into()])],
bootstrap: false,
},
],
};
for (index, id) in ["first", "second"].iter().enumerate() {
let values = Map::from_iter([
("id".into(), Value::from(*id)),
("project_id".into(), Value::from("p1")),
]);
response.frames.push(Frame::Commit {
commit_seq: index as i64 + 1,
actor_id: "writer".into(),
created_at_ms: 1,
tables: vec!["tasks".into()],
changes: vec![ssp2::model::Change {
table_index: 0,
row_id: (*id).into(),
op: Op::Upsert,
row_version: Some(index as i64 + 1),
scopes: vec![("project_id".into(), "p1".into())],
row: Some(
encode_row_json(
client.schema.table("tasks").unwrap(),
id,
&values,
&client.encryption,
)
.unwrap(),
),
}],
});
}
response.frames.push(Frame::SubEnd {
next_cursor: 2,
bootstrap_state: None,
});
let valid = response.clone();
match fault {
"row" => {
let Frame::Commit {changes, ..} = &mut response.frames[3] else {panic!("second frame")};
let mut malformed = changes[0].clone(); malformed.row_id = "malformed".into(); malformed.row = Some(vec![]); changes.push(malformed);
},
"revision" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND NEW.value = '2' AND EXISTS (SELECT 1 FROM _syncular_base_tasks WHERE id = 'second') BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END").unwrap(),
"commit" => client.conn.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE frame_parent(id INTEGER PRIMARY KEY); CREATE TABLE frame_child(id INTEGER REFERENCES frame_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_frame AFTER INSERT ON _syncular_base_tasks WHEN NEW.id = 'second' BEGIN INSERT INTO frame_child VALUES(1); END").unwrap(),
"cursor" => client.conn.execute_batch("CREATE TRIGGER fail_frame BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected cursor failure'); END").unwrap(),
"error" => {response.frames.truncate(3);response.frames.push(Frame::Error {code:"sync.invalid_request".into(),message:"later error".into(),category:"protocol".into(),retryable:false,recommended_action:"retry".into(),details:None});},
"missing-end" => {response.frames.truncate(3);},
_ => {}
}
if realtime {
client.on_realtime_binary(&mut transport, &encode_message(&response));
} else {
assert_eq!(
matches!(
client.process_response(&mut transport, response, &meta),
SyncOutcome::Ok(_)
),
fault == "none",
"{fault}"
);
}
assert!(
client.conn.is_autocommit(),
"no open transaction after {realtime}/{fault}"
);
let count = if matches!(fault, "none" | "cursor") {
2
} else {
1
};
let expected_revision = count + u64::from(realtime && fault != "none");
assert_eq!(
client.local_revision(),
expected_revision,
"{realtime}/{fault}"
);
let batches = client.drain_change_batches();
assert_eq!(
batches.len(),
expected_revision as usize,
"{realtime}/{fault}"
);
let row_batches: Vec<_> = batches
.iter()
.filter(|batch| !batch.tables.is_empty())
.collect();
assert_eq!(row_batches.len(), count as usize);
assert_eq!(
row_batches
.iter()
.map(|b| b.revision.clone())
.collect::<Vec<_>>(),
(1..=count).map(|r| r.to_string()).collect::<Vec<_>>()
);
for table in ["tasks", "_syncular_base_tasks"] {
assert_eq!(
client
.conn
.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| row
.get::<_, u64>(0))
.unwrap(),
count,
"{realtime}/{fault}/{table}"
);
}
assert_eq!(client.subs[0].cursor, if fault == "none" { 2 } else { 0 });
assert_eq!(
transport
.messages
.iter()
.filter(|m| m.contains("\"ack\""))
.count(),
usize::from(fault == "none")
);
if matches!(fault, "revision" | "commit" | "cursor") {
client
.conn
.execute_batch("DROP TRIGGER fail_frame")
.unwrap();
}
drop(client);
let mut reopened = SyncClient::open_path(
"frame-test".into(),
&schema,
Default::default(),
path.to_str().unwrap(),
)
.unwrap();
assert_eq!(reopened.local_revision(), expected_revision);
assert_eq!(
reopened
.query("SELECT id FROM tasks ORDER BY id", &[])
.unwrap()
.len(),
count as usize
);
assert_eq!(reopened.subs[0].cursor, if fault == "none" { 2 } else { 0 });
let (_, meta) = reopened.build_request(false);
assert!(matches!(
reopened.process_response(&mut transport, valid, &meta),
SyncOutcome::Ok(_)
));
assert_eq!(
reopened
.query("SELECT id FROM tasks ORDER BY id", &[])
.unwrap()
.len(),
2
);
assert_eq!(reopened.subs[0].cursor, 2);
drop(reopened);
std::fs::remove_file(path).unwrap();
}
}
}
#[test]
fn failed_subscription_control_persistence_rolls_back_rows_outbox_and_memory() {
for status in [SubStatus::Reset, SubStatus::Revoked] {
for fault in ["subscription", "revision"] {
let mut client = client();
client
.subscribe(
"tasks".into(),
"tasks".into(),
vec![("project_id".into(), vec!["p1".into()])],
None,
)
.unwrap();
client.subs[0].cursor = 41;
client.subs[0].synced_once = true;
client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
client.persist_sub(&client.subs[0]).unwrap();
client
.mutate(vec![Mutation::Upsert {
table: "tasks".into(),
values: Map::from_iter([
("id".into(), Value::from("pending")),
("project_id".into(), Value::from("p1")),
]),
base_version: None,
}])
.unwrap();
let ids = client.pending_commit_ids();
let rows = client.query("SELECT * FROM tasks", &[]).unwrap();
let revision = client.local_revision();
client.drain_change_batches();
let (_, meta) = client.build_request(false);
client.conn.execute_batch(if fault == "subscription" {
"CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_subscriptions BEGIN SELECT RAISE(FAIL, 'injected subscription failure'); END"
} else {
"CREATE TRIGGER fail_control BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END"
}).unwrap();
let mut transport = CountingRealtimeTransport::default();
let mut report = SyncReport::default();
if status == SubStatus::Reset {
client.conn.execute("INSERT INTO _syncular_windows(base,unit,sub_id) VALUES ('base','p1','tasks')", []).unwrap();
}
let result = client.process_section(
&mut transport,
"tasks",
status,
"sync.scope_revoked",
vec![],
vec![],
Some((42, None)),
&meta,
&mut report,
);
assert!(result.is_err(), "{status:?}/{fault}");
assert!(client.conn.is_autocommit());
assert_eq!(client.subs[0].cursor, 41);
assert_eq!(client.subs[0].state, SubState::Active);
assert_eq!(client.pending_commit_ids(), ids);
assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
assert!(client.rejections.is_empty());
assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), rows);
assert_eq!(client.local_revision(), revision);
assert!(client.drain_change_batches().is_empty());
assert!(report.revoked.is_empty() && report.resets.is_empty());
}
}
}
#[test]
fn rows_segment_blocks_commit_independently_and_clear_with_the_first_block() {
let mut client = client();
client
.subscribe(
"tasks".into(),
"tasks".into(),
vec![("project_id".into(), vec!["p1".into()])],
None,
)
.unwrap();
client.subs[0].effective = Some(vec![("project_id".into(), vec!["p1".into()])]);
client.conn.execute_batch("INSERT INTO _syncular_base_tasks VALUES ('stale','p1',1); INSERT INTO tasks VALUES ('stale','p1',1);
CREATE TRIGGER fail_block BEFORE INSERT ON _syncular_base_tasks WHEN NEW.id = 'bad' BEGIN SELECT RAISE(FAIL,'injected later block failure'); END").unwrap();
let row = |id: &str| ssp2::segment::SegmentRow {
server_version: 1,
values: vec![
Some(ColumnValue::String(id.into())),
Some(ColumnValue::String("p1".into())),
],
};
let segment = RowsSegment {
table: "tasks".into(),
schema_version: 1,
columns: client.schema.table("tasks").unwrap().wire_columns.clone(),
blocks: vec![vec![row("first")], vec![row("second"), row("bad")]],
};
assert!(client.apply_segment(0, &segment, true).is_err());
assert!(client.conn.is_autocommit());
assert_eq!(
client
.query("SELECT id FROM tasks ORDER BY id", &[])
.unwrap(),
vec![Map::from_iter([("id".into(), Value::from("first"))])]
);
assert_eq!(client.local_revision(), 1);
assert_eq!(client.drain_change_batches().len(), 1);
assert_eq!(client.subs[0].cursor, -1);
client
.conn
.execute_batch("DROP TRIGGER fail_block")
.unwrap();
assert!(matches!(client.apply_segment(0, &segment, true), Ok(3)));
assert_eq!(client.local_revision(), 3);
assert_eq!(client.drain_change_batches().len(), 2);
}
#[test]
fn diagnostic_storage_reads_fresh_aggregates_after_writes_rollback_and_schema_changes() {
assert_eq!(
client().diagnostics_storage().blob_cache_bytes_approx,
Some(0)
);
let client = SyncClient::new(
"diagnostic-storage".to_owned(),
&json!({
"version": 1, "tables": [{"name": "attachments", "primaryKey": "id",
"columns": [{"name": "id", "type": "string", "nullable": false},
{"name": "body", "type": "blob_ref", "nullable": false}], "scopes": []}]
}),
ClientLimits {
blob_cache_max_bytes: Some(5),
..Default::default()
},
)
.unwrap();
let empty = client.diagnostics_storage();
assert_eq!(empty.status, "healthy");
assert_eq!(empty.pending_outbox_bytes_approx, Some(0));
assert_eq!(empty.retained_outcome_entries, Some(0));
assert_eq!(empty.retained_outcome_bytes_approx, Some(0));
assert_eq!(empty.blob_cache_bytes_approx, Some(0));
client.conn.execute_batch("INSERT INTO _syncular_outbox(commit_id, ops_json) VALUES ('one', '[]');
INSERT INTO _syncular_commit_outcomes(client_commit_id,status,recorded_at_ms,results_json,operations_json)
VALUES ('one','applied',0,'[]',NULL),('two','rejected',0,'[]','[]');
INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)
VALUES ('body',zeroblob(8),8,NULL,0);").unwrap();
let written = client.diagnostics_storage();
assert_eq!(written.status, "pressure");
assert_eq!(written.pending_outbox_bytes_approx, Some(2));
assert_eq!(written.retained_outcome_entries, Some(2));
assert_eq!(written.retained_outcome_bytes_approx, Some(6));
assert_eq!(written.blob_cache_bytes_approx, Some(8));
assert_eq!(
written.pressure_reason_code.as_deref(),
Some("client.blob_cache_over_limit")
);
client
.conn
.execute_batch(
"BEGIN; UPDATE _syncular_outbox SET ops_json = '[1,2]';
DELETE FROM _syncular_commit_outcomes; DELETE FROM _syncular_blobs;",
)
.unwrap();
let staged = client.diagnostics_storage();
assert_eq!(staged.pending_outbox_bytes_approx, Some(5));
assert_eq!(staged.retained_outcome_entries, Some(0));
assert_eq!(staged.blob_cache_bytes_approx, Some(0));
assert_eq!(staged.status, "healthy");
client.conn.execute_batch("ROLLBACK").unwrap();
assert_eq!(client.diagnostics_storage(), written);
client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN results_json TO hidden_results").unwrap();
let unreadable = client.diagnostics_storage();
assert_eq!(unreadable.status, "unreadable");
assert!(unreadable.database_bytes_approx.is_none());
assert!(unreadable.pending_outbox_bytes_approx.is_none());
assert!(unreadable.retained_outcome_entries.is_none());
assert!(unreadable.retained_outcome_bytes_approx.is_none());
assert!(unreadable.blob_cache_bytes_approx.is_none());
client.conn.execute_batch("ALTER TABLE _syncular_commit_outcomes RENAME COLUMN hidden_results TO results_json").unwrap();
let restored = client.diagnostics_storage();
assert_eq!(restored.status, written.status);
assert_eq!(
restored.retained_outcome_bytes_approx,
written.retained_outcome_bytes_approx
);
client.conn.set_prepared_statement_cache_capacity(0);
assert_eq!(client.diagnostics_storage(), restored);
}
struct ServerErrorTransport {
code: &'static str,
retryable: bool,
}
impl Transport for ServerErrorTransport {
fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
Ok(encode_message(&Message {
wire_version: WIRE_VERSION,
msg_kind: MsgKind::Response,
frames: vec![
Frame::RespHeader {
required_schema_version: None,
latest_schema_version: None,
log_epoch: Some("epoch-1".to_owned()),
reset_required: Some(false),
},
Frame::Error {
code: self.code.to_owned(),
message: "server error".to_owned(),
category: "internal".to_owned(),
retryable: self.retryable,
recommended_action: "retryLater".to_owned(),
details: None,
},
],
}))
}
fn realtime_sync(&mut self, request: &[u8]) -> Result<Vec<u8>, TransportError> {
self.sync(request)
}
fn download_segment(
&mut self,
_request: &SegmentRequest,
_on_progress: &mut dyn FnMut(u64),
) -> Result<Vec<u8>, TransportError> {
Err(TransportError::new("sync.transport_failed", "offline"))
}
fn realtime_connect(&mut self) -> Result<(), TransportError> {
Ok(())
}
fn realtime_send(&mut self, _text: &str) -> Result<(), TransportError> {
Ok(())
}
fn realtime_close(&mut self) -> Result<(), TransportError> {
Ok(())
}
}
#[test]
fn retryable_server_error_schedules_a_background_retry() {
let mut client = client();
client.set_meta(LOG_EPOCH_KEY, "epoch-1");
let mut transport = ServerErrorTransport {
code: "sync.internal_error",
retryable: true,
};
assert!(matches!(
client.sync(&mut transport),
SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.internal_error"
));
assert!(matches!(
client.drain_sync_intents().as_slice(),
[SyncIntent::Background { delay_ms: 250 }]
));
assert_eq!(
client.progress().snapshot().unwrap().retry_delay_ms,
Some(250)
);
let mut transport = ServerErrorTransport {
code: "sync.invalid_request",
retryable: false,
};
assert!(matches!(
client.sync(&mut transport),
SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.invalid_request"
));
assert!(client.drain_sync_intents().is_empty());
let progress = client.progress().snapshot().unwrap();
assert_eq!(progress.state, ProgressState::Failed);
assert_eq!(progress.retry_delay_ms, None);
}
#[test]
fn failed_progress_retry_delay_doubles_to_the_cap() {
let mut client = client();
client.set_meta(LOG_EPOCH_KEY, "epoch-1");
let mut transport = ServerErrorTransport {
code: "sync.internal_error",
retryable: true,
};
let delays: Vec<Option<u64>> = (0..9)
.map(|_| {
client.sync(&mut transport);
client.progress().snapshot().unwrap().retry_delay_ms
})
.collect();
assert_eq!(
delays,
[250, 500, 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000].map(Some)
);
}
#[test]
fn background_retry_deadlines_back_off_and_reset() {
let mut client = client();
client.schedule_background_retry();
assert!(matches!(
client.drain_sync_intents().as_slice(),
[SyncIntent::Background { delay_ms: 250 }]
));
client.schedule_background_retry();
assert!(matches!(
client.drain_sync_intents().as_slice(),
[SyncIntent::Background { delay_ms: 500 }]
));
client.reset_background_retry();
client.schedule_background_retry();
assert!(matches!(
client.drain_sync_intents().as_slice(),
[SyncIntent::Background { delay_ms: 250 }]
));
}
#[test]
fn log_epoch_reset_requests_an_immediate_follow_up_round() {
let mut client = client();
client
.subscribe(
"epoch-tasks".to_owned(),
"tasks".to_owned(),
vec![("project_id".to_owned(), vec!["p1".to_owned()])],
None,
)
.expect("subscribe");
client.drain_sync_intents();
client
.run_log_epoch_reset("epoch-2")
.expect("reset partition log epoch");
assert!(matches!(
client.drain_sync_intents().as_slice(),
[SyncIntent::Interactive]
));
}
#[derive(Default)]
struct CountingRealtimeTransport {
connects: usize,
closes: usize,
messages: Vec<String>,
blob_uploads: Vec<(String, Vec<u8>)>,
blob_download: Option<Vec<u8>>,
sync_calls: usize,
realtime_sync_calls: usize,
refuse_connect: bool,
}
impl Transport for CountingRealtimeTransport {
fn blob_upload(
&mut self,
blob_id: &str,
bytes: &[u8],
_media_type: Option<&str>,
) -> Result<(), TransportError> {
self.blob_uploads.push((blob_id.to_owned(), bytes.to_vec()));
Ok(())
}
fn blob_download(&mut self, _blob_id: &str) -> Result<BlobDownload, TransportError> {
self.blob_download
.clone()
.map(BlobDownload::Bytes)
.ok_or_else(|| TransportError::new("blob.not_found", "missing blob"))
}
fn sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
self.sync_calls += 1;
Err(TransportError::new("sync.transport_failed", "offline"))
}
fn realtime_sync(&mut self, _request: &[u8]) -> Result<Vec<u8>, TransportError> {
self.realtime_sync_calls += 1;
Err(TransportError::new("sync.transport_failed", "offline"))
}
fn download_segment(
&mut self,
_request: &SegmentRequest,
_on_progress: &mut dyn FnMut(u64),
) -> Result<Vec<u8>, TransportError> {
Err(TransportError::new("sync.transport_failed", "offline"))
}
fn realtime_connect(&mut self) -> Result<(), TransportError> {
self.connects += 1;
if self.refuse_connect {
return Err(TransportError::new("transport.failed", "refused"));
}
Ok(())
}
fn realtime_send(&mut self, text: &str) -> Result<(), TransportError> {
self.messages.push(text.to_owned());
Ok(())
}
fn realtime_close(&mut self) -> Result<(), TransportError> {
self.closes += 1;
Ok(())
}
}
#[test]
fn blob_results_own_downloaded_bytes() {
let schema = json!({"version":1,"tables":[{"name":"attachments","primaryKey":"id","columns":[
{"name":"id","type":"string","nullable":false},
{"name":"file","type":"blob_ref","nullable":true}],"scopes":[]}]});
let bytes = b"owned typed result".to_vec();
let blob_id = blob_id_for(&bytes);
let mut transport = CountingRealtimeTransport {
blob_download: Some(bytes.clone()),
..Default::default()
};
let result = {
let mut client =
SyncClient::new("owned-result".to_owned(), &schema, ClientLimits::default())
.unwrap();
let result = client.fetch_blob_bytes(&mut transport, &blob_id).unwrap();
assert_eq!(result.blob_id, blob_id);
assert_eq!(result.byte_length, bytes.len() as i64);
assert_eq!(result.bytes, bytes);
assert_eq!(result.media_type, None);
assert_eq!(
client
.conn
.query_row(
"SELECT count(*) FROM _syncular_blob_uploads WHERE blob_id = ?",
rusqlite::params![blob_id],
|row| row.get::<_, i64>(0),
)
.unwrap(),
0
);
result
};
assert_eq!(result.bytes, bytes);
let mut client =
SyncClient::new("blob-errors".to_owned(), &schema, ClientLimits::default()).unwrap();
assert_eq!(
client
.fetch_blob_bytes(&mut transport, "not json")
.unwrap_err(),
(
"client.failed".to_owned(),
"blob ref is not JSON".to_owned()
)
);
}
#[test]
fn visible_blob_references_survive_cache_trimming() {
let schema = json!({"version":1,"tables":[
{"name":"attachments","primaryKey":"id","columns":[
{"name":"id","type":"string","nullable":false},
{"name":"body","type":"blob_ref","nullable":true},
{"name":"preview","type":"blob_ref","nullable":true}],"scopes":[]},
{"name":"avatars","primaryKey":"id","columns":[
{"name":"id","type":"string","nullable":false},
{"name":"photo","type":"blob_ref","nullable":true}],"scopes":[]}
]});
let bytes = b"referenced above cap".to_vec();
let blob_id = blob_id_for(&bytes);
let reference = json!({"blobId": blob_id, "byteLength": bytes.len()}).to_string();
let mut client = SyncClient::new(
"visible-blob-refs".to_owned(),
&schema,
ClientLimits {
blob_cache_max_bytes: Some(1),
..Default::default()
},
)
.unwrap();
client.conn.execute(
"INSERT INTO attachments VALUES ('one', ?, ?, 0), ('two', ?, 'malformed', 0), ('three', 12, ?, 0)",
rusqlite::params![reference, reference, reference, reference],
).unwrap();
client
.conn
.execute(
"INSERT INTO avatars VALUES ('one', ?, 0), ('two', '{\"blobId\":12}', 0)",
rusqlite::params![reference],
)
.unwrap();
client.conn.execute(
"INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES ('sha256:other', X'02', 1, NULL, 0)",
[],
).unwrap();
let mut transport = CountingRealtimeTransport {
blob_download: Some(bytes.clone()),
..Default::default()
};
assert_eq!(
client
.fetch_blob_bytes(&mut transport, &blob_id)
.unwrap()
.bytes,
bytes
);
let rows = client
.conn
.prepare("SELECT blob_id FROM _syncular_blobs ORDER BY blob_id")
.unwrap()
.query_map([], |row| row.get::<_, String>(0))
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(rows, vec![blob_id]);
}
#[test]
fn realtime_connection_ownership_is_idempotent() {
let mut client = client();
let mut transport = CountingRealtimeTransport {
connects: 0,
closes: 0,
..Default::default()
};
client
.connect_realtime(&mut transport)
.expect("first connect");
client
.connect_realtime(&mut transport)
.expect("idempotent connect");
assert_eq!(transport.connects, 1);
client.disconnect_realtime(&mut transport);
client.disconnect_realtime(&mut transport);
assert_eq!(transport.closes, 1);
client
.connect_realtime(&mut transport)
.expect("deliberate reconnect");
assert_eq!(transport.connects, 2);
}
#[test]
fn required_policy_refuses_a_round_without_a_socket_and_never_uses_http() {
let mut client = client();
client
.set_realtime_policy(RealtimePolicy::Required)
.expect("set policy");
let mut transport = CountingRealtimeTransport::default();
match client.sync(&mut transport) {
SyncOutcome::RealtimeUnavailable {
state,
reason_code,
retry_delay_ms,
} => {
assert_eq!(state, RealtimeState::Disconnected);
assert_eq!(reason_code, None);
assert_eq!(retry_delay_ms, 250);
}
other => panic!("expected RealtimeUnavailable, got {other:?}"),
}
assert_eq!(transport.sync_calls, 0);
assert!(matches!(
client.drain_sync_intents().as_slice(),
[SyncIntent::Background { delay_ms: 250 }]
));
let snapshot = client
.diagnostics_snapshot(&ClientDiagnosticsRequest::default())
.expect("diagnostics");
assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Required);
assert_eq!(snapshot.host.realtime_reason_code, None);
assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
assert_eq!(
snapshot
.last_round
.as_ref()
.and_then(|round| round.error_code.clone()),
Some(REALTIME_UNAVAILABLE_CODE.to_owned())
);
}
#[test]
fn refused_handshake_is_a_visible_state_and_a_required_round_does_not_fall_back() {
let mut client = client();
client
.set_realtime_policy(RealtimePolicy::Required)
.expect("set policy");
let mut transport = CountingRealtimeTransport {
refuse_connect: true,
..Default::default()
};
assert!(client.connect_realtime(&mut transport).is_err());
assert_eq!(client.realtime_state(), RealtimeState::Refused);
let snapshot = client
.diagnostics_snapshot(&ClientDiagnosticsRequest::default())
.expect("diagnostics");
assert_eq!(
snapshot.host.realtime_reason_code.as_deref(),
Some("transport.failed")
);
match client.sync(&mut transport) {
SyncOutcome::RealtimeUnavailable {
state, reason_code, ..
} => {
assert_eq!(state, RealtimeState::Refused);
assert_eq!(reason_code.as_deref(), Some("transport.failed"));
}
other => panic!("expected RealtimeUnavailable, got {other:?}"),
}
assert_eq!(transport.sync_calls, 0);
assert_eq!(transport.realtime_sync_calls, 0);
}
#[test]
fn a_failed_socket_round_is_lost_and_the_next_required_round_refuses_http() {
let mut client = client();
client
.set_realtime_policy(RealtimePolicy::Required)
.expect("set policy");
let mut transport = CountingRealtimeTransport::default();
client.connect_realtime(&mut transport).expect("connect");
assert_eq!(client.realtime_state(), RealtimeState::Connected);
assert!(
client.set_realtime_policy(RealtimePolicy::Off).is_err(),
"a policy change cannot silently re-route a connected socket"
);
assert!(matches!(
client.sync(&mut transport),
SyncOutcome::Failed { ref error_code, .. } if error_code == "sync.transport_failed"
));
assert_eq!(client.realtime_state(), RealtimeState::Lost);
assert_eq!(transport.realtime_sync_calls, 1);
assert_eq!(transport.sync_calls, 0);
let snapshot = client
.diagnostics_snapshot(&ClientDiagnosticsRequest::default())
.expect("diagnostics");
assert_eq!(
snapshot.host.realtime_reason_code.as_deref(),
Some("sync.transport_failed")
);
assert_eq!(snapshot.host.realtime_retry_delay_ms, Some(250));
match client.sync(&mut transport) {
SyncOutcome::RealtimeUnavailable {
state, reason_code, ..
} => {
assert_eq!(state, RealtimeState::Lost);
assert_eq!(reason_code.as_deref(), Some("sync.transport_failed"));
}
other => panic!("expected RealtimeUnavailable, got {other:?}"),
}
assert_eq!(transport.sync_calls, 0);
client.disconnect_realtime(&mut transport);
assert_eq!(client.realtime_state(), RealtimeState::Disconnected);
assert_eq!(transport.closes, 1);
client.connect_realtime(&mut transport).expect("reconnect");
assert_eq!(client.realtime_state(), RealtimeState::Connected);
assert_eq!(transport.connects, 2);
}
#[test]
fn off_policy_uses_http_and_refuses_an_explicit_connect() {
let mut client = client();
client
.set_realtime_policy(RealtimePolicy::Off)
.expect("set policy");
let mut transport = CountingRealtimeTransport::default();
assert!(client.connect_realtime(&mut transport).is_err());
assert_eq!(client.realtime_state(), RealtimeState::Disabled);
assert!(matches!(
client.sync(&mut transport),
SyncOutcome::Failed { .. }
));
assert_eq!(transport.sync_calls, 1);
assert_eq!(transport.realtime_sync_calls, 0);
let snapshot = client
.diagnostics_snapshot(&ClientDiagnosticsRequest::default())
.expect("diagnostics");
assert_eq!(snapshot.host.realtime, RealtimeState::Disabled);
assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Off);
}
#[test]
fn optional_policy_keeps_the_http_round() {
let mut client = client();
let mut transport = CountingRealtimeTransport::default();
assert!(matches!(
client.sync(&mut transport),
SyncOutcome::Failed { .. }
));
assert_eq!(transport.sync_calls, 1);
assert_eq!(transport.realtime_sync_calls, 0);
let snapshot = client
.diagnostics_snapshot(&ClientDiagnosticsRequest::default())
.expect("diagnostics");
assert_eq!(snapshot.host.realtime, RealtimeState::Disconnected);
assert_eq!(snapshot.host.realtime_policy, RealtimePolicy::Optional);
}
#[test]
fn local_rebootstrap_is_atomic_idempotent_and_preserves_offline_work() {
let mut client = client();
client
.subscribe(
"repair-tasks".to_owned(),
"tasks".to_owned(),
vec![("project_id".to_owned(), vec!["p1".to_owned()])],
None,
)
.expect("subscribe");
{
let sub = client
.subs
.iter_mut()
.find(|sub| sub.id == "repair-tasks")
.expect("subscription");
sub.cursor = 42;
sub.synced_once = true;
let persisted = sub.clone();
client.persist_sub(&persisted).unwrap();
}
for table in ["_syncular_base_tasks", "tasks"] {
client
.conn
.execute(
&format!(
"INSERT INTO {table}(id, project_id, _syncular_version) VALUES (?1, ?2, 1)"
),
rusqlite::params!["server-row", "p1"],
)
.expect("seed server row");
}
let pending = client
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from("offline-row")),
("project_id".to_owned(), Value::from("p1")),
]),
base_version: None,
}])
.expect("queue offline work");
client.drain_change_batches();
client.drain_sync_intents();
assert_eq!(
client
.rebootstrap_local_data(&LocalDataRebootstrapInput {
rebootstrap_id: "support-case-001".to_owned(),
})
.expect("rebootstrap"),
LocalDataRebootstrapResult {
already_applied: false,
retained_commits: 1,
reset_subscriptions: 1,
}
);
let visible_ids = client
.conn
.prepare("SELECT id FROM tasks ORDER BY id")
.expect("prepare visible ids")
.query_map([], |row| row.get::<_, String>(0))
.expect("query visible ids")
.collect::<Result<Vec<_>, _>>()
.expect("collect visible ids");
assert_eq!(visible_ids, vec!["offline-row"]);
assert_eq!(client.pending_commit_ids(), vec![pending]);
assert_eq!(
client
.subscription_state("repair-tasks")
.expect("subscription")
.cursor,
-1
);
assert!(client.upgrading());
assert!(client.sync_needed());
assert!(matches!(
client.drain_sync_intents().as_slice(),
[SyncIntent::Interactive]
));
assert_eq!(client.drain_change_batches().len(), 1);
assert_eq!(
client
.rebootstrap_local_data(&LocalDataRebootstrapInput {
rebootstrap_id: "support-case-001".to_owned(),
})
.expect("idempotent retry"),
LocalDataRebootstrapResult {
already_applied: true,
retained_commits: 1,
reset_subscriptions: 1,
}
);
assert!(client.drain_change_batches().is_empty());
}
#[test]
fn local_rebootstrap_receipt_codec_is_bounded_and_legacy_compatible() {
let encoded = encode_local_rebootstrap_receipt(3, 4).expect("encode receipt");
assert_eq!(
decode_local_rebootstrap_receipt(&encoded).expect("decode receipt"),
(3, 4)
);
assert_eq!(
decode_local_rebootstrap_receipt("v1").expect("legacy marker"),
(0, 0)
);
for malformed in [
"",
"{}",
r#"{"version":3,"retainedCommits":1,"resetSubscriptions":1}"#,
r#"{"version":2,"retainedCommits":1,"resetSubscriptions":1,"extra":true}"#,
r#"{"version":2,"retainedCommits":9007199254740992,"resetSubscriptions":1}"#,
] {
assert_eq!(
decode_local_rebootstrap_receipt(malformed)
.expect_err("malformed receipt must fail"),
"sync.local_corrupt: persisted local rebootstrap receipt is invalid"
);
}
}
#[test]
fn local_rebootstrap_replays_the_original_receipt_after_reopen() {
let path = std::env::temp_dir().join(format!(
"syncular-rebootstrap-receipt-{}.db",
uuid::Uuid::new_v4()
));
let schema = json!({
"version": 1,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}]
});
let path_string = path.to_str().expect("UTF-8 temp path");
{
let mut first = SyncClient::open_path(
"repair-restart-client".to_owned(),
&schema,
ClientLimits::default(),
path_string,
)
.expect("first open");
first
.subscribe(
"repair-tasks".to_owned(),
"tasks".to_owned(),
vec![("project_id".to_owned(), vec!["p1".to_owned()])],
None,
)
.expect("subscribe");
first
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from("offline-row")),
("project_id".to_owned(), Value::from("p1")),
]),
base_version: None,
}])
.expect("queue offline work");
assert_eq!(
first
.rebootstrap_local_data(&LocalDataRebootstrapInput {
rebootstrap_id: "restart-receipt".to_owned(),
})
.expect("first rebootstrap"),
LocalDataRebootstrapResult {
already_applied: false,
retained_commits: 1,
reset_subscriptions: 1,
}
);
}
let mut reopened = SyncClient::open_path(
"repair-restart-client".to_owned(),
&schema,
ClientLimits::default(),
path_string,
)
.expect("reopen");
reopened.drain_change_batches();
reopened.drain_sync_intents();
assert_eq!(
reopened
.rebootstrap_local_data(&LocalDataRebootstrapInput {
rebootstrap_id: "restart-receipt".to_owned(),
})
.expect("receipt replay"),
LocalDataRebootstrapResult {
already_applied: true,
retained_commits: 1,
reset_subscriptions: 1,
}
);
assert!(reopened.drain_change_batches().is_empty());
assert!(reopened.drain_sync_intents().is_empty());
drop(reopened);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn local_rebootstrap_fails_closed_on_malformed_or_unreadable_receipts() {
let mut malformed = client();
malformed
.subscribe(
"repair-tasks".to_owned(),
"tasks".to_owned(),
vec![("project_id".to_owned(), vec!["p1".to_owned()])],
None,
)
.expect("subscribe");
malformed.set_meta("localRebootstrap:malformed", "{\"version\":2}");
malformed.drain_change_batches();
malformed.drain_sync_intents();
let malformed_error = malformed
.rebootstrap_local_data(&LocalDataRebootstrapInput {
rebootstrap_id: "malformed".to_owned(),
})
.expect_err("malformed receipt must fail closed");
assert_eq!(
malformed_error,
"sync.local_corrupt: persisted local rebootstrap receipt is invalid"
);
assert!(!malformed.upgrading());
assert_eq!(
malformed
.subscription_state("repair-tasks")
.expect("unchanged subscription")
.cursor,
-1
);
assert!(malformed.drain_change_batches().is_empty());
assert!(malformed.drain_sync_intents().is_empty());
let mut unreadable = client();
unreadable
.conn
.execute(
"INSERT INTO tasks(id, project_id, _syncular_version) VALUES (?1, ?2, 1)",
rusqlite::params!["server-row", "p1"],
)
.expect("seed visible row");
unreadable
.conn
.execute("DROP TABLE _syncular_meta", [])
.expect("break marker storage");
let unreadable_error = unreadable
.rebootstrap_local_data(&LocalDataRebootstrapInput {
rebootstrap_id: "unreadable".to_owned(),
})
.expect_err("unreadable marker storage must fail closed");
assert_eq!(
unreadable_error,
"sync.local_corrupt: persisted local rebootstrap receipt is unreadable"
);
let visible_rows = unreadable
.conn
.query_row("SELECT COUNT(*) FROM tasks", [], |row| row.get::<_, i64>(0))
.expect("visible projection remains");
assert_eq!(visible_rows, 1);
assert!(!unreadable.upgrading());
assert!(unreadable.drain_change_batches().is_empty());
assert!(unreadable.drain_sync_intents().is_empty());
}
#[test]
fn local_rebootstrap_cannot_bypass_schema_floor() {
let mut client = client();
client.set_schema_floor(Some(SchemaFloor {
required_schema_version: Some(2),
latest_schema_version: Some(2),
}));
let error = client
.rebootstrap_local_data(&LocalDataRebootstrapInput {
rebootstrap_id: "blocked-floor".to_owned(),
})
.expect_err("schema floor must block repair");
assert!(error.contains("cannot bypass an active schema-floor stop"));
}
#[test]
fn batched_push_acknowledgements_rebuild_overlay_once_per_response() {
let mut client = client();
client.set_meta(LOG_EPOCH_KEY, "epoch-1");
const COMMIT_COUNT: usize = 32;
for index in 0..COMMIT_COUNT {
client
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from(format!("task-{index}"))),
("project_id".to_owned(), Value::from("project-1")),
]),
base_version: None,
}])
.expect("queue commit");
}
let (_, request_meta) = client.build_request(false);
assert_eq!(request_meta.pushed_ids.len(), COMMIT_COUNT);
let mut frames = vec![Frame::RespHeader {
required_schema_version: None,
latest_schema_version: None,
log_epoch: Some("epoch-1".to_owned()),
reset_required: Some(false),
}];
frames.extend(request_meta.pushed_ids.iter().enumerate().map(
|(index, client_commit_id)| Frame::PushResult {
client_commit_id: client_commit_id.clone(),
status: PushStatus::Applied,
commit_seq: Some(index as i64 + 1),
results: vec![OpResult::Applied { op_index: 0 }],
},
));
let response = Message {
wire_version: WIRE_VERSION,
msg_kind: MsgKind::Response,
frames,
};
let mut transport =
HostTransport::new_from_config(&json!({})).expect("no-network host transport");
client.overlay_rebuild_count.set(0);
client.outcome_prune_count.set(0);
let outcome = client.process_response(&mut transport, response, &request_meta);
assert!(matches!(outcome, SyncOutcome::Ok(_)));
assert!(client.pending_commit_ids().is_empty());
assert_eq!(
client.overlay_rebuild_count.get(),
1,
"one response must reconcile its acknowledged commits with one overlay rebuild"
);
assert_eq!(
client.outcome_prune_count.get(),
1,
"one response must enforce outcome retention once"
);
}
#[test]
fn clean_commit_mirroring_matches_full_replay_with_fts_and_unique_indexes() {
let schema = json!({ "version": 1, "tables": [{ "name": "tasks", "primaryKey": "id",
"columns": [
{"name":"id", "type":"string", "nullable":false},
{"name":"project_id", "type":"string", "nullable":false},
{"name":"title", "type":"string", "nullable":false}
], "scopes": [{"pattern":"project:{project_id}"}],
"indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
"ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]
}] });
let mut incremental =
SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
let mut reference =
SyncClient::new("mirror".to_owned(), &schema, ClientLimits::default()).unwrap();
let tables = vec!["tasks".to_owned()];
for index in 0..32 {
let id = format!("task-{}", index % 8);
let values = Map::from_iter([
("id".to_owned(), Value::from(id.clone())),
(
"project_id".to_owned(),
Value::from(format!("p{}", index % 2)),
),
("title".to_owned(), Value::from(format!("needle{index}"))),
]);
let payload = encode_row_json(
incremental.schema.table("tasks").unwrap(),
&id,
&values,
&incremental.encryption,
)
.unwrap();
let change = ssp2::model::Change {
table_index: 0,
row_id: id,
op: if index % 5 == 0 {
Op::Delete
} else {
Op::Upsert
},
row_version: Some(index + 1),
scopes: vec![("project_id".to_owned(), format!("p{}", index % 2))],
row: Some(payload),
};
reference.overlay_dirty.set(true);
for instance in [&mut incremental, &mut reference] {
instance.begin_observation("mirror_test").unwrap();
let mut batch = ChangeAccumulator::default();
instance.record_commit_changes(&mut batch, &tables, std::slice::from_ref(&change));
instance
.apply_commit_changes(&tables, std::slice::from_ref(&change))
.unwrap();
instance.rebuild_overlay_if_dirty();
instance.finish_observation("mirror_test", batch).unwrap();
}
for sql in [
"SELECT * FROM tasks ORDER BY id",
"SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
] {
assert_eq!(
incremental.query(sql, &[]).unwrap(),
reference.query(sql, &[]).unwrap()
);
}
assert_eq!(incremental.local_revision(), reference.local_revision());
assert_eq!(
serde_json::to_value(incremental.drain_change_batches()).unwrap(),
serde_json::to_value(reference.drain_change_batches()).unwrap()
);
}
assert_eq!(incremental.overlay_rebuild_count.get(), 0);
assert_eq!(reference.overlay_rebuild_count.get(), 32);
}
#[test]
fn row_id_lookups_preserve_text_matching_and_seek_lossless_keys() {
let cases = [
(
"string",
vec![
json!("1"),
json!("01"),
json!(""),
json!("nul\u{0}id"),
json!("é"),
],
),
(
"integer",
vec![
json!(i64::MIN),
json!(-1),
json!(0),
json!(1),
json!(i64::MAX),
],
),
("boolean", vec![json!(false), json!(true)]),
("json", vec![json!("null"), json!("1"), json!("{\"a\":1}")]),
];
for (kind, ids) in cases {
let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
"columns":[{"name":"id","type":kind,"nullable":false},
{"name":"project_id","type":"string","nullable":false}],
"scopes":[{"pattern":"project:{project_id}"}]}]});
let mut client =
SyncClient::new("row-id-lookup".into(), &schema, ClientLimits::default()).unwrap();
for (index, id) in ids.iter().enumerate() {
let table = client.schema.table("tasks").unwrap();
client
.write_base_row(
"tasks",
&vec![
json_to_column_value(&table.columns[0], Some(id)).unwrap(),
Some(ColumnValue::String(format!("p{index}"))),
],
1,
)
.unwrap();
}
client.rebuild_overlay_if_dirty();
for base in [true, false] {
let full = if base {
base_table("tasks")
} else {
visible_table("tasks")
};
let predicate = row_id_predicate(client.schema.table("tasks").unwrap());
for row_id in [
"1",
"01",
"1.0",
"1.0000000000000002",
"+1",
"1e0",
" 1",
"-1",
"0",
"9223372036854775807",
"-9223372036854775808",
"true",
"false",
"",
"nul\u{0}id",
"é",
"null",
"{\"a\":1}",
] {
let old_sql =
format!("SELECT project_id FROM {full} WHERE CAST(id AS TEXT) = ?1");
let new_sql = format!("SELECT project_id FROM {full} WHERE {predicate}");
let old = client
.conn
.prepare(&old_sql)
.unwrap()
.query_map([row_id], |row| row.get::<_, String>(0))
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap();
let new = client
.conn
.prepare_cached(&new_sql)
.unwrap()
.query_map([row_id], |row| row.get::<_, String>(0))
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(old, new, "{kind}, base={base}, row_id={row_id:?}");
let mut batch = ChangeAccumulator::default();
assert_eq!(
client.record_row_scopes(&mut batch, "tasks", row_id, base),
!old.is_empty()
);
let expected = old.first().map(|scope| {
BTreeMap::from([(
"tasks".to_owned(),
Some(BTreeSet::from([format!("project:{scope}")])),
)])
});
assert_eq!(batch.tables, expected.unwrap_or_default());
client
.conn
.execute_batch("SAVEPOINT lookup_delete")
.unwrap();
if base {
client.delete_base_row("tasks", row_id).unwrap();
} else {
client.apply_outbox_ops(&[OutboxOp {
upsert: false,
table: "tasks".into(),
row_id: row_id.into(),
base_version: None,
values: None,
}]);
}
let remaining: i64 = client
.conn
.query_row(&format!("SELECT count(*) FROM {full}"), [], |row| {
row.get(0)
})
.unwrap();
assert_eq!(remaining, ids.len() as i64 - old.len() as i64);
client
.conn
.execute_batch("ROLLBACK TO lookup_delete; RELEASE lookup_delete")
.unwrap();
client.overlay_dirty.set(false);
}
for (predicate, expected) in [
("CAST(id AS TEXT) = ?1".to_owned(), "SCAN"),
(predicate, "SEARCH"),
] {
let sql = format!("EXPLAIN QUERY PLAN SELECT project_id FROM {full} WHERE {predicate} LIMIT 1");
let details = client
.conn
.prepare(&sql)
.unwrap()
.query_map(["1"], |row| row.get::<_, String>(3))
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert!(
details.iter().any(|detail| detail.contains(expected)),
"{kind}: {details:?}"
);
}
}
}
}
#[test]
fn primary_keys_reject_every_ineligible_column_type() {
for ty in ["float", "bytes", "crdt", "blob_ref"] {
let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
"columns":[{"name":"id","type":ty,"nullable":false},
{"name":"project_id","type":"string","nullable":false}],
"scopes":[{"pattern":"project:{project_id}"}]}]});
let error =
match SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default()) {
Ok(_) => panic!("ineligible primary key {ty} must be rejected"),
Err(error) => error,
};
assert!(
error.contains("has an ineligible column type (§2.4)"),
"{ty}: {error}"
);
}
for ty in ["string", "integer", "boolean", "json"] {
let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id",
"columns":[{"name":"id","type":ty,"nullable":false},
{"name":"project_id","type":"string","nullable":false}],
"scopes":[{"pattern":"project:{project_id}"}]}]});
SyncClient::new("pk-eligibility".into(), &schema, ClientLimits::default())
.unwrap_or_else(|error| panic!("{ty}: {error}"));
}
}
#[test]
fn pending_row_reconciliation_matches_full_fifo_replay() {
for unique in [false, true] {
let schema = json!({"version": 1, "tables": (["tasks", "notes"].map(|name| json!({
"name": name, "primaryKey": "id", "columns": [
{"name":"id", "type":"string", "nullable":false},
{"name":"project_id", "type":"string", "nullable":false},
{"name":"title", "type":"string", "nullable":false}],
"scopes": [{"pattern":"project:{project_id}"}],
"indexes": [{"name":format!("{name}_title"), "columns":["title"], "unique":unique || name == "notes"}],
"ftsIndexes": [{"name":format!("{name}_fts"), "columns":["title"], "tokenize":"unicode61"}]
})))});
let mut incremental =
SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
let mut reference =
SyncClient::new("pending".into(), &schema, ClientLimits::default()).unwrap();
let tables = vec!["tasks".to_owned(), "notes".to_owned()];
for instance in [&mut incremental, &mut reference] {
for table in &tables {
for index in 0..12 {
instance
.write_base_row(
table,
&vec![
Some(ColumnValue::String(format!("row{index}"))),
Some(ColumnValue::String("p0".into())),
Some(ColumnValue::String(format!("base{index}"))),
],
1,
)
.unwrap();
}
}
instance.rebuild_overlay_if_dirty();
}
for index in 0..80 {
let mutations = (0..3)
.map(|offset| {
let table = tables[(index + offset) % 2].clone();
let id = format!("row{}", (index + offset) % 16);
if (index + offset) % 7 == 0 {
Mutation::Delete {
table,
row_id: id,
base_version: None,
}
} else {
Mutation::Upsert {
table,
values: Map::from_iter([
("id".into(), Value::from(id)),
("project_id".into(), Value::from(format!("p{}", index % 3))),
(
"title".into(),
Value::from(format!("needle{}", (index + offset) % 16)),
),
]),
base_version: None,
}
}
})
.collect::<Vec<_>>();
for instance in [&mut incremental, &mut reference] {
instance.mutate(mutations.clone()).unwrap();
}
let changes = (0..3)
.map(|offset| {
let table_index = if index % 4 == 0 { offset % 2 } else { 0 };
let id = format!("row{}", (index + offset) % 16);
let values = Map::from_iter([
("id".into(), Value::from(id.clone())),
(
"project_id".into(),
Value::from(format!("p{}", (index + 1) % 3)),
),
(
"title".into(),
Value::from(format!("remote{index}_{offset}")),
),
]);
ssp2::model::Change {
table_index: table_index as u16,
row_id: id.clone(),
op: if (index + offset) % 5 == 0 {
Op::Delete
} else {
Op::Upsert
},
row_version: Some(index as i64 + 2),
scopes: vec![],
row: Some(
encode_row_json(
incremental.schema.table(&tables[table_index]).unwrap(),
&id,
&values,
&incremental.encryption,
)
.unwrap(),
),
}
})
.collect::<Vec<_>>();
reference.overlay_dirty.set(true);
incremental.apply_commit_frame(&tables, &changes).unwrap();
reference.apply_commit_frame(&tables, &changes).unwrap();
for table in &tables {
for sql in [format!("SELECT * FROM {table} ORDER BY id"),
format!("SELECT * FROM _syncular_base_{table} ORDER BY id"),
format!("SELECT _syncular_source_id, title FROM {table}_fts ORDER BY _syncular_source_id"),
format!("SELECT _syncular_source_id FROM {table}_fts WHERE {table}_fts MATCH 'needle1' ORDER BY _syncular_source_id")] {
assert_eq!(incremental.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "unique={unique}, frame={index}: {sql}");
}
}
assert_eq!(incremental.local_revision(), reference.local_revision());
assert_eq!(
serde_json::to_value(incremental.drain_change_batches()).unwrap(),
serde_json::to_value(reference.drain_change_batches()).unwrap()
);
}
assert_eq!(
incremental.overlay_rebuild_count.get(),
if unique { 81 } else { 21 }
);
assert_eq!(reference.overlay_rebuild_count.get(), 81);
}
}
#[test]
fn remote_unique_values_reconsider_pending_writes_on_other_rows() {
let schema = json!({"version": 1, "tables": [{"name":"tasks", "primaryKey":"id",
"columns":[{"name":"id", "type":"string", "nullable":false},
{"name":"title", "type":"string", "nullable":false}],
"scopes":[], "indexes":[{"name":"unique_title", "columns":["title"], "unique":true}]}]});
let mut client =
SyncClient::new("unique-pending".into(), &schema, ClientLimits::default()).unwrap();
for (id, title) in [("a", "original"), ("b", "occupied")] {
client
.write_base_row(
"tasks",
&vec![
Some(ColumnValue::String(id.into())),
Some(ColumnValue::String(title.into())),
],
1,
)
.unwrap();
}
client.rebuild_overlay_if_dirty();
let pending = client
.mutate(vec![Mutation::Upsert {
table: "tasks".into(),
values: Map::from_iter([
("id".into(), Value::from("a")),
("title".into(), Value::from("available")),
]),
base_version: None,
}])
.unwrap();
assert_eq!(
client
.conn
.query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
.get::<_, String>(
0
))
.unwrap(),
"available"
);
for (title, expected) in [("available", "original"), ("occupied", "available")] {
let change = ssp2::model::Change {
table_index: 0,
row_id: "b".into(),
op: Op::Upsert,
row_version: Some(2),
scopes: vec![],
row: Some(
encode_row_json(
client.schema.table("tasks").unwrap(),
"b",
&Map::from_iter([
("id".into(), Value::from("b")),
("title".into(), Value::from(title)),
]),
&client.encryption,
)
.unwrap(),
),
};
client
.apply_commit_frame(&["tasks".into()], &[change])
.unwrap();
assert_eq!(
client
.conn
.query_row("SELECT title FROM tasks WHERE id = 'a'", [], |row| row
.get::<_, String>(
0
))
.unwrap(),
expected
);
assert_eq!(client.pending_commit_ids(), vec![pending.clone()]);
}
assert_eq!(client.overlay_rebuild_count.get(), 3);
}
#[test]
fn pending_row_reconciliation_failure_preserves_queue_and_frame_state() {
for failure in ["visible", "revision", "commit"] {
let mut client = client();
let id = client
.mutate(vec![Mutation::Upsert {
table: "tasks".into(),
values: Map::from_iter([
("id".into(), Value::from("t1")),
("project_id".into(), Value::from("pending")),
]),
base_version: None,
}])
.unwrap();
client.drain_change_batches();
let revision = client.local_revision();
let before = client.query("SELECT * FROM tasks", &[]).unwrap();
let trigger = match failure {
"visible" => "CREATE TRIGGER fail_pending BEFORE INSERT ON tasks WHEN NEW.project_id = 'remote' BEGIN SELECT RAISE(ABORT, 'injected visible failure'); END".to_owned(),
"revision" => format!("CREATE TRIGGER fail_pending BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(ABORT, 'injected revision failure'); END"),
_ => "PRAGMA foreign_keys = ON; CREATE TABLE pending_parent(id INTEGER PRIMARY KEY); CREATE TABLE pending_child(id INTEGER REFERENCES pending_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_pending AFTER INSERT ON _syncular_base_tasks BEGIN INSERT INTO pending_child VALUES(1); END".to_owned(),
};
client.conn.execute_batch(&trigger).unwrap();
let change = ssp2::model::Change {
table_index: 0,
row_id: "t1".into(),
op: Op::Upsert,
row_version: Some(2),
scopes: vec![],
row: Some(
encode_row_json(
client.schema.table("tasks").unwrap(),
"t1",
&Map::from_iter([
("id".into(), Value::from("t1")),
("project_id".into(), Value::from("remote")),
]),
&client.encryption,
)
.unwrap(),
),
};
assert!(client
.apply_commit_frame(&["tasks".into()], std::slice::from_ref(&change))
.is_err());
assert!(client.conn.is_autocommit());
assert!(!client.overlay_dirty.get());
assert_eq!(client.local_revision(), revision);
assert!(client.drain_change_batches().is_empty());
assert_eq!(client.pending_commit_ids(), vec![id.clone()]);
assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap(), before);
assert!(client
.query("SELECT * FROM _syncular_base_tasks", &[])
.unwrap()
.is_empty());
client
.conn
.execute_batch("DROP TRIGGER fail_pending")
.unwrap();
client
.apply_commit_frame(&["tasks".into()], &[change])
.unwrap();
assert_eq!(client.local_revision(), revision + 1);
assert_eq!(client.pending_commit_ids(), vec![id]);
let mut expected = before[0].clone();
expected.insert("_syncular_version".to_owned(), Value::from(2));
assert_eq!(
client.query("SELECT * FROM tasks", &[]).unwrap(),
vec![expected]
);
assert_eq!(
client
.conn
.query_row("SELECT project_id FROM _syncular_base_tasks", [], |row| row
.get::<_, String>(0))
.unwrap(),
"remote"
);
}
}
#[test]
fn appended_overlay_matches_full_fifo_replay_with_constraints_and_fts() {
let schema = json!({"version": 1, "tables": [{"name": "tasks", "primaryKey": "id",
"columns": [
{"name":"id", "type":"string", "nullable":false},
{"name":"project_id", "type":"string", "nullable":false},
{"name":"title", "type":"string", "nullable":false}],
"scopes": [{"pattern":"project:{project_id}"}],
"indexes": [{"name":"unique_title", "columns":["title"], "unique":true}],
"ftsIndexes": [{"name":"tasks_fts", "columns":["title"], "tokenize":"unicode61"}]}]});
let mut incremental =
SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
let mut reference =
SyncClient::new("append".to_owned(), &schema, ClientLimits::default()).unwrap();
for instance in [&mut incremental, &mut reference] {
for index in 0..16 {
instance
.write_base_row(
"tasks",
&vec![
Some(ColumnValue::String(format!("task{index}"))),
Some(ColumnValue::String("p0".to_owned())),
Some(ColumnValue::String(format!("base{index}"))),
],
1,
)
.unwrap();
}
instance.rebuild_overlay_if_dirty();
instance.overlay_rebuild_count.set(0);
}
for index in 0..100 {
let mutations = (0..1 + index % 4)
.map(|offset| {
let id = format!("task{}", (index + offset) % 24);
if (index + offset) % 7 == 0 {
Mutation::Delete {
table: "tasks".to_owned(),
row_id: id,
base_version: None,
}
} else {
Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from(id)),
(
"project_id".to_owned(),
Value::from(format!("p{}", index % 2)),
),
(
"title".to_owned(),
Value::from(format!("needle{}", (index + offset) % 24)),
),
]),
base_version: None,
}
}
})
.collect::<Vec<_>>();
reference.overlay_dirty.set(true);
incremental.mutate(mutations.clone()).unwrap();
reference.mutate(mutations).unwrap();
for sql in ["SELECT * FROM tasks ORDER BY id", "SELECT _syncular_source_id, title FROM tasks_fts ORDER BY _syncular_source_id",
"SELECT _syncular_source_id FROM tasks_fts WHERE tasks_fts MATCH 'needle1' ORDER BY _syncular_source_id"] {
assert_eq!(incremental.query(sql, &[]).unwrap(), reference.query(sql, &[]).unwrap(), "commit {index}: {sql}");
}
assert_eq!(incremental.local_revision(), reference.local_revision());
assert_eq!(
serde_json::to_value(incremental.drain_change_batches()).unwrap(),
serde_json::to_value(reference.drain_change_batches()).unwrap()
);
}
assert_eq!(incremental.outbox.len(), 100);
assert_eq!(incremental.overlay_rebuild_count.get(), 0);
assert_eq!(reference.overlay_rebuild_count.get(), 100);
}
#[test]
fn failed_append_rolls_back_durable_queue_visible_rows_and_memory() {
for fail_at in ["outbox", "revision"] {
let mut client = client();
let values = Map::from_iter([
("id".to_owned(), Value::from("t1")),
("project_id".to_owned(), Value::from("p1")),
]);
let mutation = Mutation::Upsert {
table: "tasks".to_owned(),
values,
base_version: None,
};
let trigger = if fail_at == "outbox" {
"CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_outbox BEGIN SELECT RAISE(FAIL, 'injected outbox failure'); END".to_owned()
} else {
format!("CREATE TRIGGER fail_append BEFORE INSERT ON _syncular_meta WHEN NEW.key = '{LOCAL_REVISION_KEY}' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END")
};
client.conn.execute_batch(&trigger).unwrap();
let revision = client.local_revision();
assert!(client.mutate(vec![mutation.clone()]).is_err(), "{fail_at}");
assert_eq!(client.local_revision(), revision);
assert!(client.outbox.is_empty());
assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
assert_eq!(
client
.conn
.query_row("SELECT count(*) FROM _syncular_outbox", [], |row| row
.get::<_, i64>(0))
.unwrap(),
0
);
assert!(client.drain_change_batches().is_empty());
assert!(!client.overlay_dirty.get());
client
.conn
.execute_batch("DROP TRIGGER fail_append")
.unwrap();
client.mutate(vec![mutation]).unwrap();
assert_eq!(client.outbox.len(), 1);
assert_eq!(client.query("SELECT * FROM tasks", &[]).unwrap().len(), 1);
}
}
#[test]
fn mirrored_commit_failure_rolls_back_base_visible_and_revision() {
let mut client = client();
let tables = vec!["tasks".to_owned(), "missing".to_owned()];
let values = Map::from_iter([
("id".to_owned(), Value::from("t1")),
("project_id".to_owned(), Value::from("p1")),
]);
let payload = encode_row_json(
client.schema.table("tasks").unwrap(),
"t1",
&values,
&client.encryption,
)
.unwrap();
let first = ssp2::model::Change {
table_index: 0,
row_id: "t1".to_owned(),
op: Op::Upsert,
row_version: Some(1),
scopes: vec![("project_id".to_owned(), "p1".to_owned())],
row: Some(payload),
};
let mut invalid = first.clone();
invalid.table_index = 1;
let revision = client.local_revision();
client.begin_observation("failed_mirror").unwrap();
assert!(client
.apply_commit_changes(&tables, &[first, invalid])
.is_err());
client.rollback_observation("failed_mirror");
client.rebuild_overlay_if_dirty();
assert!(client.query("SELECT * FROM tasks", &[]).unwrap().is_empty());
assert_eq!(
client
.conn
.query_row("SELECT count(*) FROM _syncular_base_tasks", [], |row| row
.get::<_, i64>(
0
))
.unwrap(),
0
);
assert_eq!(client.local_revision(), revision);
assert!(client.drain_change_batches().is_empty());
}
#[test]
fn successful_acknowledgement_runs_are_atomic_and_preserve_completed_runs() {
for failure in ["journal", "revision", "commit", "later-frame"] {
let mut client = client();
client.set_meta(LOG_EPOCH_KEY, "epoch-1");
for id in ["first", "second"] {
client
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from(id)),
("project_id".to_owned(), Value::from("p1")),
]),
base_version: None,
}])
.unwrap();
}
let (_, meta) = client.build_request(false);
client
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from("later")),
("project_id".to_owned(), Value::from("p1")),
]),
base_version: None,
}])
.unwrap();
let ids = client.pending_commit_ids();
let rows = client
.query("SELECT * FROM tasks ORDER BY id", &[])
.unwrap();
let revision = client.local_revision();
client.drain_change_batches();
if failure != "later-frame" {
client.conn.execute_batch(match failure {
"journal" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 1 BEGIN SELECT RAISE(FAIL, 'injected second journal failure'); END",
"revision" => "CREATE TRIGGER fail_run BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' AND (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN SELECT RAISE(FAIL, 'injected run revision failure'); END",
_ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_run AFTER INSERT ON _syncular_commit_outcomes WHEN (SELECT count(*) FROM _syncular_commit_outcomes) = 2 BEGIN INSERT INTO ack_child VALUES (1); END",
}).unwrap();
}
let mut response = Message {
wire_version: WIRE_VERSION,
msg_kind: MsgKind::Response,
frames: vec![Frame::RespHeader {
required_schema_version: None,
latest_schema_version: None,
log_epoch: Some("epoch-1".to_owned()),
reset_required: Some(false),
}],
};
for id in [&ids[0], &ids[0], "unknown", &ids[1], &ids[2]] {
response.frames.push(Frame::PushResult {
client_commit_id: id.to_owned(),
status: PushStatus::Applied,
commit_seq: Some(1),
results: vec![OpResult::Applied { op_index: 0 }],
});
}
if failure == "later-frame" {
response.frames.push(Frame::Error {
code: "sync.invalid_request".to_owned(),
message: "later error".to_owned(),
category: "protocol".to_owned(),
retryable: false,
recommended_action: "retry".to_owned(),
details: None,
});
}
let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
let outcome = client.process_response(&mut transport, response.clone(), &meta);
assert!(
matches!(outcome, SyncOutcome::Failed { error_code, .. } if error_code == if failure == "later-frame" { "sync.invalid_request" } else { "client.outcome_persistence_failed" })
);
if failure == "later-frame" {
assert_eq!(client.pending_commit_ids(), ids[2..]);
assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
assert!(client.commit_outcome(&ids[1]).unwrap().is_some());
let batches = client.drain_change_batches();
assert_eq!(batches.len(), 1);
assert_eq!(batches[0].status.as_ref().unwrap().outbox, 1);
assert_eq!(client.local_revision(), revision + 1);
response.frames.pop();
} else {
assert_eq!(client.pending_commit_ids(), ids);
assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
assert!(client.commit_outcome(&ids[1]).unwrap().is_none());
assert_eq!(client.local_revision(), revision);
assert!(client.drain_change_batches().is_empty());
assert_eq!(
client
.query("SELECT * FROM tasks ORDER BY id", &[])
.unwrap(),
rows
);
client.conn.execute_batch("DROP TRIGGER fail_run").unwrap();
}
for frame in &mut response.frames {
if let Frame::PushResult { status, .. } = frame {
*status = PushStatus::Cached;
}
}
let SyncOutcome::Ok(report) = client.process_response(&mut transport, response, &meta)
else {
panic!("retry failed");
};
assert_eq!(
report.applied,
if failure == "later-frame" {
vec![]
} else {
ids[..2].to_vec()
}
);
assert_eq!(client.pending_commit_ids(), ids[2..]);
let batches = client.drain_change_batches();
assert_eq!(batches.len(), usize::from(failure != "later-frame"));
if let Some(batch) = batches.first() {
assert_eq!(batch.status.as_ref().unwrap().outbox, 1);
}
assert_eq!(client.local_revision(), revision + 1);
}
}
#[test]
fn failed_push_result_preserves_memory_disk_and_retry_identity() {
for failure in ["revision", "journal", "commit"] {
for status in ["applied", "cached", "rejected", "conflict"] {
let mut client = client();
client.set_meta(LOG_EPOCH_KEY, "epoch-1");
for id in ["first", "later"] {
client
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from(id)),
("project_id".to_owned(), Value::from("p1")),
]),
base_version: None,
}])
.unwrap();
}
let (_, meta) = client.build_request(false);
let ids = client.pending_commit_ids();
let rows = client
.query("SELECT * FROM tasks ORDER BY id", &[])
.unwrap();
let revision = client.local_revision();
client.drain_change_batches();
let (push_status, results) = match status {
"applied" => (PushStatus::Applied, vec![OpResult::Applied { op_index: 0 }]),
"cached" => (PushStatus::Cached, vec![OpResult::Applied { op_index: 0 }]),
"conflict" => (
PushStatus::Rejected,
vec![OpResult::Conflict {
op_index: 0,
code: "sync.version_conflict".to_owned(),
message: "conflict".to_owned(),
server_version: 1,
server_row: encode_row_json(
client.schema.table("tasks").unwrap(),
"first",
&Map::from_iter([
("id".to_owned(), Value::from("first")),
("project_id".to_owned(), Value::from("p1")),
]),
&client.encryption,
)
.unwrap(),
conflict_columns: Vec::new(),
}],
),
_ => (
PushStatus::Rejected,
vec![OpResult::Error {
op_index: 0,
code: "sync.validation_failed".to_owned(),
message: "rejected".to_owned(),
retryable: false,
}],
),
};
let response = Message {
wire_version: WIRE_VERSION,
msg_kind: MsgKind::Response,
frames: vec![
Frame::RespHeader {
required_schema_version: None,
latest_schema_version: None,
log_epoch: Some("epoch-1".to_owned()),
reset_required: Some(false),
},
Frame::PushResult {
client_commit_id: ids[0].clone(),
status: push_status,
commit_seq: Some(1),
results,
},
],
};
client.conn.execute_batch(match failure {
"journal" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_commit_outcomes BEGIN SELECT RAISE(FAIL, 'injected journal failure'); END",
"revision" => "CREATE TRIGGER fail_ack BEFORE INSERT ON _syncular_meta WHEN NEW.key = 'localRevision' BEGIN SELECT RAISE(FAIL, 'injected revision failure'); END",
_ => "PRAGMA foreign_keys = ON; CREATE TABLE ack_parent(id INTEGER PRIMARY KEY); CREATE TABLE ack_child(id INTEGER REFERENCES ack_parent(id) DEFERRABLE INITIALLY DEFERRED); CREATE TRIGGER fail_ack AFTER INSERT ON _syncular_commit_outcomes BEGIN INSERT INTO ack_child VALUES (1); END",
}).unwrap();
let mut transport = HostTransport::new_from_config(&json!({})).unwrap();
let outcome = client.process_response(&mut transport, response.clone(), &meta);
assert_eq!(
client.pending_commit_ids(),
ids,
"{failure}/{status}: pending identity"
);
assert_eq!(
client.local_revision(),
revision,
"{failure}/{status}: revision"
);
assert!(client.commit_outcome(&ids[0]).unwrap().is_none());
assert!(client.conflicts().is_empty());
assert!(client.rejections().is_empty());
assert!(client.drain_change_batches().is_empty());
assert_eq!(
client
.query("SELECT * FROM tasks ORDER BY id", &[])
.unwrap(),
rows
);
assert!(
matches!(outcome, SyncOutcome::Failed { error_code, .. } if error_code == "client.outcome_persistence_failed"),
"{failure}/{status}: failed round"
);
client.conn.execute_batch("DROP TRIGGER fail_ack").unwrap();
assert!(matches!(
client.process_response(&mut transport, response, &meta),
SyncOutcome::Ok(_)
));
assert_eq!(client.pending_commit_ids(), ids[1..]);
assert!(client.commit_outcome(&ids[0]).unwrap().is_some());
assert_eq!(client.drain_change_batches().len(), 1);
}
}
}
#[test]
fn mixed_push_results_reconcile_and_prune_once_per_response() {
let mut client = client();
client.set_meta(LOG_EPOCH_KEY, "epoch-1");
for index in 0..4 {
client
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from(format!("task-{index}"))),
("project_id".to_owned(), Value::from("project-1")),
]),
base_version: None,
}])
.expect("queue commit");
}
let (_, request_meta) = client.build_request(false);
let ids = &request_meta.pushed_ids;
assert_eq!(ids.len(), 4);
let response = Message {
wire_version: WIRE_VERSION,
msg_kind: MsgKind::Response,
frames: vec![
Frame::RespHeader {
required_schema_version: None,
latest_schema_version: None,
log_epoch: Some("epoch-1".to_owned()),
reset_required: Some(false),
},
Frame::PushResult {
client_commit_id: ids[0].clone(),
status: PushStatus::Applied,
commit_seq: Some(1),
results: vec![OpResult::Applied { op_index: 0 }],
},
Frame::PushResult {
client_commit_id: ids[1].clone(),
status: PushStatus::Cached,
commit_seq: Some(2),
results: vec![OpResult::Applied { op_index: 0 }],
},
Frame::PushResult {
client_commit_id: ids[2].clone(),
status: PushStatus::Rejected,
commit_seq: None,
results: vec![OpResult::Error {
op_index: 0,
code: "sync.validation_failed".to_owned(),
message: "rejected".to_owned(),
retryable: false,
}],
},
Frame::PushResult {
client_commit_id: ids[3].clone(),
status: PushStatus::Rejected,
commit_seq: None,
results: vec![OpResult::Error {
op_index: 0,
code: "sync.idempotency_cache_miss".to_owned(),
message: "retry".to_owned(),
retryable: true,
}],
},
],
};
let mut transport =
HostTransport::new_from_config(&json!({})).expect("no-network host transport");
client.overlay_rebuild_count.set(0);
client.outcome_prune_count.set(0);
let outcome = client.process_response(&mut transport, response, &request_meta);
let SyncOutcome::Ok(report) = outcome else {
panic!("mixed push-result response failed");
};
assert_eq!(report.applied, ids[..2]);
assert_eq!(report.rejected, ids[2..3]);
assert_eq!(report.retryable, ids[3..4]);
assert_eq!(client.pending_commit_ids(), ids[3..4]);
assert_eq!(
client
.query("SELECT id FROM tasks ORDER BY id", &[])
.expect("query visible overlay"),
vec![Map::from_iter([("id".to_owned(), Value::from("task-3"))])]
);
assert_eq!(client.overlay_rebuild_count.get(), 1);
assert_eq!(client.outcome_prune_count.get(), 1);
}
#[test]
fn secondary_unique_collision_preserves_existing_synced_row() {
let client = SyncClient::new(
"unique-upsert-test".to_owned(),
&json!({
"version": 1,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false },
{ "name": "title", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }],
"indexes": [{
"name": "tasks_by_project_title",
"columns": ["project_id", "title"],
"unique": true
}]
}]
}),
ClientLimits::default(),
)
.expect("test client");
let table = client.schema.table("tasks").expect("tasks table");
let sql = client.insert_row_sql(&base_table("tasks"), table);
client
.conn
.execute(&sql, rusqlite::params!["t1", "p1", "original", 1])
.expect("insert first row");
client
.conn
.execute(&sql, rusqlite::params!["t1", "p1", "updated", 2])
.expect("update same primary key");
client
.conn
.execute(&sql, rusqlite::params!["t2", "p1", "original", 1])
.expect("insert second row");
assert!(client
.conn
.execute(&sql, rusqlite::params!["t3", "p1", "original", 2])
.is_err());
let rows = client
.conn
.prepare("SELECT id, title FROM _syncular_base_tasks ORDER BY id")
.expect("prepare rows")
.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})
.expect("query rows")
.collect::<Result<Vec<_>, _>>()
.expect("collect rows");
assert_eq!(
rows,
vec![
("t1".to_owned(), "updated".to_owned()),
("t2".to_owned(), "original".to_owned())
]
);
}
#[test]
fn reopening_active_subscriptions_emits_a_catch_up_intent() {
let path = std::env::temp_dir().join(format!(
"syncular-startup-intent-{}.db",
uuid::Uuid::new_v4()
));
let schema = json!({
"version": 1,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}]
});
{
let mut first = SyncClient::open_path_with_identity(
None,
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("first open");
first
.set_window(
&WindowBase {
table: "tasks".to_owned(),
variable: "project_id".to_owned(),
fixed_scopes: Vec::new(),
params: None,
},
&["persisted".to_owned()],
)
.expect("persist window");
}
let mut reopened = SyncClient::open_path_with_identity(
None,
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("reopen");
assert!(reopened.sync_needed());
assert!(matches!(
reopened.drain_sync_intents().as_slice(),
[SyncIntent::Interactive]
));
drop(reopened);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn reopening_preserves_immutable_subscription_identity_and_progress() {
let path = std::env::temp_dir().join(format!(
"syncular-subscription-identity-{}.db",
uuid::Uuid::new_v4()
));
let schema = json!({
"version": 1,
"tables": [
{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
},
{
"name": "docs",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "org_id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [
{ "pattern": "org:{org_id}" },
{ "pattern": "project:{projectId}", "column": "project_id" }
]
}
]
});
{
let mut first = SyncClient::open_path_with_identity(
None,
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("first open");
first
.subscribe(
"stable-subscription".to_owned(),
"tasks".to_owned(),
vec![(
"project_id".to_owned(),
vec!["p2".to_owned(), "p1".to_owned()],
)],
Some(r#"{"view":"v1"}"#.to_owned()),
)
.expect("persist subscription");
let persisted = {
let subscription = first
.subs
.iter_mut()
.find(|subscription| subscription.id == "stable-subscription")
.expect("subscription");
subscription.cursor = 41;
subscription.bootstrap_state = Some("resume-token".to_owned());
subscription.effective = Some(vec![(
"project_id".to_owned(),
vec!["p1".to_owned(), "p2".to_owned()],
)]);
subscription.synced_once = true;
subscription.clone()
};
first.persist_sub(&persisted).unwrap();
}
let mut reopened = SyncClient::open_path_with_identity(
None,
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("reopen");
let progress = reopened
.subscription_state("stable-subscription")
.expect("persisted state");
assert_eq!(progress.cursor, 41);
assert!(progress.has_resume_token);
reopened
.subscribe(
"stable-subscription".to_owned(),
"tasks".to_owned(),
vec![(
"project_id".to_owned(),
vec!["p1".to_owned(), "p2".to_owned(), "p1".to_owned()],
)],
Some(r#"{"view":"v1"}"#.to_owned()),
)
.expect("canonical intent is idempotent");
assert_eq!(
reopened
.subscription_state("stable-subscription")
.expect("unchanged state")
.cursor,
progress.cursor
);
for (table, scopes, params) in [
(
"tasks",
vec![("project_id".to_owned(), vec!["p1".to_owned()])],
Some(r#"{"view":"v1"}"#.to_owned()),
),
(
"tasks",
vec![(
"project_id".to_owned(),
vec!["p2".to_owned(), "p1".to_owned()],
)],
Some(r#"{"view":"v2"}"#.to_owned()),
),
(
"docs",
vec![
("org_id".to_owned(), vec!["o1".to_owned()]),
("projectId".to_owned(), vec!["p1".to_owned()]),
],
Some(r#"{"view":"v1"}"#.to_owned()),
),
] {
let error = reopened
.subscribe(
"stable-subscription".to_owned(),
table.to_owned(),
scopes,
params,
)
.expect_err("identity rebind must fail");
assert!(error.starts_with("client.subscription_intent_mismatch:"));
assert_eq!(
reopened
.subscription_state("stable-subscription")
.expect("unchanged state")
.cursor,
progress.cursor
);
}
drop(reopened);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn reopening_clears_a_schema_floor_the_running_app_already_satisfies() {
let path = std::env::temp_dir().join(format!(
"syncular-satisfied-schema-floor-{}.db",
uuid::Uuid::new_v4()
));
let schema = json!({
"version": 23,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}]
});
{
let mut first = SyncClient::open_path_with_identity(
None,
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("first open");
first
.subscribe(
"tasks".to_owned(),
"tasks".to_owned(),
vec![("project_id".to_owned(), vec!["p1".to_owned()])],
None,
)
.expect("persist subscription");
first.set_schema_floor(Some(SchemaFloor {
required_schema_version: Some(22),
latest_schema_version: Some(22),
}));
}
let mut reopened = SyncClient::open_path_with_identity(
None,
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("reopen");
assert!(reopened.schema_floor().is_none());
assert!(reopened.get_meta(SCHEMA_FLOOR_KEY).is_none());
assert!(reopened.sync_needed());
assert!(matches!(
reopened.drain_sync_intents().as_slice(),
[SyncIntent::Interactive]
));
drop(reopened);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn reopening_keeps_an_unsatisfied_schema_floor_stopped() {
let path = std::env::temp_dir().join(format!(
"syncular-unsatisfied-schema-floor-{}.db",
uuid::Uuid::new_v4()
));
let schema = json!({
"version": 1,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}]
});
{
let mut first = SyncClient::open_path_with_identity(
None,
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("first open");
first
.subscribe(
"tasks".to_owned(),
"tasks".to_owned(),
vec![("project_id".to_owned(), vec!["p1".to_owned()])],
None,
)
.expect("persist subscription");
first.set_schema_floor(Some(SchemaFloor {
required_schema_version: Some(2),
latest_schema_version: Some(2),
}));
}
let reopened = SyncClient::open_path_with_identity(
None,
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("reopen");
assert_eq!(
reopened.schema_floor(),
Some(&SchemaFloor {
required_schema_version: Some(2),
latest_schema_version: Some(2),
})
);
assert!(!reopened.sync_needed());
drop(reopened);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn schema_bump_precedes_index_ddl_and_prunes_removed_subscriptions() {
let path = std::env::temp_dir().join(format!(
"syncular-indexed-column-bump-{}.db",
uuid::Uuid::new_v4()
));
let old_schema = json!({
"version": 1,
"tables": [
{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
},
{
"name": "legacy",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}
]
});
{
let mut first = SyncClient::open_path_with_identity(
None,
&old_schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("open old schema");
first
.subscribe(
"legacy-sub".to_owned(),
"legacy".to_owned(),
vec![("project_id".to_owned(), vec!["p1".to_owned()])],
None,
)
.expect("persist legacy subscription");
}
let new_schema = json!({
"version": 2,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false },
{ "name": "facility_membership_id", "type": "string", "nullable": true }
],
"scopes": [{ "pattern": "project:{project_id}" }],
"indexes": [{
"name": "tasks_by_membership",
"columns": ["project_id", "facility_membership_id"],
"unique": false
}]
}]
});
let upgraded = SyncClient::open_path_with_identity(
None,
&new_schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("open upgraded schema");
let columns = upgraded
.conn
.prepare("PRAGMA table_info(tasks)")
.expect("prepare columns")
.query_map([], |row| row.get::<_, String>(1))
.expect("query columns")
.collect::<Result<Vec<_>, _>>()
.expect("collect columns");
assert!(columns.contains(&"facility_membership_id".to_owned()));
let index_count: i64 = upgraded
.conn
.query_row(
"SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'tasks_by_membership'",
[],
|row| row.get(0),
)
.expect("query index");
assert_eq!(index_count, 1);
assert!(upgraded.subscription_state("legacy-sub").is_none());
drop(upgraded);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn migrates_pre_envelope_outcome_journal_additively() {
let path = std::env::temp_dir().join(format!(
"syncular-outcome-migration-{}.db",
uuid::Uuid::new_v4()
));
let conn = Connection::open(&path).expect("open legacy database");
conn.execute_batch(
"CREATE TABLE _syncular_commit_outcomes (
seq INTEGER PRIMARY KEY AUTOINCREMENT,
client_commit_id TEXT NOT NULL UNIQUE,
status TEXT NOT NULL,
recorded_at_ms INTEGER NOT NULL,
results_json TEXT NOT NULL,
resolution TEXT NOT NULL DEFAULT 'active',
resolved_at_ms INTEGER,
replacement_client_commit_id TEXT);",
)
.expect("create legacy outcome journal");
drop(conn);
let schema = json!({
"version": 1,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}]
});
let client = SyncClient::open_path(
"migration-native".to_owned(),
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("migrate database");
let has_operations = client
.conn
.prepare("PRAGMA table_info(_syncular_commit_outcomes)")
.expect("prepare table info")
.query_map([], |row| row.get::<_, String>(1))
.expect("query table info")
.filter_map(Result::ok)
.any(|column| column == "operations_json");
assert!(has_operations);
drop(client);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn durable_conflict_outcome_and_resolution_survive_reopen() {
let path = std::env::temp_dir().join(format!(
"syncular-durable-outcome-{}.db",
uuid::Uuid::new_v4()
));
let schema = json!({
"version": 1,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}]
});
let conflict = ConflictRecord {
client_commit_id: "losing-commit".to_owned(),
op_index: 0,
table: "tasks".to_owned(),
row_id: "t1".to_owned(),
code: "sync.version_conflict".to_owned(),
message: "stale base version".to_owned(),
server_version: 2,
server_row: Map::from_iter([("id".to_owned(), json!("t1"))]),
conflict_columns: vec!["project_id".to_owned()],
operation: Some(CommitOperation {
table: "tasks".to_owned(),
row_id: "t1".to_owned(),
op: "upsert".to_owned(),
base_version: Some(1),
values: None,
}),
};
let failed_operations = vec![
OutboxOp {
upsert: true,
table: "tasks".to_owned(),
row_id: "t1".to_owned(),
base_version: Some(1),
values: None,
},
OutboxOp {
upsert: true,
table: "tasks".to_owned(),
row_id: "status-event-1".to_owned(),
base_version: Some(0),
values: None,
},
];
{
let mut first = SyncClient::open_path(
"durable-native".to_owned(),
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("first open");
first
.begin_observation("test_outcome")
.expect("begin outcome");
first
.persist_commit_outcome(
"losing-commit",
CommitOutcomeStatus::Conflict,
&[CommitOperationOutcome::Conflict {
conflict: conflict.clone(),
}],
Some(&failed_operations),
)
.expect("persist outcome");
first.conflicts.push(conflict);
first
.finish_observation(
"test_outcome",
ChangeAccumulator {
conflicts: true,
outcomes: true,
..ChangeAccumulator::default()
},
)
.expect("commit outcome");
}
{
let mut reopened = SyncClient::open_path(
"durable-native".to_owned(),
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("reopen");
assert_eq!(reopened.conflicts().len(), 1);
assert_eq!(
reopened.conflicts()[0].conflict_columns,
vec!["project_id".to_owned()],
"conflictColumns survives the journal round trip"
);
let outcome = reopened
.commit_outcome("losing-commit")
.expect("read outcome")
.expect("outcome");
assert_eq!(outcome.status, CommitOutcomeStatus::Conflict);
let operations = outcome.operations.expect("aggregate envelope");
assert_eq!(operations.len(), 2);
assert_eq!(operations[1].row_id, "status-event-1");
let resolved = reopened
.resolve_commit_outcome(ResolveCommitOutcomeInput {
client_commit_id: "losing-commit".to_owned(),
resolution: CommitOutcomeResolution::ResolvedKeepServer,
replacement_client_commit_id: None,
})
.expect("resolve");
assert_eq!(
resolved.resolution,
CommitOutcomeResolution::ResolvedKeepServer
);
assert!(reopened.conflicts().is_empty());
}
let reopened = SyncClient::open_path(
"durable-native".to_owned(),
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("second reopen");
assert!(reopened.conflicts().is_empty());
assert_eq!(
reopened
.commit_outcome("losing-commit")
.expect("read outcome")
.expect("outcome")
.resolution,
CommitOutcomeResolution::ResolvedKeepServer
);
drop(reopened);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn file_snapshot_reader_matches_owner_rows_revision_and_coverage() {
let path =
std::env::temp_dir().join(format!("syncular-read-sidecar-{}.db", uuid::Uuid::new_v4()));
let schema = json!({
"version": 1,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "project_id", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "project:{project_id}" }]
}]
});
let mut client = SyncClient::open_path_with_identity(
Some("sidecar-client".to_owned()),
&schema,
ClientLimits::default(),
path.to_str().expect("UTF-8 temp path"),
)
.expect("open owner");
let base = WindowBase {
table: "tasks".to_owned(),
variable: "project_id".to_owned(),
fixed_scopes: Vec::new(),
params: None,
};
client
.set_window(&base, &["one".to_owned()])
.expect("set window");
client
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from("t1")),
("project_id".to_owned(), Value::from("one")),
]),
base_version: None,
}])
.expect("local mutate");
let coverage = [WindowCoverage {
base,
units: vec!["one".to_owned(), "missing".to_owned()],
}];
let owner = client
.query_snapshot(
"SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
&[],
&coverage,
None,
)
.expect("owner snapshot");
let mut reader = FileQuerySnapshotReader::new(path.to_string_lossy());
let sidecar = reader
.query_snapshot(
"SELECT id, project_id, _sync_version AS server_version FROM tasks ORDER BY id",
&[],
&coverage,
)
.expect("sidecar snapshot");
assert_eq!(sidecar.revision, owner.revision);
assert_eq!(sidecar.rows, owner.rows);
assert_eq!(
serde_json::to_value(&sidecar.coverage).expect("serialize sidecar coverage"),
serde_json::to_value(&owner.coverage).expect("serialize owner coverage")
);
assert_eq!(sidecar.revision, "2");
assert_eq!(sidecar.rows[0]["id"], "t1");
assert_eq!(sidecar.rows[0]["server_version"], -1);
assert!(!sidecar.coverage.complete);
assert_eq!(sidecar.coverage.pending.len(), 1);
assert_eq!(sidecar.coverage.missing.len(), 1);
drop(reader);
drop(client);
std::fs::remove_file(path).expect("remove temp database");
}
#[test]
fn owned_query_failures_classify_order_bound_and_clear() {
let io = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
rusqlite::ffi::Error::new(266),
Some("private SQLite prose".to_owned()),
));
assert_eq!(io.code, Some("client.storage_io"));
assert_eq!(io.sqlite_code, Some(266));
assert_eq!(io.message, "local SQLite storage I/O failed");
let corrupt = QueryReadFailure::from(rusqlite::Error::SqliteFailure(
rusqlite::ffi::Error::new(26),
None,
));
assert_eq!(corrupt.code, Some("client.storage_corrupt"));
let mut client = client();
let generic = QueryReadFailure::from("private query prose".to_owned());
client.record_query_read(
QueryOwner {
id: "first",
tables: &["tasks", "tasks"],
},
Some(&generic),
);
client.record_query_read(
QueryOwner {
id: "second",
tables: &["tasks"],
},
Some(&generic),
);
client.record_query_read(
QueryOwner {
id: "first",
tables: &["docs"],
},
Some(&io),
);
let failures = &client
.diagnostics_snapshot(&Default::default())
.expect("diagnostics")
.query_failures;
assert_eq!(
failures
.iter()
.map(|entry| entry.id.as_str())
.collect::<Vec<_>>(),
vec!["second", "first"]
);
assert_eq!(failures[1].tables, vec!["docs"]);
assert_eq!(failures[1].code, "client.storage_io");
for index in 0..=MAX_DIAGNOSTIC_QUERY_FAILURES {
client.record_query_read(
QueryOwner {
id: &format!("query-{index}"),
tables: &["tasks"],
},
Some(&generic),
);
}
let failures = &client
.diagnostics_snapshot(&Default::default())
.expect("bounded diagnostics")
.query_failures;
assert_eq!(failures.len(), MAX_DIAGNOSTIC_QUERY_FAILURES);
assert_eq!(failures.last().expect("last failure").id, "query-256");
client
.query_snapshot(
"SELECT id FROM tasks",
&[],
&[],
Some(QueryOwner {
id: "query-256",
tables: &["tasks"],
}),
)
.expect("successful owned read");
assert!(client
.diagnostics_snapshot(&Default::default())
.expect("cleared diagnostics")
.query_failures
.iter()
.all(|entry| entry.id != "query-256"));
}
#[test]
fn image_chunks_keep_their_committed_prefix_after_a_later_invalid_row() {
let mut client = client();
let image = Connection::open_in_memory().unwrap();
image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT, _syncular_version INTEGER NOT NULL);
WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
INSERT INTO tasks SELECT printf('%05d',x),CASE WHEN x=1500 THEN NULL ELSE 'p1' END,1 FROM n;
CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',2050);").unwrap();
let table = client.schema.table("tasks").unwrap().clone();
let scopes = vec![("project_id".into(), vec!["p1".into()])];
assert!(client
.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest")
.is_err());
assert!(client.conn.is_autocommit());
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks", &[])
.unwrap()[0]["n"],
1024
);
assert_eq!(client.drain_change_batches().len(), 1);
image
.execute("UPDATE tasks SET project_id='p1' WHERE id='01500'", [])
.unwrap();
assert!(matches!(
client.apply_sqlite_image(&image, &table, true, &scopes, 2050, 7, "digest"),
Ok(2050)
));
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks", &[])
.unwrap()[0]["n"],
2050
);
assert_eq!(client.drain_change_batches().len(), 3);
}
#[test]
fn eviction_chunks_commit_before_failure_and_resume_with_exact_scope() {
let mut client = client();
let base = WindowBase {
table: "tasks".into(),
variable: "project_id".into(),
fixed_scopes: Vec::new(),
params: None,
};
client
.set_window(&base, &["p1".into(), "p2".into()])
.unwrap();
client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
INSERT INTO _syncular_base_tasks VALUES ('held','p2',1);
INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
client.drain_change_batches();
assert!(client.set_window(&base, &["p2".into()]).is_err());
assert!(client.conn.is_autocommit());
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks", &[])
.unwrap()[0]["n"],
1027
);
assert_eq!(client.window_state(&base).units, vec!["p2"]);
assert_eq!(client.load_pending_evictions().unwrap().len(), 1);
assert_eq!(client.drain_change_batches().len(), 1);
client
.conn
.execute_batch("DROP TRIGGER interrupt_evict")
.unwrap();
client.drain_pending_evictions().unwrap();
assert!(client.load_pending_evictions().unwrap().is_empty());
assert_eq!(
client.query("SELECT id FROM tasks", &[]).unwrap()[0]["id"],
"held"
);
assert_eq!(client.drain_change_batches().len(), 2);
}
#[test]
fn eviction_marker_failures_roll_back_window_changes() {
let mut client = client();
let base = WindowBase {
table: "tasks".into(),
variable: "project_id".into(),
fixed_scopes: Vec::new(),
params: None,
};
client.set_window(&base, &["p1".into()]).unwrap();
client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<2050)
INSERT INTO _syncular_base_tasks SELECT printf('%05d',x),'p1',1 FROM n;
INSERT INTO tasks SELECT * FROM _syncular_base_tasks;
CREATE TRIGGER fail_marker BEFORE INSERT ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
let revision = client.local_revision();
assert!(client.set_window(&base, &[]).is_err());
assert!(client.conn.is_autocommit());
assert_eq!(client.local_revision(), revision);
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks", &[])
.unwrap()[0]["n"],
2050
);
assert_eq!(client.window_state(&base).units, vec!["p1"]);
client.conn.execute_batch("DROP TRIGGER fail_marker;
CREATE TRIGGER interrupt_evict BEFORE DELETE ON tasks WHEN old.id='01500' BEGIN SELECT RAISE(ABORT,'interrupted chunk'); END;").unwrap();
assert!(client.set_window(&base, &[]).is_err());
client.conn.execute_batch("DROP TRIGGER interrupt_evict;
CREATE TRIGGER fail_marker BEFORE DELETE ON _syncular_window_pending_evict BEGIN SELECT RAISE(ABORT,'marker failed'); END;").unwrap();
assert!(client.drain_pending_evictions().is_err());
assert!(client.conn.is_autocommit());
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks", &[])
.unwrap()[0]["n"],
2
);
assert!(client.set_window(&base, &["p1".into()]).is_err());
assert!(client.conn.is_autocommit());
assert!(client.window_state(&base).units.is_empty());
assert!(client.subs.is_empty());
assert_eq!(
client
.query(
"SELECT count(*) AS n FROM _syncular_window_pending_evict",
&[]
)
.unwrap()[0]["n"],
1
);
client
.conn
.execute_batch("DROP TRIGGER fail_marker")
.unwrap();
client.set_window(&base, &["p1".into()]).unwrap();
assert_eq!(client.window_state(&base).units, vec!["p1"]);
}
#[test]
fn malformed_eviction_marker_is_reported_without_starting_cleanup() {
let mut client = client();
client
.conn
.execute(
"INSERT INTO _syncular_window_pending_evict VALUES ('broken','tasks','{')",
[],
)
.unwrap();
assert_eq!(
client.drain_pending_evictions().unwrap_err(),
"sync.local_corrupt: invalid pending eviction scopes"
);
assert!(client.conn.is_autocommit());
assert!(client.enqueue_startup_sync_if_needed().is_err());
}
#[test]
fn image_chunks_reject_null_primary_keys() {
let mut client = client();
let image = Connection::open_in_memory().unwrap();
image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<1025)
INSERT INTO tasks SELECT NULL,'p1',1 FROM n;
CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1025);").unwrap();
let table = client.schema.table("tasks").unwrap().clone();
assert!(client
.apply_sqlite_image(
&image,
&table,
true,
&[("project_id".into(), vec!["p1".into()])],
1025,
7,
"digest"
)
.is_err());
assert!(client.conn.is_autocommit());
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks", &[])
.unwrap()[0]["n"],
0
);
}
#[test]
fn import_chunks_with_unique_constraints_match_full_pending_replay() {
for images in [false, true] {
let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
"name":name,"primaryKey":"id","columns":[
{"name":"id","type":"string","nullable":false},
{"name":"project_id","type":"string","nullable":false},
{"name":"title","type":"string","nullable":false}],
"scopes":[{"pattern":"project:{project_id}"}],
"indexes":[{"name":format!("{name}_unique_title"),"columns":["title"],"unique":true}],
"ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
})))});
let mut actual =
SyncClient::new("actual".into(), &schema, ClientLimits::default()).unwrap();
let mut reference =
SyncClient::new("reference".into(), &schema, ClientLimits::default()).unwrap();
let scopes = vec![("project_id".into(), vec!["p1".into()])];
for client in [&mut actual, &mut reference] {
client
.subscribe("tasks".into(), "tasks".into(), scopes.clone(), None)
.unwrap();
client.subs[0].effective = Some(scopes.clone());
for (table, id, title) in [
("tasks", "a", "original"),
("tasks", "b", "occupied"),
("tasks", "c", "deleted"),
("notes", "n", "untouched"),
] {
client
.write_base_row(
table,
&vec![
Some(ColumnValue::String(id.into())),
Some(ColumnValue::String("p1".into())),
Some(ColumnValue::String(title.into())),
],
1,
)
.unwrap();
}
client.rebuild_overlay_if_dirty();
client
.mutate(vec![
Mutation::Upsert {
table: "tasks".into(),
values: Map::from_iter([
("id".into(), Value::from("a")),
("project_id".into(), Value::from("p1")),
("title".into(), Value::from("available")),
]),
base_version: None,
},
Mutation::Delete {
table: "tasks".into(),
row_id: "c".into(),
base_version: None,
},
Mutation::Upsert {
table: "tasks".into(),
values: Map::from_iter([
("id".into(), Value::from("local")),
("project_id".into(), Value::from("p1")),
("title".into(), Value::from("local value")),
]),
base_version: None,
},
Mutation::Upsert {
table: "notes".into(),
values: Map::from_iter([
("id".into(), Value::from("n")),
("project_id".into(), Value::from("p1")),
("title".into(), Value::from("pending note")),
]),
base_version: None,
},
])
.unwrap();
}
let pending = actual.pending_commit_ids();
actual.overlay_rebuild_count.set(0);
for (id, title) in [
("b", "available"),
("b", "occupied"),
("a", "server original"),
("d", "local value"),
("d", "released"),
("c", "redelivered"),
] {
let table = actual.schema.table("tasks").unwrap().clone();
let values = vec![
Some(ColumnValue::String(id.into())),
Some(ColumnValue::String("p1".into())),
Some(ColumnValue::String(title.into())),
];
if images {
let image = Connection::open_in_memory().unwrap();
image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
image
.execute("INSERT INTO tasks VALUES (?1,'p1',?2,2)", [id, title])
.unwrap();
assert!(matches!(
actual.apply_sqlite_image(&image, &table, false, &scopes, 1, 7, "digest"),
Ok(1)
));
} else {
let segment = RowsSegment {
table: "tasks".into(),
schema_version: 1,
columns: table.wire_columns.clone(),
blocks: vec![vec![ssp2::segment::SegmentRow {
server_version: 2,
values: values.clone(),
}]],
};
assert!(matches!(actual.apply_segment(0, &segment, false), Ok(1)));
}
reference.write_base_row("tasks", &values, 2).unwrap();
reference.rebuild_overlay();
for table in ["tasks", "notes"] {
for sql in [format!("SELECT * FROM {table} ORDER BY id"),format!("SELECT _syncular_source_id,title FROM {table}_fts ORDER BY _syncular_source_id"),format!("SELECT source_id FROM _syncular_fts_{table}_fts ORDER BY source_id")] {
assert_eq!(actual.query(&sql, &[]).unwrap(), reference.query(&sql, &[]).unwrap(), "images={images}, {id}/{title}: {sql}");
}
}
assert_eq!(actual.pending_commit_ids(), pending);
}
assert_eq!(
actual.overlay_rebuild_count.get(),
0,
"each chunk must leave unrelated tables alone"
);
let before = actual
.query("SELECT * FROM tasks ORDER BY id", &[])
.unwrap();
let fts_before = actual
.query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
.unwrap();
let revision = actual.local_revision();
actual.conn.execute_batch("CREATE TRIGGER fail_import BEFORE INSERT ON tasks WHEN new.id='b' BEGIN SELECT RAISE(ABORT,'restore failed'); END").unwrap();
let table = actual.schema.table("tasks").unwrap().clone();
let segment = RowsSegment {
table: "tasks".into(),
schema_version: 1,
columns: table.wire_columns.clone(),
blocks: vec![vec![ssp2::segment::SegmentRow {
server_version: 3,
values: vec![
Some(ColumnValue::String("b".into())),
Some(ColumnValue::String("p1".into())),
Some(ColumnValue::String("fresh".into())),
],
}]],
};
assert!(actual.apply_segment(0, &segment, false).is_err());
assert!(actual.conn.is_autocommit());
assert_eq!(
actual
.query("SELECT * FROM tasks ORDER BY id", &[])
.unwrap(),
before
);
assert_eq!(
actual
.query("SELECT * FROM tasks_fts ORDER BY _syncular_source_id", &[])
.unwrap(),
fts_before
);
assert_eq!(actual.local_revision(), revision);
assert_eq!(actual.pending_commit_ids(), pending);
}
}
#[test]
fn import_reconciliation_preserves_typed_pending_primary_keys() {
for (kind, a, b) in [
("string", json!("a"), json!("b")),
("integer", json!(1), json!(2)),
("boolean", json!(true), json!(false)),
("json", json!("1"), json!("2")),
] {
let schema = json!({"version":1,"tables":[{"name":"tasks","primaryKey":"id","columns":[{"name":"id","type":kind,"nullable":false},{"name":"title","type":"string","nullable":false}],"scopes":[],"indexes":[{"name":"unique_title","columns":["title"],"unique":true}]}]});
for images in [false, true] {
let mut client =
SyncClient::new("typed".into(), &schema, ClientLimits::default()).unwrap();
let table = client.schema.table("tasks").unwrap().clone();
client
.subscribe("tasks".into(), "tasks".into(), vec![], None)
.unwrap();
for (id, title) in [(&a, "original"), (&b, "occupied")] {
client
.write_base_row(
"tasks",
&vec![
json_to_column_value(&table.columns[0], Some(id)).unwrap(),
Some(ColumnValue::String(title.into())),
],
1,
)
.unwrap();
}
client.rebuild_overlay_if_dirty();
client
.mutate(vec![Mutation::Upsert {
table: "tasks".into(),
values: Map::from_iter([
("id".into(), a.clone()),
("title".into(), json!("available")),
]),
base_version: None,
}])
.unwrap();
client.overlay_rebuild_count.set(0);
for (title, expected) in [("available", "original"), ("occupied", "available")] {
let values = vec![
json_to_column_value(&table.columns[0], Some(&b)).unwrap(),
Some(ColumnValue::String(title.into())),
];
if images {
let image = Connection::open_in_memory().unwrap();
image.execute_batch("CREATE TABLE tasks(id PRIMARY KEY, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);
INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',1);").unwrap();
image
.execute(
"INSERT INTO tasks VALUES (?1,?2,2)",
rusqlite::params![RowParam::Cell(&values[0]), title],
)
.unwrap();
assert!(
matches!(
client.apply_sqlite_image(
&image,
&table,
false,
&[],
1,
7,
"digest"
),
Ok(1)
),
"{kind}, images={images}"
);
} else {
let segment = RowsSegment {
table: "tasks".into(),
schema_version: 1,
columns: table.wire_columns.clone(),
blocks: vec![vec![ssp2::segment::SegmentRow {
server_version: 2,
values,
}]],
};
assert!(
matches!(client.apply_segment(0, &segment, false), Ok(1)),
"{kind}"
);
}
let key = json_to_column_value(&table.columns[0], Some(&a)).unwrap();
let actual: String = client
.conn
.query_row(
"SELECT title FROM tasks WHERE id=?1",
[RowParam::Cell(&key)],
|row| row.get(0),
)
.unwrap();
assert_eq!(actual, expected, "{kind}, images={images}");
}
assert_eq!(client.overlay_rebuild_count.get(), 0);
}
}
}
fn unique_import_fixture(rows: usize) -> (SyncClient, Connection, TableSchema) {
let schema = json!({"version":1,"tables": (["tasks", "notes"].map(|name| json!({
"name":name,"primaryKey":"id","columns":[{"name":"id","type":"string","nullable":false},{"name":"project_id","type":"string","nullable":false},{"name":"title","type":"string","nullable":false}],
"scopes":[{"pattern":"project:{project_id}"}],"indexes":[{"name":format!("{name}_title"),"columns":["title"],"unique":true}],
"ftsIndexes":[{"name":format!("{name}_fts"),"columns":["title"],"tokenize":"unicode61"}]
})))});
let mut client =
SyncClient::new("unique-import".into(), &schema, ClientLimits::default()).unwrap();
client.conn.execute_batch("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<8192) INSERT INTO _syncular_base_notes SELECT printf('%08d',x),'p2','note-'||x,1 FROM n").unwrap();
client.rebuild_overlay();
client
.mutate(vec![Mutation::Upsert {
table: "tasks".into(),
values: Map::from_iter([
("id".into(), json!("offline")),
("project_id".into(), json!("p1")),
("title".into(), json!("pending edit")),
]),
base_version: None,
}])
.unwrap();
let image = Connection::open_in_memory().unwrap();
image.execute_batch("CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT NOT NULL, title TEXT NOT NULL, _syncular_version INTEGER NOT NULL);
CREATE TABLE _syncular_segment(format INTEGER, \"table\" TEXT, \"schemaVersion\" INTEGER, \"asOfCommitSeq\" INTEGER, \"scopeDigest\" TEXT, \"rowCount\" INTEGER);").unwrap();
image.execute("WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM n WHERE x<?1) INSERT INTO tasks SELECT printf('%08d',x),'p1','item-'||x,1 FROM n",[rows]).unwrap();
image
.execute(
"INSERT INTO _syncular_segment VALUES (1,'tasks',1,7,'digest',?1)",
[rows],
)
.unwrap();
client.overlay_rebuild_count.set(0);
let table = client.schema.table("tasks").unwrap().clone();
(client, image, table)
}
#[test]
fn large_unique_import_copies_no_unrelated_tables() {
let (mut client, image, table) = unique_import_fixture(10240);
let pending = client.pending_commit_ids();
client.conn.execute_batch("CREATE TRIGGER unrelated_insert BEFORE INSERT ON notes BEGIN SELECT RAISE(ABORT,'unrelated insert'); END;
CREATE TRIGGER unrelated_delete BEFORE DELETE ON notes BEGIN SELECT RAISE(ABORT,'unrelated delete'); END;").unwrap();
assert!(matches!(
client.apply_sqlite_image(
&image,
&table,
true,
&[("project_id".into(), vec!["p1".into()])],
10240,
7,
"digest"
),
Ok(10240)
));
assert_eq!(client.overlay_rebuild_count.get(), 0);
assert_eq!(client.pending_commit_ids(), pending);
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks", &[])
.unwrap()[0]["n"],
10241
);
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks_fts", &[])
.unwrap()[0]["n"],
10241
);
assert_eq!(
client
.query("SELECT count(*) AS n FROM notes_fts", &[])
.unwrap()[0]["n"],
8192
);
}
#[test]
#[ignore = "manual scaling benchmark; see bench/README.md"]
fn benchmark_unique_image_import() {
let mut samples = Vec::new();
for rows in [10240, 51200, 102400] {
for trial in 0..4 {
let (mut client, image, table) = unique_import_fixture(rows);
let start = std::time::Instant::now();
assert!(
matches!(client.apply_sqlite_image(&image,&table,true,&[("project_id".into(),vec!["p1".into()])],rows as i64,7,"digest"),Ok(n) if n as usize == rows)
);
let elapsed = start.elapsed().as_secs_f64() * 1000.0;
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks", &[])
.unwrap()[0]["n"],
rows + 1
);
assert_eq!(
client
.query("SELECT count(*) AS n FROM tasks_fts", &[])
.unwrap()[0]["n"],
rows + 1
);
assert_eq!(
client
.query("SELECT count(*) AS n FROM notes_fts", &[])
.unwrap()[0]["n"],
8192
);
assert_eq!(client.pending_commit_ids().len(), 1);
if trial > 0 {
samples.push(json!({"rows":rows,"trial":trial,"elapsedMs":elapsed,"fullRebuilds":client.overlay_rebuild_count.get()}));
}
}
}
let artifact = json!({"boundary":"Rust core; in-memory SQLite; one pending edit; secondary unique indexes and FTS; 8192 unrelated rows; one warmup per size","optimized":!cfg!(debug_assertions),"sqliteVersion":rusqlite::version(),"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"sourceHash":bytes_to_hex(&Sha256::digest(include_str!("client.rs").as_bytes())),"samples":samples});
let output = serde_json::to_string_pretty(&artifact).unwrap();
if let Ok(path) = std::env::var("SYNCULAR_IMPORT_BENCH_OUTPUT") {
use std::io::Write;
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.unwrap();
file.write_all(output.as_bytes()).unwrap();
} else {
println!("{output}");
}
}
#[test]
fn local_fts_projection_tracks_optimistic_overlay_rebuilds() {
let schema = json!({
"version": 1,
"tables": [{
"name": "catalogue_codes",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "release_id", "type": "string", "nullable": false },
{ "name": "code", "type": "string", "nullable": false },
{ "name": "title", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "release:{release_id}" }],
"ftsIndexes": [{
"name": "catalogue_codes_fts",
"columns": ["code", "title"],
"tokenize": "unicode61 remove_diacritics 2"
}]
}]
});
let mut client = SyncClient::new("fts-test".to_owned(), &schema, ClientLimits::default())
.expect("FTS5 client");
let insert_trigger: String = client
.conn
.query_row(
"SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_ai'",
[],
|row| row.get(0),
)
.expect("insert trigger");
let replace_guard: String = client
.conn
.query_row(
"SELECT sql FROM sqlite_master WHERE type='trigger' AND name='catalogue_codes_fts_bi'",
[],
|row| row.get(0),
)
.expect("replace guard");
assert!(!insert_trigger.contains("DELETE FROM"));
assert!(replace_guard.contains("BEFORE INSERT"));
assert!(replace_guard.contains("WHEN EXISTS"));
let search = |client: &SyncClient, query: &str| {
client
.query(
"SELECT c.id FROM catalogue_codes_fts f JOIN catalogue_codes c ON CAST(c.id AS TEXT) = f._syncular_source_id WHERE catalogue_codes_fts MATCH ?1 ORDER BY c.id",
&[Value::from(query)],
)
.expect("FTS query")
};
client
.mutate(vec![Mutation::Upsert {
table: "catalogue_codes".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from("c1")),
("release_id".to_owned(), Value::from("r1")),
("code".to_owned(), Value::from("A01")),
("title".to_owned(), Value::from("Cholera")),
]),
base_version: None,
}])
.expect("insert code");
assert_eq!(search(&client, "cholera").len(), 1);
let prior = client
.query("SELECT rowid, * FROM catalogue_codes_fts", &[])
.unwrap();
client
.conn
.execute_batch("DROP TABLE _syncular_fts_catalogue_codes_fts")
.unwrap();
client.create_synced_tables().unwrap();
assert_eq!(
client
.query("SELECT rowid, * FROM catalogue_codes_fts", &[])
.unwrap(),
prior
);
let plan = client.query("EXPLAIN QUERY PLAN SELECT rowid FROM catalogue_codes_fts WHERE rowid = (SELECT id FROM _syncular_fts_catalogue_codes_fts WHERE source_id = 'c1')", &[]).unwrap();
let rendered = serde_json::to_string(&plan).unwrap();
assert!(rendered.contains("COVERING INDEX"));
assert!(rendered.contains("INDEX 0:="));
client.conn.execute_batch("SAVEPOINT test_mapping; DELETE FROM catalogue_codes WHERE id='c1'; ROLLBACK TO test_mapping; RELEASE test_mapping").unwrap();
assert_eq!(search(&client, "cholera").len(), 1);
client
.mutate(vec![Mutation::Upsert {
table: "catalogue_codes".to_owned(),
values: Map::from_iter([
("id".to_owned(), Value::from("c1")),
("release_id".to_owned(), Value::from("r1")),
("code".to_owned(), Value::from("A01")),
("title".to_owned(), Value::from("Enteric infection")),
]),
base_version: None,
}])
.expect("update code");
assert!(search(&client, "cholera").is_empty());
assert_eq!(search(&client, "enteric").len(), 1);
client
.mutate(vec![Mutation::Delete {
table: "catalogue_codes".to_owned(),
row_id: "c1".to_owned(),
base_version: None,
}])
.expect("delete code");
assert!(search(&client, "enteric").is_empty());
}
#[test]
fn application_authorized_local_purge_is_exact_atomic_and_idempotent() {
let schema = json!({
"version": 1,
"tables": [{
"name": "patient_notes",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "practice_id", "type": "string", "nullable": false },
{ "name": "encryption_key_id", "type": "string", "nullable": false },
{ "name": "title", "type": "string", "nullable": false }
],
"scopes": [{ "pattern": "practice:{practice_id}" }],
"ftsIndexes": [{
"name": "patient_notes_fts",
"columns": ["title"],
"tokenize": "unicode61 remove_diacritics 2"
}]
}]
});
let mut client = SyncClient::new(
"local-purge-test".to_owned(),
&schema,
ClientLimits::default(),
)
.expect("local purge client");
client
.conn
.execute(
"INSERT INTO _syncular_base_patient_notes(id, practice_id, encryption_key_id, title, _syncular_version) VALUES
('target', 'practice-1', 'key-revoked', 'Target original', 1),
('unrelated', 'practice-1', 'key-held', 'Unrelated original', 1)",
[],
)
.expect("seed base rows");
client.overlay_dirty.set(true);
client.rebuild_overlay_if_dirty();
let note = |id: &str, key_id: &str, title: &str| {
Map::from_iter([
("id".to_owned(), Value::from(id)),
("practice_id".to_owned(), Value::from("practice-1")),
("encryption_key_id".to_owned(), Value::from(key_id)),
("title".to_owned(), Value::from(title)),
])
};
let doomed = client
.mutate(vec![
Mutation::Upsert {
table: "patient_notes".to_owned(),
values: note("target", "key-revoked", "Target changed"),
base_version: None,
},
Mutation::Upsert {
table: "patient_notes".to_owned(),
values: note("unrelated", "key-held", "Unrelated changed"),
base_version: None,
},
])
.expect("doomed commit");
let kept = client
.mutate(vec![Mutation::Upsert {
table: "patient_notes".to_owned(),
values: note("kept", "key-held", "Kept optimistic"),
base_version: None,
}])
.expect("kept commit");
client.drain_change_batches();
let input = LocalDataPurgeInput {
purge_id: "purge-001".to_owned(),
targets: vec![LocalDataPurgeTarget {
table: "patient_notes".to_owned(),
selectors: BTreeMap::from([(
"encryption_key_id".to_owned(),
vec!["key-revoked".to_owned()],
)]),
}],
};
assert_eq!(
client.purge_local_data(&input).expect("apply purge"),
LocalDataPurgeResult {
already_applied: false,
purged_rows: 1,
dropped_commits: 1,
}
);
let rows = client
.query("SELECT id, title FROM patient_notes ORDER BY id", &[])
.expect("visible rows");
assert_eq!(rows.len(), 2);
assert_eq!(rows[0]["id"], "kept");
assert_eq!(rows[1]["id"], "unrelated");
assert_eq!(rows[1]["title"], "Unrelated original");
let fts = client
.query(
"SELECT n.id FROM patient_notes_fts f JOIN patient_notes n ON CAST(n.id AS TEXT) = f._syncular_source_id WHERE patient_notes_fts MATCH 'target'",
&[],
)
.expect("fts query");
assert!(fts.is_empty());
assert_eq!(client.outbox.len(), 1);
assert_eq!(client.outbox[0].client_commit_id, kept);
let outcome = client
.commit_outcome(&doomed)
.expect("read doomed outcome")
.expect("doomed outcome");
assert_eq!(outcome.status, CommitOutcomeStatus::Rejected);
match &outcome.results[0] {
CommitOperationOutcome::Error { rejection } => {
assert_eq!(rejection.code, "client.local_data_purged");
assert_eq!(rejection.client_commit_id, doomed);
}
other => panic!("expected local purge rejection, got {other:?}"),
}
assert_eq!(client.drain_change_batches().len(), 1);
assert_eq!(
client.purge_local_data(&input).expect("retry purge"),
LocalDataPurgeResult {
already_applied: true,
purged_rows: 0,
dropped_commits: 0,
}
);
let conflicting = LocalDataPurgeInput {
purge_id: input.purge_id.clone(),
targets: vec![LocalDataPurgeTarget {
table: "patient_notes".to_owned(),
selectors: BTreeMap::from([(
"encryption_key_id".to_owned(),
vec!["key-held".to_owned()],
)]),
}],
};
assert!(client
.purge_local_data(&conflicting)
.expect_err("id collision must fail")
.contains("already used with a different plan"));
}
}
impl SubState {
fn name(self) -> &'static str {
match self {
SubState::Active => "active",
SubState::Revoked => "revoked",
SubState::Failed => "failed",
}
}
fn parse(value: &str) -> Self {
match value {
"revoked" => Self::Revoked,
"failed" => Self::Failed,
_ => Self::Active,
}
}
}
#[derive(Debug, Clone)]
struct Subscription {
id: String,
table: String,
requested: Vec<(String, Vec<String>)>,
params: Option<String>,
cursor: i64,
bootstrap_state: Option<String>,
state: SubState,
reason_code: Option<String>,
effective: Option<Vec<(String, Vec<String>)>>,
synced_once: bool,
}
#[derive(Debug, Clone)]
struct OutboxOp {
upsert: bool,
table: String,
row_id: String,
base_version: Option<i64>,
values: Option<Map<String, Value>>,
}
impl From<&OutboxOp> for CommitOperation {
fn from(operation: &OutboxOp) -> Self {
Self {
table: operation.table.clone(),
row_id: operation.row_id.clone(),
op: if operation.upsert { "upsert" } else { "delete" }.to_owned(),
base_version: operation.base_version,
values: operation.values.clone(),
}
}
}
#[derive(Debug, Clone)]
struct OutboxCommit {
client_commit_id: String,
ops: Vec<OutboxOp>,
}
fn commit_audit_operations(commit: &OutboxCommit) -> Vec<(&str, Vec<&str>)> {
commit
.ops
.iter()
.filter(|operation| operation.upsert)
.map(|operation| {
(
operation.table.as_str(),
operation
.values
.as_ref()
.map(|values| values.keys().map(String::as_str).collect())
.unwrap_or_default(),
)
})
.collect()
}
#[derive(Debug, Clone)]
struct CompiledLocalDataPurgeTarget {
table: String,
selectors: Vec<(String, Vec<String>)>,
}
struct StoredCommitOutcomeRow {
sequence: i64,
client_commit_id: String,
status: String,
recorded_at_ms: i64,
results_json: String,
operations_json: Option<String>,
resolution: String,
resolved_at_ms: Option<i64>,
replacement_client_commit_id: Option<String>,
}
enum SectionError {
FailClosed,
Abort(String, String),
}
struct RequestMeta {
pushed_ids: Vec<String>,
fresh: Vec<(String, bool)>,
accept: u8,
deferred_commits: usize,
}
#[derive(Default)]
struct ChangeAccumulator {
tables: BTreeMap<String, Option<BTreeSet<String>>>,
windows: BTreeMap<(String, String), BTreeSet<String>>,
status: bool,
conflicts: bool,
rejections: bool,
outcomes: bool,
}
impl ChangeAccumulator {
fn table(&mut self, table: &str) {
self.tables.insert(table.to_owned(), None);
}
fn scope(&mut self, table: &str, key: String) {
match self.tables.get_mut(table) {
Some(None) => {}
Some(Some(keys)) => {
keys.insert(key);
}
None => {
self.tables
.insert(table.to_owned(), Some(BTreeSet::from([key])));
}
}
}
fn window(&mut self, base_key: &str, table: &str, unit: &str) {
self.windows
.entry((base_key.to_owned(), table.to_owned()))
.or_default()
.insert(unit.to_owned());
}
fn touched(&self) -> bool {
!self.tables.is_empty()
|| !self.windows.is_empty()
|| self.status
|| self.conflicts
|| self.rejections
|| self.outcomes
}
}
const PUSH_OPS_PER_REQUEST: usize = 500;
const MAX_LOCAL_PURGE_TARGETS: usize = 64;
const MAX_LOCAL_PURGE_SELECTORS: usize = 8;
const MAX_LOCAL_PURGE_VALUES: usize = 128;
const MAX_LOCAL_PURGE_VALUE_LENGTH: usize = 256;
pub const SECURITY_PREFLIGHT_REQUIRED_CODE: &str = "client.security_preflight_required";
pub struct SyncClient {
#[cfg(feature = "bench-internals")]
benchmark_phases: Recorder,
conn: Connection,
schema: ClientSchema,
client_id: String,
limits: ClientLimits,
subs: Vec<Subscription>,
outbox: Vec<OutboxCommit>,
conflicts: Vec<ConflictRecord>,
rejections: Vec<RejectionRecord>,
schema_floor: Option<SchemaFloor>,
lease_state: Option<LeaseState>,
stopped: bool,
upgrading: bool,
sync_needed: bool,
realtime_policy: RealtimePolicy,
realtime_state: RealtimeState,
realtime_reason_code: Option<String>,
realtime_retry_delay_ms: Option<u64>,
presence: HashMap<String, HashMap<String, PresencePeer>>,
now_ms: Option<i64>,
encryption: crate::values::EncryptionConfig,
security_preflight: bool,
previous_version: Option<PreviousVersionContextConfig>,
insert_sql: RefCell<HashMap<String, String>>,
overlay_dirty: Cell<bool>,
#[cfg(test)]
overlay_rebuild_count: Cell<usize>,
#[cfg(test)]
outcome_prune_count: Cell<usize>,
progress: ProgressObserver,
change_queue: VecDeque<ClientChangeBatch>,
sync_intent_queue: VecDeque<SyncIntent>,
retry_delay_ms: u64,
round_retry_delay_ms: Option<u64>,
last_round: Option<DiagnosticLastRound>,
last_change: Option<DiagnosticLastChange>,
query_failures: Vec<DiagnosticQueryFailure>,
}
pub(crate) fn quote_ident(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}
pub(crate) fn meta_get(conn: &Connection, key: &str) -> Option<String> {
conn.query_row(
"SELECT value FROM _syncular_meta WHERE key = ?1",
rusqlite::params![key],
|row| row.get::<_, String>(0),
)
.ok()
}
pub(crate) fn meta_set(conn: &Connection, key: &str, value: &str) {
let _ = conn.execute(
"INSERT OR REPLACE INTO _syncular_meta (key, value) VALUES (?1, ?2)",
rusqlite::params![key, value],
);
}
pub(crate) fn meta_delete(conn: &Connection, key: &str) {
let _ = conn.execute(
"DELETE FROM _syncular_meta WHERE key = ?1",
rusqlite::params![key],
);
}
fn row_id_predicate(table: &TableSchema) -> String {
let key = quote_ident(&table.primary_key);
let text_match = format!("CAST({key} AS TEXT) = ?1");
match table.columns[table.pk_index].ty {
ColumnType::String | ColumnType::Json => {
format!("{key} = ?1 AND {text_match}")
}
ColumnType::Integer | ColumnType::Boolean => {
format!("{key} = +CAST(?1 AS INTEGER) AND {text_match}")
}
_ => text_match,
}
}
fn is_local_operation_code_like(value: &str) -> bool {
let bytes = value.as_bytes();
bytes.first().is_some_and(u8::is_ascii_alphanumeric)
&& bytes
.iter()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(*byte, b'.' | b'_' | b':' | b'-'))
}
fn base_table(name: &str) -> String {
quote_ident(&format!("_syncular_base_{name}"))
}
type PendingEvict = (String, String, Vec<(String, Vec<String>)>);
fn window_base_key(base: &WindowBase) -> String {
format!(
"{}\0{}\0{}",
base.table,
base.variable,
canonical_scope_json(&base.fixed_scopes)
)
}
fn unit_scopes(base: &WindowBase, unit: &str) -> Vec<(String, Vec<String>)> {
let mut scopes = base.fixed_scopes.clone();
scopes.retain(|(k, _)| k != &base.variable);
scopes.push((base.variable.clone(), vec![unit.to_owned()]));
scopes
}
fn derive_sub_id(base: &WindowBase, unit: &str) -> String {
let canonical = canonical_scope_json(&unit_scopes(base, unit));
let digest = Sha256::digest(canonical.as_bytes());
let hex = bytes_to_hex(&digest);
format!("w:{}:{}", base.table, &hex[..16])
}
fn visible_table(name: &str) -> String {
quote_ident(name)
}
const FTS_SOURCE_ID_COLUMN: &str = "_syncular_source_id";
fn is_synced_table_name(name: &str) -> bool {
if name.starts_with("sqlite_") {
return false;
}
if name.starts_with("_syncular_base_") {
return true; }
!name.starts_with("_syncular_")
}
fn blob_id_for(bytes: &[u8]) -> String {
let digest = Sha256::digest(bytes);
format!("sha256:{}", bytes_to_hex(&digest))
}
enum RowParam<'a> {
Cell(&'a Option<ColumnValue>),
Version(i64),
}
impl rusqlite::ToSql for RowParam<'_> {
fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
Ok(match self {
RowParam::Version(v) => ToSqlOutput::Owned(SqlValue::Integer(*v)),
RowParam::Cell(cell) => match cell {
None => ToSqlOutput::Owned(SqlValue::Null),
Some(ColumnValue::String(s)) => ToSqlOutput::Borrowed(ValueRef::Text(s.as_bytes())),
Some(ColumnValue::Integer(i)) => ToSqlOutput::Owned(SqlValue::Integer(*i)),
Some(ColumnValue::Float(f)) => ToSqlOutput::Owned(SqlValue::Real(*f)),
Some(ColumnValue::Boolean(b)) => {
ToSqlOutput::Owned(SqlValue::Integer(i64::from(*b)))
}
Some(ColumnValue::Json(raw)) | Some(ColumnValue::BlobRef(raw)) => {
ToSqlOutput::Borrowed(ValueRef::Text(raw.0.as_bytes()))
}
Some(ColumnValue::Bytes(b)) | Some(ColumnValue::Crdt(b)) => {
ToSqlOutput::Borrowed(ValueRef::Blob(b))
}
},
})
}
}
fn owned_sql_value(param: impl rusqlite::ToSql) -> Result<SqlValue, String> {
match param.to_sql().map_err(|error| error.to_string())? {
ToSqlOutput::Owned(value) => Ok(value),
ToSqlOutput::Borrowed(value) => {
SqlValue::try_from(value).map_err(|error| error.to_string())
}
_ => Err("sync.invalid_request: unsupported SQLite key parameter".into()),
}
}
fn image_cell_param<'a>(column: &Column, value: ValueRef<'a>) -> Result<ToSqlOutput<'a>, String> {
use ssp2::segment::ColumnType;
let mismatch = || {
Err(format!(
"image column {:?} holds a value of the wrong type",
column.name
))
};
match value {
ValueRef::Null => {
if !column.nullable {
return Err(format!(
"image column {:?} is NULL but not nullable",
column.name
));
}
Ok(ToSqlOutput::Owned(SqlValue::Null))
}
ValueRef::Integer(i) => match column.ty {
ColumnType::Integer => Ok(ToSqlOutput::Borrowed(value)),
ColumnType::Boolean => Ok(ToSqlOutput::Owned(SqlValue::Integer(i64::from(i != 0)))),
ColumnType::Float => Ok(ToSqlOutput::Owned(SqlValue::Real(i as f64))),
_ => mismatch(),
},
ValueRef::Real(_) => match column.ty {
ColumnType::Float => Ok(ToSqlOutput::Borrowed(value)),
_ => mismatch(),
},
ValueRef::Text(t) => {
std::str::from_utf8(t)
.map_err(|_| format!("image column {:?} is not UTF-8", column.name))?;
match column.ty {
ColumnType::String | ColumnType::Json | ColumnType::BlobRef => {
Ok(ToSqlOutput::Borrowed(value))
}
_ => mismatch(),
}
}
ValueRef::Blob(_) => match column.ty {
ColumnType::Bytes | ColumnType::Crdt => Ok(ToSqlOutput::Borrowed(value)),
_ => mismatch(),
},
}
}
pub(crate) fn sql_ref_to_json(column: &Column, value: rusqlite::types::ValueRef<'_>) -> Value {
use rusqlite::types::ValueRef;
match value {
ValueRef::Null => Value::Null,
ValueRef::Integer(i) => match column.ty {
ssp2::segment::ColumnType::Boolean => Value::Bool(i != 0),
ssp2::segment::ColumnType::Float => {
serde_json::Number::from_f64(i as f64).map_or(Value::Null, Value::Number)
}
_ => Value::from(i),
},
ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
ValueRef::Blob(b) => {
let mut map = Map::new();
map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
Value::Object(map)
}
}
}
fn json_param_to_sql(value: &Value) -> Result<SqlValue, String> {
Ok(match value {
Value::Null => SqlValue::Null,
Value::Bool(b) => SqlValue::Integer(i64::from(*b)),
Value::Number(n) => {
if let Some(i) = n.as_i64() {
SqlValue::Integer(i)
} else if let Some(f) = n.as_f64() {
SqlValue::Real(f)
} else {
return Err(format!("query param number {n} is out of range"));
}
}
Value::String(s) => SqlValue::Text(s.clone()),
Value::Object(_) => {
if let Some(hex) = value.get("$bytes").and_then(Value::as_str) {
SqlValue::Blob(crate::values::hex_to_bytes(hex)?)
} else if let Some(decimal) = value.get("$bigint").and_then(Value::as_str) {
SqlValue::Integer(decimal.parse::<i64>().map_err(|_| {
format!("query bigint param {decimal:?} is outside SQLite's i64 range")
})?)
} else {
return Err(
"query object param must be a {$bytes: hex} or {$bigint: decimal} value"
.to_owned(),
);
}
}
Value::Array(_) => return Err("query array params are not supported".to_owned()),
})
}
fn sql_ref_to_json_dynamic(value: rusqlite::types::ValueRef<'_>) -> Value {
use rusqlite::types::ValueRef;
match value {
ValueRef::Null => Value::Null,
ValueRef::Integer(i) => {
const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991;
if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&i) {
Value::from(i)
} else {
let mut map = Map::new();
map.insert("$bigint".to_owned(), Value::from(i.to_string()));
Value::Object(map)
}
}
ValueRef::Real(f) => serde_json::Number::from_f64(f).map_or(Value::Null, Value::Number),
ValueRef::Text(t) => Value::from(String::from_utf8_lossy(t).into_owned()),
ValueRef::Blob(b) => {
let mut map = Map::new();
map.insert("$bytes".to_owned(), Value::from(bytes_to_hex(b)));
Value::Object(map)
}
}
}
fn query_connection(
conn: &Connection,
sql: &str,
params: &[QueryValue],
) -> Result<Vec<QueryRow>, QueryReadFailure> {
crate::query_guard::assert_read_only_query(sql)?;
let lowered_sql = crate::query_guard::lower_public_query_sql(sql);
let bound: Vec<SqlValue> = params
.iter()
.map(json_param_to_sql)
.collect::<Result<_, _>>()?;
let mut stmt = conn.prepare(&lowered_sql)?;
let column_names: Vec<String> = stmt.column_names().into_iter().map(str::to_owned).collect();
let bound_refs: Vec<&dyn rusqlite::ToSql> =
bound.iter().map(|v| v as &dyn rusqlite::ToSql).collect();
let mut sql_rows = stmt.query(bound_refs.as_slice())?;
let mut out = Vec::new();
while let Some(row) = sql_rows.next()? {
let mut record = Map::new();
for (i, name) in column_names.iter().enumerate() {
let value = row.get_ref(i)?;
record.insert(name.clone(), sql_ref_to_json_dynamic(value));
}
out.push(record);
}
Ok(out)
}
fn persisted_window_state(
conn: &Connection,
base: &WindowBase,
) -> Result<WindowState, QueryReadFailure> {
let mut stmt = conn.prepare(
"SELECT windows.unit, subscriptions.state_json
FROM _syncular_windows AS windows
JOIN _syncular_subscriptions AS subscriptions
ON subscriptions.id = windows.sub_id
WHERE windows.base = ?1
ORDER BY windows.unit ASC",
)?;
let rows = stmt.query_map(rusqlite::params![window_base_key(base)], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})?;
let mut units = Vec::new();
let mut pending = Vec::new();
for row in rows {
let (unit, raw) = row?;
let state: Value = serde_json::from_str(&raw)
.map_err(|error| format!("invalid persisted window subscription: {error}"))?;
let is_pending = state.get("status").and_then(Value::as_str) != Some("active")
|| state.get("cursor").and_then(Value::as_i64).unwrap_or(-1) < 0
|| state
.get("bootstrapState")
.is_some_and(|value| !value.is_null());
if is_pending {
pending.push(unit.clone());
}
units.push(unit);
}
Ok(WindowState { units, pending })
}
fn snapshot_connection(
conn: &Connection,
sql: &str,
params: &[Value],
coverage: &[WindowCoverage],
) -> Result<QuerySnapshot, QueryReadFailure> {
conn.execute_batch("SAVEPOINT syncular_snapshot_read")?;
let result = (|| {
let revision = conn
.query_row(
"SELECT value FROM _syncular_meta WHERE key = ?1",
rusqlite::params![LOCAL_REVISION_KEY],
|row| row.get::<_, String>(0),
)
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(0);
let rows = query_connection(conn, sql, params)?;
let mut pending = Vec::new();
let mut missing = Vec::new();
for requested in coverage {
let base_key = window_base_key(&requested.base);
let state = persisted_window_state(conn, &requested.base)?;
for unit in BTreeSet::from_iter(requested.units.iter().cloned()) {
let reference = WindowUnitRef {
base_key: base_key.clone(),
unit: unit.clone(),
};
if !state.units.iter().any(|held| held == &unit) {
missing.push(reference);
} else if state.pending.iter().any(|held| held == &unit) {
pending.push(reference);
}
}
}
Ok(QuerySnapshot {
revision: revision.to_string(),
rows,
coverage: CoverageSnapshot {
complete: pending.is_empty() && missing.is_empty(),
pending,
missing,
},
})
})();
match result {
Ok(snapshot) => {
conn.execute_batch("RELEASE syncular_snapshot_read")?;
Ok(snapshot)
}
Err(error) => {
let _ = conn.execute_batch(
"ROLLBACK TO syncular_snapshot_read; RELEASE syncular_snapshot_read",
);
Err(error)
}
}
}
pub struct FileQuerySnapshotReader {
path: String,
conn: Option<Connection>,
}
impl FileQuerySnapshotReader {
#[must_use]
pub fn new(path: impl Into<String>) -> Self {
Self {
path: path.into(),
conn: None,
}
}
fn connection(&mut self) -> Result<&Connection, String> {
if self.conn.is_none() {
let conn = Connection::open_with_flags(
&self.path,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)
.map_err(|error| format!("open read sidecar {:?}: {error}", self.path))?;
conn.busy_timeout(std::time::Duration::from_millis(250))
.map_err(|error| error.to_string())?;
self.conn = Some(conn);
}
self.conn
.as_ref()
.ok_or_else(|| "read sidecar connection missing".to_owned())
}
pub fn query_snapshot(
&mut self,
sql: &str,
params: &[Value],
coverage: &[WindowCoverage],
) -> Result<QuerySnapshot, QueryReadFailure> {
snapshot_connection(self.connection()?, sql, params, coverage)
}
}
impl SyncClient {
pub fn new_with_identity(
client_id: Option<String>,
schema_json: &Value,
limits: ClientLimits,
) -> Result<Self, String> {
let resolved = client_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
Self::with_connection(resolved, schema_json, limits, conn)
}
pub fn new(
client_id: String,
schema_json: &Value,
limits: ClientLimits,
) -> Result<Self, String> {
let conn = Connection::open_in_memory().map_err(|e| e.to_string())?;
Self::with_connection(client_id, schema_json, limits, conn)
}
pub fn open_path(
client_id: String,
schema_json: &Value,
limits: ClientLimits,
path: &str,
) -> Result<Self, String> {
let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
Self::with_connection(client_id, schema_json, limits, conn)
}
pub fn open_path_with_identity(
client_id: Option<String>,
schema_json: &Value,
limits: ClientLimits,
path: &str,
) -> Result<Self, String> {
let conn = Connection::open(path).map_err(|e| format!("open db {path:?}: {e}"))?;
conn.busy_timeout(std::time::Duration::from_millis(250))
.map_err(|error| format!("configure db {path:?} busy timeout: {error}"))?;
conn.pragma_update(None, "journal_mode", "WAL")
.map_err(|error| format!("configure db {path:?} WAL mode: {error}"))?;
let persisted = conn
.query_row(
"SELECT value FROM _syncular_meta WHERE key = 'clientId'",
[],
|row| row.get::<_, String>(0),
)
.ok();
let resolved = persisted
.clone()
.or(client_id.clone())
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
if let (Some(existing), Some(requested)) = (persisted, client_id) {
if existing != requested {
return Err(format!(
"client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {requested:?}"
));
}
}
Self::with_connection(resolved, schema_json, limits, conn)
}
#[must_use]
pub fn client_id(&self) -> &str {
&self.client_id
}
pub fn with_connection(
client_id: String,
schema_json: &Value,
limits: ClientLimits,
conn: Connection,
) -> Result<Self, String> {
if limits.outcome_retention_max_entries == Some(0) {
return Err(
"sync.invalid_request: outcomeRetentionMaxEntries must be positive".to_owned(),
);
}
let schema = parse_schema_json(schema_json)?;
let previous_version = match limits.previous_version_context {
Some(config) => {
config.validate()?;
Some(config)
}
None => None,
};
let mut client = SyncClient {
#[cfg(feature = "bench-internals")]
benchmark_phases: Recorder::default(),
conn,
schema,
client_id,
limits,
subs: Vec::new(),
outbox: Vec::new(),
conflicts: Vec::new(),
rejections: Vec::new(),
schema_floor: None,
lease_state: None,
stopped: false,
upgrading: false,
sync_needed: false,
realtime_policy: RealtimePolicy::Optional,
realtime_state: RealtimeState::Disconnected,
realtime_reason_code: None,
realtime_retry_delay_ms: None,
presence: HashMap::new(),
now_ms: None,
encryption: crate::values::EncryptionConfig::default(),
security_preflight: false,
previous_version,
insert_sql: RefCell::new(HashMap::new()),
overlay_dirty: Cell::new(false),
#[cfg(test)]
overlay_rebuild_count: Cell::new(0),
#[cfg(test)]
outcome_prune_count: Cell::new(0),
progress: ProgressObserver::default(),
change_queue: VecDeque::new(),
sync_intent_queue: VecDeque::new(),
retry_delay_ms: 250,
round_retry_delay_ms: None,
last_round: None,
last_change: None,
query_failures: Vec::new(),
};
client
.conn
.set_prepared_statement_cache_capacity(64.max(client.schema.tables.len() * 4));
client.create_bookkeeping_tables()?;
match client.get_meta(CLIENT_ID_KEY) {
Some(existing) if existing != client.client_id => {
return Err(format!(
"client.identity_mismatch: this database belongs to {existing:?}; refusing to rebind it to {:?}",
client.client_id
));
}
None => client.set_meta(CLIENT_ID_KEY, &client.client_id),
_ => {}
}
client.restore_persisted_state()?;
let marker = client
.get_meta(LOCAL_SCHEMA_VERSION_KEY)
.and_then(|value| value.parse::<i32>().ok());
match marker {
None => {
client.create_synced_tables()?;
client.set_meta(LOCAL_SCHEMA_VERSION_KEY, &client.schema.version.to_string());
set_local_schema_descriptor(&client.conn, &client.schema);
}
Some(version) if version == client.schema.version => {
client.create_synced_tables()?;
set_local_schema_descriptor(&client.conn, &client.schema);
}
Some(_) => client.run_schema_reset()?,
}
client.reconcile_previous_version_at_boot()?;
client.clear_satisfied_persisted_schema_floor();
client.prune_unknown_subscriptions()?;
if marker == Some(client.schema.version) && !client.outbox.is_empty() {
client.overlay_dirty.set(true);
client.rebuild_overlay();
}
client.enqueue_startup_sync_if_needed()?;
Ok(client)
}
pub fn set_now_ms(&mut self, now_ms: i64) {
self.now_ms = Some(now_ms);
}
pub fn set_encryption(&mut self, encryption: crate::values::EncryptionConfig) {
self.encryption = encryption;
}
#[must_use]
pub fn security_lifecycle(&self) -> &'static str {
if self.security_preflight {
"preflight"
} else {
"active"
}
}
#[must_use]
pub fn security_preflight(&self) -> bool {
self.security_preflight
}
pub fn begin_security_preflight(&mut self) {
self.seal_security_on_teardown();
self.set_meta(SECURITY_PREFLIGHT_PENDING_KEY, "1");
}
pub fn seal_security_on_teardown(&mut self) {
self.security_preflight = true;
self.encryption = crate::values::EncryptionConfig::default();
self.sync_intent_queue.clear();
}
pub fn activate_security(
&mut self,
encryption: crate::values::EncryptionConfig,
) -> Result<(), String> {
if !self.security_preflight {
return Err(
"sync.invalid_request: activateSecurity requires security preflight".to_owned(),
);
}
self.enqueue_startup_sync_if_needed()?;
self.encryption = encryption;
self.security_preflight = false;
self.delete_meta(SECURITY_PREFLIGHT_PENDING_KEY);
Ok(())
}
fn clock_now_ms(&self) -> i64 {
self.now_ms.unwrap_or_else(|| {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as i64)
.unwrap_or(0)
})
}
fn create_bookkeeping_tables(&self) -> Result<(), String> {
self.conn
.execute_batch(
"CREATE TABLE IF NOT EXISTS _syncular_outbox (
seq INTEGER PRIMARY KEY AUTOINCREMENT,
commit_id TEXT NOT NULL UNIQUE, ops_json TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS _syncular_commit_outcomes (
seq INTEGER PRIMARY KEY AUTOINCREMENT,
client_commit_id TEXT NOT NULL UNIQUE,
status TEXT NOT NULL CHECK(status IN ('applied', 'cached', 'conflict', 'rejected')),
recorded_at_ms INTEGER NOT NULL,
results_json TEXT NOT NULL,
operations_json TEXT,
resolution TEXT NOT NULL DEFAULT 'active'
CHECK(resolution IN ('active', 'resolved_keep_server', 'superseded', 'dismissed')),
resolved_at_ms INTEGER,
replacement_client_commit_id TEXT);
CREATE INDEX IF NOT EXISTS _syncular_commit_outcomes_resolution_seq
ON _syncular_commit_outcomes(resolution, seq);
CREATE TABLE IF NOT EXISTS _syncular_subscriptions (
id TEXT PRIMARY KEY, tbl TEXT NOT NULL, state_json TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS _syncular_meta (
key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS _syncular_windows (
base TEXT NOT NULL, unit TEXT NOT NULL, sub_id TEXT NOT NULL,
PRIMARY KEY (base, unit));
CREATE TABLE IF NOT EXISTS _syncular_window_pending_evict (
sub_id TEXT PRIMARY KEY, tbl TEXT NOT NULL,
effective_scopes TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL,
PRIMARY KEY(commit_id, blob_id));
CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body
ON _syncular_blob_commit_refs(blob_id);",
)
.map_err(|e| e.to_string())?;
let _ = self
.conn
.execute_batch("ALTER TABLE _syncular_commit_outcomes ADD COLUMN operations_json TEXT");
if self.get_meta(LOCAL_REVISION_KEY).is_none() {
self.set_meta(LOCAL_REVISION_KEY, "0");
}
if self.schema_has_blobs() {
let exists = self
.conn
.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs')",
[],
|row| row.get::<_, bool>(0),
)
.map_err(|e| e.to_string())?;
if exists {
let mut statement = self
.conn
.prepare("PRAGMA table_info(_syncular_blobs)")
.map_err(|e| e.to_string())?;
let mut columns = statement
.query_map([], |row| row.get::<_, String>(1))
.map_err(|e| e.to_string())?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| e.to_string())?;
columns.sort();
let mut expected = vec![
"blob_id",
"byte_length",
"bytes",
"created_at_ms",
"media_type",
];
expected.sort_unstable();
if columns != expected {
return Err(
"sync.schema_mismatch: Local blob schema is incompatible".to_owned()
);
}
} else {
self.conn
.execute_batch(
"CREATE TABLE _syncular_blobs(
blob_id TEXT PRIMARY KEY,
bytes BLOB NOT NULL,
byte_length INTEGER NOT NULL,
media_type TEXT,
created_at_ms INTEGER NOT NULL);",
)
.map_err(|e| e.to_string())?;
}
self.conn
.execute_batch(
"CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
blob_id TEXT PRIMARY KEY,
media_type TEXT,
created_at_ms INTEGER NOT NULL);",
)
.map_err(|e| e.to_string())?;
}
Ok(())
}
fn schema_has_blobs(&self) -> bool {
self.schema
.tables
.iter()
.any(|t| t.columns.iter().any(|c| c.ty == ColumnType::BlobRef))
}
fn get_meta(&self, key: &str) -> Option<String> {
meta_get(&self.conn, key)
}
fn get_meta_strict(&self, key: &str) -> Result<Option<String>, String> {
self.conn
.query_row(
"SELECT value FROM _syncular_meta WHERE key = ?1",
rusqlite::params![key],
|row| row.get::<_, String>(0),
)
.optional()
.map_err(|_| {
"sync.local_corrupt: persisted local rebootstrap receipt is unreadable".to_owned()
})
}
fn set_meta(&self, key: &str, value: &str) {
meta_set(&self.conn, key, value);
}
fn delete_meta(&self, key: &str) {
meta_delete(&self.conn, key);
}
fn restore_persisted_state(&mut self) -> Result<(), String> {
self.subs = {
let mut stmt = self
.conn
.prepare("SELECT id, tbl, state_json FROM _syncular_subscriptions ORDER BY id ASC")
.map_err(|error| error.to_string())?;
let rows = stmt
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
))
})
.map_err(|error| error.to_string())?;
let mut subscriptions = Vec::new();
for row in rows {
let (id, table, raw) = row.map_err(|error| error.to_string())?;
let state: Value = serde_json::from_str(&raw)
.map_err(|error| format!("invalid persisted subscription {id:?}: {error}"))?;
let requested = json_to_scope_map(
state.get("requested").unwrap_or(&Value::Object(Map::new())),
)?;
let effective = state
.get("effectiveScopes")
.filter(|value| !value.is_null())
.map(json_to_scope_map)
.transpose()?;
subscriptions.push(Subscription {
id,
table,
requested,
params: state
.get("params")
.and_then(Value::as_str)
.map(str::to_owned),
cursor: state.get("cursor").and_then(Value::as_i64).unwrap_or(-1),
bootstrap_state: state
.get("bootstrapState")
.and_then(Value::as_str)
.map(str::to_owned),
state: SubState::parse(
state
.get("status")
.and_then(Value::as_str)
.unwrap_or("active"),
),
reason_code: state
.get("reasonCode")
.and_then(Value::as_str)
.map(str::to_owned),
effective,
synced_once: state
.get("syncedOnce")
.and_then(Value::as_bool)
.unwrap_or(false),
});
}
subscriptions
};
self.outbox = {
let mut stmt = self
.conn
.prepare("SELECT commit_id, ops_json FROM _syncular_outbox ORDER BY seq ASC")
.map_err(|error| error.to_string())?;
let rows = stmt
.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
})
.map_err(|error| error.to_string())?;
let mut commits = Vec::new();
for row in rows {
let (client_commit_id, raw) = row.map_err(|error| error.to_string())?;
let entries: Vec<Value> = serde_json::from_str(&raw).map_err(|error| {
format!("invalid persisted outbox {client_commit_id:?}: {error}")
})?;
let mut ops = Vec::with_capacity(entries.len());
for entry in entries {
let op = entry.get("op").and_then(Value::as_str).unwrap_or("delete");
ops.push(OutboxOp {
upsert: op == "upsert",
table: entry
.get("table")
.and_then(Value::as_str)
.ok_or_else(|| "persisted outbox operation missing table".to_owned())?
.to_owned(),
row_id: entry
.get("rowId")
.and_then(Value::as_str)
.ok_or_else(|| "persisted outbox operation missing rowId".to_owned())?
.to_owned(),
base_version: entry.get("baseVersion").and_then(Value::as_i64),
values: entry.get("values").and_then(Value::as_object).cloned(),
});
}
commits.push(OutboxCommit {
client_commit_id,
ops,
});
}
commits
};
self.prune_commit_outcomes()?;
let active = self.commit_outcomes(CommitOutcomeQuery {
active_only: true,
..CommitOutcomeQuery::default()
})?;
self.conflicts = active
.iter()
.flat_map(|outcome| outcome.results.iter())
.filter_map(|result| match result {
CommitOperationOutcome::Conflict { conflict } => Some(conflict.clone()),
_ => None,
})
.collect();
self.rejections = active
.iter()
.flat_map(|outcome| outcome.results.iter())
.filter_map(|result| match result {
CommitOperationOutcome::Error { rejection } => Some(rejection.clone()),
_ => None,
})
.collect();
self.lease_state = self
.get_meta(LEASE_STATE_KEY)
.map(|raw| serde_json::from_str(&raw))
.transpose()
.map_err(|error| format!("invalid persisted lease state: {error}"))?;
self.schema_floor = self
.get_meta(SCHEMA_FLOOR_KEY)
.map(|raw| serde_json::from_str(&raw))
.transpose()
.map_err(|error| format!("invalid persisted schema floor: {error}"))?;
self.stopped = self.schema_floor.is_some();
if self.get_meta(SECURITY_PREFLIGHT_PENDING_KEY).as_deref() == Some("1") {
self.security_preflight = true;
}
Ok(())
}
fn prune_unknown_subscriptions(&mut self) -> Result<(), String> {
let valid_tables: BTreeSet<String> = self
.schema
.tables
.iter()
.map(|table| table.name.clone())
.collect();
let stale_ids: Vec<String> = self
.subs
.iter()
.filter(|sub| !valid_tables.contains(&sub.table))
.map(|sub| sub.id.clone())
.collect();
for id in &stale_ids {
self.conn
.execute(
"DELETE FROM _syncular_windows WHERE sub_id = ?1",
rusqlite::params![id],
)
.map_err(|error| error.to_string())?;
self.conn
.execute(
"DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
rusqlite::params![id],
)
.map_err(|error| error.to_string())?;
self.conn
.execute(
"DELETE FROM _syncular_subscriptions WHERE id = ?1",
rusqlite::params![id],
)
.map_err(|error| error.to_string())?;
}
self.subs.retain(|sub| valid_tables.contains(&sub.table));
Ok(())
}
#[must_use]
pub fn local_revision(&self) -> u64 {
self.get_meta(LOCAL_REVISION_KEY)
.and_then(|value| value.parse().ok())
.unwrap_or(0)
}
#[must_use]
pub fn status_snapshot(&self) -> SyncStatusSnapshot {
SyncStatusSnapshot {
current_schema_version: self.schema.version,
outbox: self.outbox.len(),
upgrading: self.upgrading,
lease_state: self.lease_state.clone(),
schema_floor: self.schema_floor.clone(),
sync_needed: self.sync_needed,
previous_version_context: self.previous_version_status(),
}
}
fn previous_version_status(&self) -> PreviousVersionStatus {
let Some(replica_path) = self.previous_version_replica_path() else {
return PreviousVersionStatus {
present: false,
created_at_ms: None,
};
};
match crate::previous_version::read_previous_version_record(&replica_path) {
Ok(Some(record)) => PreviousVersionStatus {
present: true,
created_at_ms: Some(record.created_at_ms),
},
_ => PreviousVersionStatus {
present: std::path::Path::new(
&crate::previous_version::previous_version_container_path(&replica_path),
)
.exists(),
created_at_ms: None,
},
}
}
pub fn previous_version_snapshot(
&self,
spec: &PreviousVersionReadSpec,
) -> Result<PreviousVersionSnapshot, String> {
let now_ms = self.clock_now_ms();
crate::previous_version::previous_version_snapshot(
&self.conn,
self.previous_version_replica_path().as_deref(),
self.previous_version.as_ref(),
PreviousVersionLifecycle {
lease_inactive: self.previous_version_lease_inactive(now_ms),
scope_revoked: self.subs.iter().any(|sub| sub.state == SubState::Revoked),
coverage_complete: self.previous_version_coverage_complete(),
},
now_ms,
self.schema.version,
spec,
)
}
#[must_use]
pub fn previous_version_audit(&self) -> Option<crate::previous_version::PreviousVersionAudit> {
crate::previous_version::stored_previous_version_audit(&self.conn)
}
pub fn previous_version_discard(
&self,
) -> crate::previous_version::PreviousVersionDiscardOutcome {
self.drop_previous_version()
}
fn drop_previous_version(&self) -> crate::previous_version::PreviousVersionDiscardOutcome {
let absent = crate::previous_version::PreviousVersionDiscardOutcome {
present: false,
discarded: false,
};
let Some(replica_path) = self.previous_version_replica_path() else {
return absent;
};
crate::previous_version::discard_previous_version(&self.conn, &replica_path)
.unwrap_or(absent)
}
fn previous_version_lease_inactive(&self, now_ms: i64) -> bool {
let Some(lease) = &self.lease_state else {
return false;
};
if lease.error_code.is_some() {
return true;
}
lease.expires_at_ms.is_some_and(|expires| expires <= now_ms)
}
fn previous_version_coverage_complete(&self) -> bool {
let active = self
.subs
.iter()
.filter(|sub| sub.state == SubState::Active)
.collect::<Vec<_>>();
if active.is_empty() {
return false;
}
active
.iter()
.all(|sub| sub.cursor >= 0 && sub.bootstrap_state.is_none())
}
fn previous_version_lifetime_check(&self) {
let Some(config) = self.previous_version.filter(|config| config.enabled) else {
return;
};
let now_ms = self.clock_now_ms();
if self.previous_version_lease_inactive(now_ms) {
let _ = self.drop_previous_version();
return;
}
let Some(replica_path) = self.previous_version_replica_path() else {
return;
};
let Ok(Some(record)) = crate::previous_version::read_previous_version_record(&replica_path)
else {
return;
};
if self.subs.iter().any(|sub| sub.state == SubState::Revoked) {
let _ = self.drop_previous_version();
return;
}
if now_ms - record.created_at_ms > config.max_age_ms {
let _ = self.drop_previous_version();
return;
}
if self.previous_version_coverage_complete() {
let _ = self.drop_previous_version();
}
}
pub fn diagnostics_snapshot(
&self,
request: &ClientDiagnosticsRequest,
) -> Result<ClientDiagnosticsSnapshot, String> {
if request.expected_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS {
return Err(format!(
"sync.invalid_request: diagnosticsSnapshot accepts at most {MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS} expected subscriptions"
));
}
let mut subscriptions = BTreeMap::<String, DiagnosticSubscription>::new();
for sub in &self.subs {
let reset = sub.cursor < 0 && sub.reason_code.as_deref() == Some("sync.cursor_expired");
let complete =
sub.state == SubState::Active && sub.cursor >= 0 && sub.bootstrap_state.is_none();
let state = match sub.state {
SubState::Revoked => "revoked",
SubState::Failed => "failed",
SubState::Active if reset => "reset",
SubState::Active if complete => "complete",
SubState::Active => "bootstrapping",
};
subscriptions.insert(
sub.id.clone(),
DiagnosticSubscription {
id: sub.id.clone(),
table: sub.table.clone(),
state: state.to_owned(),
complete,
cursor: Some(sub.cursor),
reason_code: sub.reason_code.as_deref().map(Self::diagnostic_code),
},
);
}
for expected in &request.expected_subscriptions {
if expected.id.is_empty() || expected.table.is_empty() {
return Err("sync.invalid_request: diagnosticsSnapshot expected subscriptions require non-empty id and table strings".to_owned());
}
if subscriptions
.get(&expected.id)
.is_some_and(|registered| registered.table != expected.table)
{
subscriptions.insert(
expected.id.clone(),
DiagnosticSubscription {
id: expected.id.clone(),
table: expected.table.clone(),
state: "failed".to_owned(),
complete: false,
cursor: None,
reason_code: Some("client.subscription_intent_mismatch".to_owned()),
},
);
} else {
subscriptions.entry(expected.id.clone()).or_insert_with(|| {
DiagnosticSubscription {
id: expected.id.clone(),
table: expected.table.clone(),
state: "unregistered".to_owned(),
complete: false,
cursor: None,
reason_code: None,
}
});
}
}
let mut ordered_subscriptions = Vec::new();
let mut included = BTreeSet::new();
for expected in &request.expected_subscriptions {
if included.insert(expected.id.clone()) {
if let Some(subscription) = subscriptions.get(&expected.id) {
ordered_subscriptions.push(subscription.clone());
}
}
}
for (id, subscription) in &subscriptions {
if included.insert(id.clone()) {
ordered_subscriptions.push(subscription.clone());
}
}
let subscriptions_truncated =
ordered_subscriptions.len() > MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS;
ordered_subscriptions.truncate(MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS);
let captured_at_ms = self.clock_now_ms();
let lease = if let Some(error_code) = self
.lease_state
.as_ref()
.and_then(|state| state.error_code.clone())
{
ClientDiagnosticsLease {
state: "stopped".to_owned(),
expires_at_ms: self
.lease_state
.as_ref()
.and_then(|state| state.expires_at_ms),
error_code: Some(Self::diagnostic_code(&error_code)),
}
} else if let Some(expires_at_ms) = self
.lease_state
.as_ref()
.and_then(|state| state.expires_at_ms)
{
ClientDiagnosticsLease {
state: if expires_at_ms <= captured_at_ms {
"expired".to_owned()
} else {
"active".to_owned()
},
expires_at_ms: Some(expires_at_ms),
error_code: None,
}
} else {
ClientDiagnosticsLease {
state: "none".to_owned(),
expires_at_ms: None,
error_code: None,
}
};
let connectivity = match self.last_round.as_ref() {
Some(round) if round.status == "succeeded" => "online",
Some(round)
if round.status == "failed"
&& round
.error_code
.as_deref()
.is_some_and(Self::retryable_transport_code) =>
{
"offline"
}
_ => "unknown",
};
Ok(ClientDiagnosticsSnapshot {
version: CLIENT_DIAGNOSTICS_VERSION,
captured_at_ms,
host: ClientDiagnosticsHost {
kind: "direct".to_owned(),
role: "single".to_owned(),
connectivity: connectivity.to_owned(),
realtime: self.realtime_state,
realtime_policy: self.realtime_policy,
realtime_reason_code: self.realtime_reason_code.clone(),
realtime_retry_delay_ms: self.realtime_retry_delay_ms,
},
security_lifecycle: self.security_lifecycle().to_owned(),
schema: ClientDiagnosticsSchema {
current_version: self.schema.version,
upgrading: self.upgrading,
required_version: self
.schema_floor
.as_ref()
.and_then(|floor| floor.required_schema_version),
latest_version: self
.schema_floor
.as_ref()
.and_then(|floor| floor.latest_schema_version),
},
replica: ClientDiagnosticsReplica {
local_revision: self.local_revision().to_string(),
sync_needed: self.sync_needed,
pending_outbox: self.outbox.len(),
},
lease,
subscriptions: ordered_subscriptions,
subscriptions_truncated,
last_round: self.last_round.clone(),
last_change: self.last_change.clone(),
storage: self.diagnostics_storage(),
query_failures: self.query_failures.clone(),
})
}
fn diagnostics_storage(&self) -> ClientDiagnosticsStorage {
let read = || -> Result<ClientDiagnosticsStorage, rusqlite::Error> {
let page_count: i64 = self
.conn
.query_row("PRAGMA page_count", [], |row| row.get(0))?;
let page_size: i64 = self
.conn
.query_row("PRAGMA page_size", [], |row| row.get(0))?;
let outbox_bytes: i64 = self
.conn
.prepare_cached("SELECT COALESCE(SUM(LENGTH(ops_json)), 0) FROM _syncular_outbox")?
.query_row([], |row| row.get(0))?;
let (outcome_entries, outcome_bytes): (i64, i64) = self.conn.prepare_cached(
"SELECT COUNT(*), COALESCE(SUM(LENGTH(results_json) + COALESCE(LENGTH(operations_json), 0)), 0) FROM _syncular_commit_outcomes",
)?.query_row([], |row| Ok((row.get(0)?, row.get(1)?)))?;
let blob_bytes = if self.schema_has_blobs() {
self.conn
.prepare_cached("SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs")?
.query_row([], |row| row.get(0))?
} else {
0
};
let pressure = self
.limits
.blob_cache_max_bytes
.is_some_and(|limit| blob_bytes > limit);
Ok(ClientDiagnosticsStorage {
status: if pressure { "pressure" } else { "healthy" }.to_owned(),
database_bytes_approx: Some(page_count.saturating_mul(page_size).max(0)),
pending_outbox_bytes_approx: Some(outbox_bytes.max(0)),
retained_outcome_bytes_approx: Some(outcome_bytes.max(0)),
retained_outcome_entries: Some(outcome_entries.max(0)),
blob_cache_bytes_approx: Some(blob_bytes.max(0)),
pressure_reason_code: pressure.then(|| "client.blob_cache_over_limit".to_owned()),
})
};
read().unwrap_or_else(|_| ClientDiagnosticsStorage {
status: "unreadable".to_owned(),
database_bytes_approx: None,
pending_outbox_bytes_approx: None,
retained_outcome_bytes_approx: None,
retained_outcome_entries: None,
blob_cache_bytes_approx: None,
pressure_reason_code: None,
})
}
pub fn drain_change_batches(&mut self) -> Vec<ClientChangeBatch> {
self.change_queue.drain(..).collect()
}
pub fn drain_sync_intents(&mut self) -> Vec<SyncIntent> {
self.sync_intent_queue.drain(..).collect()
}
fn schedule_background_retry(&mut self) -> u64 {
let delay_ms = self.retry_delay_ms;
self.sync_intent_queue
.push_back(SyncIntent::Background { delay_ms });
self.retry_delay_ms = (self.retry_delay_ms * 2).min(30_000);
self.round_retry_delay_ms = Some(delay_ms);
delay_ms
}
fn reset_background_retry(&mut self) {
self.retry_delay_ms = 250;
}
fn retryable_transport_code(code: &str) -> bool {
code == "transport.failed"
|| code == "transport.unavailable"
|| code == "sync.transport_failed"
}
fn retryable_failure_code(code: &str) -> bool {
Self::retryable_transport_code(code)
|| matches!(
code,
"sync.auth_required"
| "sync.auth_lease_required"
| "sync.auth_lease_revoked"
| "sync.internal_error"
| "sync.idempotency_cache_miss"
| "sync.schema_not_ready"
| "sync.segment_expired"
| "sync.rate_limited"
| "sync.websocket_connection_limit"
)
}
fn diagnostic_code(code: &str) -> String {
let valid = !code.is_empty()
&& code.len() <= 96
&& code.contains('.')
&& code
.bytes()
.next()
.is_some_and(|byte| byte.is_ascii_lowercase())
&& code.bytes().all(|byte| {
byte.is_ascii_lowercase()
|| byte.is_ascii_digit()
|| matches!(byte, b'.' | b'_' | b'-')
});
if valid {
code.to_owned()
} else {
"client.unknown_failure".to_owned()
}
}
fn set_sync_needed(&mut self, value: bool, interactive: bool) {
if self.sync_needed != value {
if self.begin_observation("syncular_status").is_ok() {
self.sync_needed = value;
let batch = ChangeAccumulator {
status: true,
..ChangeAccumulator::default()
};
if self.finish_observation("syncular_status", batch).is_err() {
self.rollback_observation("syncular_status");
}
} else {
self.sync_needed = value;
}
}
if value && interactive {
self.sync_intent_queue.push_back(SyncIntent::Interactive);
}
}
fn begin_observation(&self, name: &str) -> Result<(), String> {
self.conn
.execute_batch(&format!("SAVEPOINT {name}"))
.map_err(|error| error.to_string())
}
fn rollback_observation(&self, name: &str) {
let _ = self
.conn
.execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}"));
}
fn finish_observation(&mut self, name: &str, batch: ChangeAccumulator) -> Result<(), String> {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::ObservationCommit);
if !batch.touched() {
self.conn
.execute_batch(&format!("RELEASE {name}"))
.map_err(|error| error.to_string())?;
return Ok(());
}
let revision = self
.local_revision()
.checked_add(1)
.ok_or_else(|| "local revision exhausted u64".to_owned())?;
self.conn
.execute(
"INSERT OR REPLACE INTO _syncular_meta(key, value) VALUES (?1, ?2)",
rusqlite::params![LOCAL_REVISION_KEY, revision.to_string()],
)
.map_err(|error| error.to_string())?;
let status = batch.status.then(|| self.status_snapshot());
let event = ClientChangeBatch {
revision: revision.to_string(),
tables: batch
.tables
.into_iter()
.map(|(table, scope_keys)| TableChange {
table,
scope_keys: scope_keys.map(|keys| keys.into_iter().collect()),
})
.collect(),
windows: batch
.windows
.into_iter()
.map(|((base_key, table), units)| WindowChange {
base_key,
table,
units: units.into_iter().collect(),
})
.collect(),
status,
conflicts_changed: batch.conflicts,
rejections_changed: batch.rejections,
outcomes_changed: batch.outcomes,
};
self.conn
.execute_batch(&format!("RELEASE {name}"))
.map_err(|error| error.to_string())?;
let mut diagnostic_tables = event
.tables
.iter()
.map(|entry| entry.table.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let mut diagnostic_windows = event
.windows
.iter()
.map(|entry| entry.table.clone())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let domains_truncated = diagnostic_tables.len() > MAX_DIAGNOSTIC_DOMAINS
|| diagnostic_windows.len() > MAX_DIAGNOSTIC_DOMAINS;
diagnostic_tables.truncate(MAX_DIAGNOSTIC_DOMAINS);
diagnostic_windows.truncate(MAX_DIAGNOSTIC_DOMAINS);
self.last_change = Some(DiagnosticLastChange {
revision: event.revision.clone(),
recorded_at_ms: self.clock_now_ms(),
tables: diagnostic_tables,
windows: diagnostic_windows,
domains_truncated,
status_changed: event.status.is_some(),
conflicts_changed: event.conflicts_changed,
rejections_changed: event.rejections_changed,
outcomes_changed: event.outcomes_changed,
});
self.change_queue.push_back(event);
Ok(())
}
fn record_scope_map(
&self,
batch: &mut ChangeAccumulator,
table_name: &str,
scopes: &[(String, Vec<String>)],
) {
let Some(table) = self.schema.table(table_name) else {
return;
};
for (variable, values) in scopes {
let Some(scope) = table
.scope_variables
.iter()
.find(|scope| &scope.variable == variable)
else {
continue;
};
for value in values {
batch.scope(table_name, format!("{}:{value}", scope.prefix));
}
}
}
fn record_row_scopes(
&self,
batch: &mut ChangeAccumulator,
table_name: &str,
row_id: &str,
base: bool,
) -> bool {
let Some(table) = self.schema.table(table_name) else {
return false;
};
if table.scope_variables.is_empty() {
return false;
}
let columns = table
.scope_variables
.iter()
.map(|scope| quote_ident(&scope.column))
.collect::<Vec<_>>()
.join(", ");
let full_table = if base {
base_table(table_name)
} else {
visible_table(table_name)
};
let sql = format!(
"SELECT {columns} FROM {full_table} WHERE {} LIMIT 1",
row_id_predicate(table)
);
let Ok(mut stmt) = self.conn.prepare_cached(&sql) else {
return false;
};
let values = stmt.query_row(rusqlite::params![row_id], |row| {
let mut values = Vec::with_capacity(table.scope_variables.len());
for index in 0..table.scope_variables.len() {
values.push(row.get::<_, Option<String>>(index)?);
}
Ok(values)
});
let Ok(values) = values else {
return false;
};
let mut recorded = false;
for (scope, value) in table.scope_variables.iter().zip(values) {
if let Some(value) = value {
batch.scope(table_name, format!("{}:{value}", scope.prefix));
recorded = true;
}
}
recorded
}
fn record_commit_changes(
&self,
batch: &mut ChangeAccumulator,
tables: &[String],
changes: &[ssp2::model::Change],
) {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::ObservationPrepare);
for change in changes {
let Some(table_name) = tables.get(change.table_index as usize) else {
continue;
};
let mut precise = self.record_row_scopes(batch, table_name, &change.row_id, true);
if let Some(table) = self.schema.table(table_name) {
for (variable, value) in &change.scopes {
if let Some(scope) = table
.scope_variables
.iter()
.find(|scope| &scope.variable == variable)
{
batch.scope(table_name, format!("{}:{value}", scope.prefix));
precise = true;
}
}
}
if !precise {
batch.table(table_name);
}
}
}
fn scoped_rows_exist(&self, table_name: &str, effective: &[(String, Vec<String>)]) -> bool {
if effective.is_empty() {
return false;
}
let Some(table) = self.schema.table(table_name) else {
return false;
};
let mut clauses = Vec::new();
let mut params = Vec::new();
for (variable, values) in effective {
let Some(column) = table.scope_column(variable) else {
return false;
};
if values.is_empty() {
return false;
}
let placeholders = values
.iter()
.map(|value| {
params.push(SqlValue::Text(value.clone()));
"?"
})
.collect::<Vec<_>>()
.join(", ");
clauses.push(format!("{} IN ({placeholders})", quote_ident(column)));
}
let sql = format!(
"SELECT 1 FROM {} WHERE {} LIMIT 1",
base_table(table_name),
clauses.join(" AND ")
);
self.conn
.query_row(&sql, rusqlite::params_from_iter(params), |_| Ok(()))
.is_ok()
}
pub fn upgrading(&self) -> bool {
self.upgrading
}
pub fn recreate_with_schema(&mut self, schema_json: &Value) -> Result<(), String> {
let new_schema = parse_schema_json(schema_json)?;
let marker: Option<i32> = self
.get_meta(LOCAL_SCHEMA_VERSION_KEY)
.and_then(|v| v.parse().ok());
self.schema = new_schema;
self.create_bookkeeping_tables()?;
if marker != Some(self.schema.version) {
self.run_schema_reset()?;
}
self.prune_unknown_subscriptions()?;
self.enqueue_startup_sync_if_needed()?;
Ok(())
}
fn enqueue_startup_sync_if_needed(&mut self) -> Result<(), String> {
let startup_work = !self.stopped
&& (!self.outbox.is_empty()
|| !self.load_pending_evictions()?.is_empty()
|| self.subs.iter().any(|sub| sub.state == SubState::Active));
if startup_work {
self.sync_needed = true;
self.sync_intent_queue.push_back(SyncIntent::Interactive);
}
Ok(())
}
fn clear_satisfied_persisted_schema_floor(&mut self) {
let satisfied = self
.schema_floor
.as_ref()
.and_then(|floor| floor.required_schema_version)
.is_some_and(|required| self.schema.version >= required);
if !satisfied {
return;
}
self.schema_floor = None;
self.stopped = false;
self.delete_meta(SCHEMA_FLOOR_KEY);
}
fn reconcile_previous_version_at_boot(&self) -> Result<(), String> {
let Some(replica_path) = self.previous_version_replica_path() else {
return Ok(());
};
let max_age_ms = self
.previous_version
.filter(|config| config.enabled)
.map(|config| config.max_age_ms);
reconcile_previous_version_at_boot(
&self.conn,
&replica_path,
self.schema.version,
max_age_ms,
self.clock_now_ms(),
)
}
fn previous_version_replica_path(&self) -> Option<String> {
let path = self.conn.path()?;
if path.is_empty() || path.starts_with("file:") {
return None;
}
Some(path.to_owned())
}
fn run_schema_reset(&mut self) -> Result<(), String> {
self.begin_observation("syncular_schema_reset")?;
let mut batch = ChangeAccumulator::default();
let result = self.run_schema_reset_observed(&mut batch, true, true);
if let Err(error) = result {
self.rollback_observation("syncular_schema_reset");
return Err(error);
}
if let Err(error) = self.finish_observation("syncular_schema_reset", batch) {
self.rollback_observation("syncular_schema_reset");
return Err(error);
}
Ok(())
}
fn run_log_epoch_reset(&mut self, log_epoch: &str) -> Result<Vec<String>, String> {
let resets = self.subs.iter().map(|sub| sub.id.clone()).collect();
self.begin_observation("syncular_log_epoch_reset")?;
let mut batch = ChangeAccumulator::default();
let result = self.run_schema_reset_observed(&mut batch, false, false);
if let Err(error) = result {
self.rollback_observation("syncular_log_epoch_reset");
return Err(error);
}
self.set_meta(LOG_EPOCH_KEY, log_epoch);
self.sync_needed = true;
batch.status = true;
if let Err(error) = self.finish_observation("syncular_log_epoch_reset", batch) {
self.rollback_observation("syncular_log_epoch_reset");
return Err(error);
}
self.sync_intent_queue.push_back(SyncIntent::Interactive);
Ok(resets)
}
fn run_schema_reset_observed(
&mut self,
batch: &mut ChangeAccumulator,
drop_incompatible: bool,
capture: bool,
) -> Result<(), String> {
self.upgrading = true;
batch.status = true;
if self.previous_version.is_some_and(|config| config.enabled) {
let previous_version = self
.get_meta(LOCAL_SCHEMA_VERSION_KEY)
.and_then(|value| value.parse::<i32>().ok())
.unwrap_or(self.schema.version);
let pending = self
.outbox
.iter()
.map(|commit| PendingCommitAudit {
commit_id: &commit.client_commit_id,
operations: commit_audit_operations(commit),
})
.collect::<Vec<_>>();
crate::previous_version::write_previous_version_audit(
&self.conn,
&crate::previous_version::build_previous_version_audit(
&self.schema,
&pending,
previous_version,
self.schema.version,
self.clock_now_ms(),
),
);
}
if let Some(replica_path) = self.previous_version_replica_path() {
sweep_previous_version_container(&self.conn, &replica_path)?;
if capture {
if let Some(config) = self.previous_version.filter(|config| config.enabled) {
let previous_version = self
.get_meta(LOCAL_SCHEMA_VERSION_KEY)
.and_then(|value| value.parse::<i32>().ok())
.unwrap_or(self.schema.version);
capture_previous_version_from_replica(
&self.conn,
&replica_path,
previous_version,
self.schema.version,
&config,
self.clock_now_ms(),
)?;
}
}
}
for table in &self.schema.tables {
batch.table(&table.name);
}
for (base_key, unit, table) in self.load_registered_window_units() {
batch.window(&base_key, &table, &unit);
}
self.insert_sql.borrow_mut().clear();
self.overlay_dirty.set(true);
let virtual_tables: Vec<String> = {
let mut stmt = self
.conn
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND sql LIKE 'CREATE VIRTUAL TABLE%'",
)
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map([], |row| row.get::<_, String>(0))
.map_err(|e| e.to_string())?;
rows.filter_map(Result::ok)
.filter(|name| is_synced_table_name(name))
.collect()
};
for name in virtual_tables {
self.conn
.execute_batch(&format!(
"DROP TABLE IF EXISTS {}",
quote_ident(&format!("_syncular_fts_{name}"))
))
.map_err(|e| e.to_string())?;
self.conn
.execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
.map_err(|e| e.to_string())?;
}
let existing: Vec<String> = {
let mut stmt = self
.conn
.prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
.map_err(|e| e.to_string())?;
let rows = stmt
.query_map([], |row| row.get::<_, String>(0))
.map_err(|e| e.to_string())?;
rows.filter_map(Result::ok)
.filter(|name| is_synced_table_name(name))
.collect()
};
for name in &existing {
if let Some(table) = name.strip_prefix("_syncular_base_") {
batch.table(table);
} else if !name.starts_with("_syncular_") {
batch.table(name);
}
}
for name in existing {
self.conn
.execute(&format!("DROP TABLE IF EXISTS {}", quote_ident(&name)), [])
.map_err(|e| e.to_string())?;
}
self.create_synced_tables()?;
for sub in &mut self.subs {
sub.cursor = -1;
sub.bootstrap_state = None;
sub.effective = None;
sub.state = SubState::Active;
sub.reason_code = None;
sub.synced_once = false;
}
let subs = self.subs.clone();
for sub in &subs {
self.persist_sub(sub)?;
}
self.stopped = false;
self.schema_floor = None;
self.delete_meta(SCHEMA_FLOOR_KEY);
self.set_meta(LOCAL_SCHEMA_VERSION_KEY, &self.schema.version.to_string());
set_local_schema_descriptor(&self.conn, &self.schema);
if drop_incompatible && self.drop_incompatible_outbox()? {
batch.rejections = true;
batch.status = true;
batch.outcomes = true;
}
self.rebuild_overlay();
Ok(())
}
fn drop_incompatible_outbox(&mut self) -> Result<bool, String> {
let incompatible = self
.outbox
.iter()
.filter(|commit| {
first_incompatibility(&self.schema, &commit_audit_operations(commit)).is_some()
})
.cloned()
.collect::<Vec<_>>();
if incompatible.is_empty() {
return Ok(false);
}
let mut rejections = Vec::new();
for commit in &incompatible {
let results = commit
.ops
.iter()
.enumerate()
.map(|(op_index, operation)| {
let rejection = RejectionRecord {
client_commit_id: commit.client_commit_id.clone(),
op_index: op_index as i32,
code: OUTBOX_INCOMPATIBLE_CODE.to_owned(),
message: "the persisted commit cannot encode under the current schema"
.to_owned(),
retryable: false,
details: None,
operation: Some(CommitOperation::from(operation)),
};
rejections.push(rejection.clone());
CommitOperationOutcome::Error { rejection }
})
.collect::<Vec<_>>();
self.persist_commit_outcome(
&commit.client_commit_id,
CommitOutcomeStatus::Rejected,
&results,
Some(&commit.ops),
)?;
self.delete_outbox_persisted(&commit.client_commit_id)?;
}
self.prune_commit_outcomes()?;
let incompatible_ids = incompatible
.iter()
.map(|commit| commit.client_commit_id.as_str())
.collect::<BTreeSet<_>>();
self.outbox
.retain(|commit| !incompatible_ids.contains(commit.client_commit_id.as_str()));
self.rejections.extend(rejections);
Ok(true)
}
fn create_synced_tables(&self) -> Result<(), String> {
for table in &self.schema.tables {
for (full, index_prefix) in [
(base_table(&table.name), "_syncular_base_"),
(visible_table(&table.name), ""),
] {
let mut cols: Vec<String> =
table.columns.iter().map(|c| quote_ident(&c.name)).collect();
cols.push("\"_syncular_version\" INTEGER NOT NULL".to_owned());
let sql = format!(
"CREATE TABLE IF NOT EXISTS {full} ({} , PRIMARY KEY ({}))",
cols.join(", "),
quote_ident(&table.primary_key)
);
self.conn.execute(&sql, []).map_err(|e| e.to_string())?;
for index in &table.indexes {
let unique = if index.unique { "UNIQUE " } else { "" };
let index_name = quote_ident(&format!("{index_prefix}{}", index.name));
let cols_sql = index
.columns
.iter()
.map(|c| quote_ident(c))
.collect::<Vec<_>>()
.join(", ");
let index_sql = format!(
"CREATE {unique}INDEX IF NOT EXISTS {index_name} ON {full} ({cols_sql})"
);
self.conn
.execute(&index_sql, [])
.map_err(|e| e.to_string())?;
}
}
}
for table in &self.schema.tables {
for index in &table.fts_indexes {
self.create_fts_projection(table, index)?;
}
}
Ok(())
}
fn fts_projection_exists(&self, index: &FtsIndexSchema) -> Result<bool, String> {
let count: i64 = self
.conn
.query_row(
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?1",
rusqlite::params![index.name],
|row| row.get(0),
)
.map_err(|e| e.to_string())?;
Ok(count > 0)
}
fn drop_fts_triggers(&self, index: &FtsIndexSchema) -> Result<(), String> {
for suffix in ["ai", "ad", "au"] {
self.conn
.execute(
&format!(
"DROP TRIGGER IF EXISTS {}",
quote_ident(&format!("{}_{suffix}", index.name))
),
[],
)
.map_err(|e| e.to_string())?;
}
Ok(())
}
fn create_fts_triggers(
&self,
table: &TableSchema,
index: &FtsIndexSchema,
) -> Result<(), String> {
let fts = quote_ident(&index.name);
let source = visible_table(&table.name);
let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
let pk = quote_ident(&table.primary_key);
let projection_columns = std::iter::once(source_id.clone())
.chain(index.columns.iter().map(|column| quote_ident(column)))
.collect::<Vec<_>>()
.join(", ");
let new_values = std::iter::once(format!("CAST(new.{pk} AS TEXT)"))
.chain(
index
.columns
.iter()
.map(|column| format!("new.{}", quote_ident(column))),
)
.collect::<Vec<_>>()
.join(", ");
let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
let delete_new = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)");
let delete_old = format!("DELETE FROM {fts} WHERE rowid = (SELECT id FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)); DELETE FROM {mapping} WHERE source_id = CAST(old.{pk} AS TEXT)");
let insert_new = format!("INSERT INTO {mapping}(source_id) VALUES (CAST(new.{pk} AS TEXT)); INSERT INTO {fts} (rowid, {projection_columns}) VALUES ((SELECT id FROM {mapping} WHERE source_id = CAST(new.{pk} AS TEXT)), {new_values})");
let bi = quote_ident(&format!("{}_bi", index.name));
let ai = quote_ident(&format!("{}_ai", index.name));
let ad = quote_ident(&format!("{}_ad", index.name));
let au = quote_ident(&format!("{}_au", index.name));
let replacement_exists = format!("EXISTS (SELECT 1 FROM {source} WHERE {pk} = new.{pk})");
let sql = format!(
"DROP TRIGGER IF EXISTS {bi};
DROP TRIGGER IF EXISTS {ai};
DROP TRIGGER IF EXISTS {ad};
DROP TRIGGER IF EXISTS {au};
CREATE TRIGGER {bi} BEFORE INSERT ON {source} WHEN {replacement_exists} BEGIN {delete_new}; END;
CREATE TRIGGER {ai} AFTER INSERT ON {source} BEGIN {insert_new}; END;
CREATE TRIGGER {ad} AFTER DELETE ON {source} BEGIN {delete_old}; END;
CREATE TRIGGER {au} AFTER UPDATE ON {source} BEGIN {delete_old}; {delete_new}; {insert_new}; END;",
);
self.conn.execute_batch(&sql).map_err(|e| e.to_string())
}
fn rebuild_fts_projection(
&self,
table: &TableSchema,
index: &FtsIndexSchema,
) -> Result<(), String> {
let fts = quote_ident(&index.name);
let source_id = quote_ident(FTS_SOURCE_ID_COLUMN);
let pk = quote_ident(&table.primary_key);
let indexed_columns = index
.columns
.iter()
.map(|column| quote_ident(column))
.collect::<Vec<_>>();
let projection_columns = std::iter::once(source_id.clone())
.chain(indexed_columns.iter().cloned())
.collect::<Vec<_>>()
.join(", ");
let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
let sql = format!(
"DELETE FROM {fts}; DELETE FROM {mapping}; INSERT INTO {fts} ({projection_columns}) SELECT CAST({pk} AS TEXT), {columns} FROM {source}; INSERT INTO {mapping}(id, source_id) SELECT rowid, {source_id} FROM {fts};",
columns = indexed_columns.join(", "),
source = visible_table(&table.name),
);
self.conn.execute_batch(&sql).map_err(|e| e.to_string())
}
fn create_fts_projection(
&self,
table: &TableSchema,
index: &FtsIndexSchema,
) -> Result<(), String> {
self.begin_observation("syncular_fts_schema")?;
let result = (|| {
let existed = self.fts_projection_exists(index)?;
let tokenizer = index.tokenize.replace('\'', "''");
let columns = index
.columns
.iter()
.map(|column| quote_ident(column))
.collect::<Vec<_>>()
.join(", ");
let sql = format!(
"CREATE VIRTUAL TABLE IF NOT EXISTS {fts} USING fts5({source_id} UNINDEXED, {columns}, tokenize='{tokenizer}')",
fts = quote_ident(&index.name),
source_id = quote_ident(FTS_SOURCE_ID_COLUMN),
);
self.conn.execute(&sql, []).map_err(|error| {
format!(
"cannot create local FTS5 projection {:?}: {error}",
index.name
)
})?;
let mapping = quote_ident(&format!("_syncular_fts_{}", index.name));
let mapped: bool = self
.conn
.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)",
[format!("_syncular_fts_{}", index.name)],
|row| row.get(0),
)
.map_err(|e| e.to_string())?;
self.conn.execute_batch(&format!("CREATE TABLE IF NOT EXISTS {mapping}(id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)")).map_err(|e| e.to_string())?;
if !mapped && existed {
self.conn
.execute_batch(&format!(
"INSERT INTO {mapping}(id, source_id) SELECT rowid, {} FROM {}",
quote_ident(FTS_SOURCE_ID_COLUMN),
quote_ident(&index.name)
))
.map_err(|e| e.to_string())?;
}
self.create_fts_triggers(table, index)?;
if !existed {
self.rebuild_fts_projection(table, index)?;
}
Ok(())
})();
if let Err(error) = result {
self.rollback_observation("syncular_fts_schema");
return Err(error);
}
if let Err(error) = self.conn.execute_batch("RELEASE syncular_fts_schema") {
self.rollback_observation("syncular_fts_schema");
return Err(error.to_string());
}
Ok(())
}
fn persist_sub(&self, sub: &Subscription) -> Result<(), String> {
let state = serde_json::json!({
"requested": scope_map_to_json(&sub.requested),
"params": sub.params,
"cursor": sub.cursor,
"bootstrapState": sub.bootstrap_state,
"status": sub.state.name(),
"reasonCode": sub.reason_code,
"effectiveScopes": sub.effective.as_ref().map(|e| scope_map_to_json(e)),
"syncedOnce": sub.synced_once,
});
self.conn.execute(
"INSERT OR REPLACE INTO _syncular_subscriptions (id, tbl, state_json) VALUES (?1, ?2, ?3)",
rusqlite::params![sub.id, sub.table, state.to_string()],
).map(|_| ()).map_err(|error| error.to_string())
}
fn persist_outbox_insert(&self, commit: &OutboxCommit) -> Result<(), String> {
let ops: Vec<Value> = commit
.ops
.iter()
.map(|op| {
serde_json::json!({
"op": if op.upsert { "upsert" } else { "delete" },
"table": op.table,
"rowId": op.row_id,
"baseVersion": op.base_version,
"values": op.values.clone().map(Value::Object),
})
})
.collect();
self.conn
.execute(
"INSERT OR REPLACE INTO _syncular_outbox (commit_id, ops_json) VALUES (?1, ?2)",
rusqlite::params![commit.client_commit_id, Value::Array(ops).to_string()],
)
.map_err(|error| error.to_string())?;
for operation in &commit.ops {
let Some(values) = &operation.values else {
continue;
};
let Some(table) = self.schema.table(&operation.table) else {
continue;
};
for column in table
.columns
.iter()
.filter(|column| column.ty == ColumnType::BlobRef)
{
let Some(Value::String(raw)) = values.get(&column.name) else {
continue;
};
let Ok(blob_ref) = serde_json::from_str::<Value>(raw) else {
continue;
};
let Some(blob_id) = blob_ref.get("blobId").and_then(Value::as_str) else {
continue;
};
self.conn
.execute(
"INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
SELECT ?1, ?2 WHERE EXISTS (SELECT 1 FROM _syncular_blobs WHERE blob_id = ?2)",
rusqlite::params![commit.client_commit_id, blob_id],
)
.map_err(|error| error.to_string())?;
}
}
Ok(())
}
fn delete_outbox_persisted(&self, client_commit_id: &str) -> Result<(), String> {
self.conn
.execute(
"DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?1",
rusqlite::params![client_commit_id],
)
.map_err(|error| error.to_string())?;
self.conn
.execute(
"DELETE FROM _syncular_outbox WHERE commit_id = ?1",
rusqlite::params![client_commit_id],
)
.map(|_| ())
.map_err(|error| error.to_string())
}
fn outcome_status_name(status: CommitOutcomeStatus) -> &'static str {
match status {
CommitOutcomeStatus::Applied => "applied",
CommitOutcomeStatus::Cached => "cached",
CommitOutcomeStatus::Conflict => "conflict",
CommitOutcomeStatus::Rejected => "rejected",
}
}
fn outcome_resolution_name(resolution: CommitOutcomeResolution) -> &'static str {
match resolution {
CommitOutcomeResolution::Active => "active",
CommitOutcomeResolution::ResolvedKeepServer => "resolved_keep_server",
CommitOutcomeResolution::Superseded => "superseded",
CommitOutcomeResolution::Dismissed => "dismissed",
}
}
fn parse_outcome_status(value: &str) -> Result<CommitOutcomeStatus, String> {
match value {
"applied" => Ok(CommitOutcomeStatus::Applied),
"cached" => Ok(CommitOutcomeStatus::Cached),
"conflict" => Ok(CommitOutcomeStatus::Conflict),
"rejected" => Ok(CommitOutcomeStatus::Rejected),
_ => Err(format!("invalid persisted commit outcome status {value:?}")),
}
}
fn parse_outcome_resolution(value: &str) -> Result<CommitOutcomeResolution, String> {
match value {
"active" => Ok(CommitOutcomeResolution::Active),
"resolved_keep_server" => Ok(CommitOutcomeResolution::ResolvedKeepServer),
"superseded" => Ok(CommitOutcomeResolution::Superseded),
"dismissed" => Ok(CommitOutcomeResolution::Dismissed),
_ => Err(format!(
"invalid persisted commit outcome resolution {value:?}"
)),
}
}
fn persist_commit_outcome(
&self,
client_commit_id: &str,
status: CommitOutcomeStatus,
results: &[CommitOperationOutcome],
operations: Option<&[OutboxOp]>,
) -> Result<(), String> {
let results_json = serde_json::to_string(results).map_err(|error| error.to_string())?;
let operations_json = operations
.map(|items| {
serde_json::to_string(&items.iter().map(CommitOperation::from).collect::<Vec<_>>())
})
.transpose()
.map_err(|error| error.to_string())?;
self.conn
.execute(
"INSERT INTO _syncular_commit_outcomes (
client_commit_id, status, recorded_at_ms, results_json,
operations_json, resolution
) VALUES (?1, ?2, ?3, ?4, ?5, 'active')",
rusqlite::params![
client_commit_id,
Self::outcome_status_name(status),
self.clock_now_ms(),
results_json,
operations_json
],
)
.map(|_| ())
.map_err(|error| error.to_string())
}
fn outcome_from_row(row: StoredCommitOutcomeRow) -> Result<CommitOutcome, String> {
let StoredCommitOutcomeRow {
sequence,
client_commit_id,
status,
recorded_at_ms,
results_json,
operations_json,
resolution,
resolved_at_ms,
replacement_client_commit_id,
} = row;
Ok(CommitOutcome {
sequence,
client_commit_id,
status: Self::parse_outcome_status(&status)?,
recorded_at_ms,
results: serde_json::from_str(&results_json)
.map_err(|error| format!("invalid persisted commit outcome results: {error}"))?,
operations: operations_json
.map(|value| {
serde_json::from_str(&value).map_err(|error| {
format!("invalid persisted commit outcome operations: {error}")
})
})
.transpose()?,
resolution: Self::parse_outcome_resolution(&resolution)?,
resolved_at_ms,
replacement_client_commit_id,
})
}
pub fn commit_outcome(&self, client_commit_id: &str) -> Result<Option<CommitOutcome>, String> {
let row = self
.conn
.query_row(
"SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
resolution, resolved_at_ms, replacement_client_commit_id
FROM _syncular_commit_outcomes WHERE client_commit_id = ?1",
rusqlite::params![client_commit_id],
|row| {
Ok(StoredCommitOutcomeRow {
sequence: row.get(0)?,
client_commit_id: row.get(1)?,
status: row.get(2)?,
recorded_at_ms: row.get(3)?,
results_json: row.get(4)?,
operations_json: row.get(5)?,
resolution: row.get(6)?,
resolved_at_ms: row.get(7)?,
replacement_client_commit_id: row.get(8)?,
})
},
)
.optional()
.map_err(|error| error.to_string())?;
row.map(Self::outcome_from_row).transpose()
}
pub fn commit_outcomes(&self, query: CommitOutcomeQuery) -> Result<Vec<CommitOutcome>, String> {
if query.limit == Some(0) {
return Err("sync.invalid_request: commit outcome limit must be positive".to_owned());
}
let mut sql = String::from(
"SELECT seq, client_commit_id, status, recorded_at_ms, results_json, operations_json,
resolution, resolved_at_ms, replacement_client_commit_id
FROM _syncular_commit_outcomes",
);
if query.active_only {
sql.push_str(" WHERE resolution = 'active' AND status IN ('conflict', 'rejected')");
}
sql.push_str(" ORDER BY seq DESC");
if let Some(limit) = query.limit {
sql.push_str(&format!(" LIMIT {limit}"));
}
let mut stmt = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
let rows = stmt
.query_map([], |row| {
Ok(StoredCommitOutcomeRow {
sequence: row.get(0)?,
client_commit_id: row.get(1)?,
status: row.get(2)?,
recorded_at_ms: row.get(3)?,
results_json: row.get(4)?,
operations_json: row.get(5)?,
resolution: row.get(6)?,
resolved_at_ms: row.get(7)?,
replacement_client_commit_id: row.get(8)?,
})
})
.map_err(|error| error.to_string())?;
let mut outcomes = Vec::new();
for row in rows {
outcomes.push(Self::outcome_from_row(
row.map_err(|error| error.to_string())?,
)?);
}
Ok(outcomes)
}
fn prune_commit_outcomes(&self) -> Result<(), String> {
#[cfg(test)]
self.outcome_prune_count
.set(self.outcome_prune_count.get() + 1);
let max_entries = self.limits.outcome_retention_max_entries.unwrap_or(1_000);
let count = self
.conn
.query_row(
"SELECT COUNT(*) FROM _syncular_commit_outcomes",
[],
|row| row.get::<_, i64>(0),
)
.map_err(|error| error.to_string())? as usize;
let excess = count.saturating_sub(max_entries);
if excess == 0 {
return Ok(());
}
let mut stmt = self
.conn
.prepare(
"SELECT seq FROM _syncular_commit_outcomes
WHERE status IN ('applied', 'cached') OR resolution != 'active'
ORDER BY seq ASC LIMIT ?1",
)
.map_err(|error| error.to_string())?;
let rows = stmt
.query_map(rusqlite::params![excess as i64], |row| row.get::<_, i64>(0))
.map_err(|error| error.to_string())?;
let sequences = rows
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?;
drop(stmt);
for sequence in sequences {
self.conn
.execute(
"DELETE FROM _syncular_commit_outcomes WHERE seq = ?1",
rusqlite::params![sequence],
)
.map_err(|error| error.to_string())?;
}
Ok(())
}
pub fn subscribe(
&mut self,
id: String,
table: String,
scopes: Vec<(String, Vec<String>)>,
params: Option<String>,
) -> Result<(), String> {
if self.schema.table(&table).is_none() {
return Err(format!("unknown table {table:?}"));
}
if let Some(existing) = self.subs.iter().find(|subscription| subscription.id == id) {
let same_intent = existing.table == table
&& canonical_scope_json(&existing.requested) == canonical_scope_json(&scopes)
&& existing.params == params;
if same_intent {
return Ok(());
}
return Err(
"client.subscription_intent_mismatch: the subscription id is already registered for a different table, scopes, or params"
.to_owned(),
);
}
let sub = Subscription {
id: id.clone(),
table,
requested: scopes,
params,
cursor: -1,
bootstrap_state: None,
state: SubState::Active,
reason_code: None,
effective: None,
synced_once: false,
};
self.persist_sub(&sub)?;
self.subs.push(sub);
Ok(())
}
pub fn unsubscribe(&mut self, id: &str) {
self.subs.retain(|s| s.id != id);
let _ = self.conn.execute(
"DELETE FROM _syncular_subscriptions WHERE id = ?1",
rusqlite::params![id],
);
}
pub fn set_window(
&mut self,
base: &WindowBase,
units: &[String],
) -> Result<CommandEffects, String> {
let table = self
.schema
.table(&base.table)
.ok_or_else(|| format!("unknown table {:?}", base.table))?;
if table.scope_column(&base.variable).is_none() {
return Err(format!(
"setWindow: table {:?} has no scope variable {:?} (§4.8)",
base.table, base.variable
));
}
let base_key = window_base_key(base);
let wanted: std::collections::HashSet<&String> = units.iter().collect();
let live = self.load_window_units(&base_key);
self.begin_observation("syncular_window")?;
let mut batch = ChangeAccumulator::default();
let mut changed = false;
let prior_subs = self.subs.clone();
let widened = (|| {
for unit in units {
if live.iter().any(|(u, _)| u == unit) {
continue;
}
let sub_id = derive_sub_id(base, unit);
self.delete_pending_evict(&sub_id)?;
self.insert_window_unit(&base_key, unit, &sub_id)?;
self.subscribe(
sub_id,
base.table.clone(),
unit_scopes(base, unit),
base.params.clone(),
)?;
batch.window(&base_key, &base.table, unit);
changed = true;
}
self.finish_observation("syncular_window", batch)
})();
if let Err(error) = widened {
self.rollback_observation("syncular_window");
self.subs = prior_subs;
return Err(error);
}
for (unit, sub_id) in live {
if wanted.contains(&unit) {
continue;
}
self.rebuild_overlay_if_dirty();
let prior_subs = self.subs.clone();
self.begin_observation("syncular_window")?;
let effective = self
.subs
.iter()
.find(|sub| sub.id == sub_id)
.and_then(|sub| sub.effective.clone())
.unwrap_or_else(|| unit_scopes(base, &unit));
let mut batch = ChangeAccumulator::default();
self.record_scope_map(&mut batch, &base.table, &effective);
batch.window(&base_key, &base.table, &unit);
let result = self.evict_unit(&base_key, base, &unit, &sub_id);
let remaining = match result {
Ok(remaining) => remaining,
Err(error) => {
self.rollback_observation("syncular_window");
self.subs = prior_subs;
return Err(error);
}
};
if let Err(error) = self.finish_observation("syncular_window", batch) {
self.rollback_observation("syncular_window");
self.subs = prior_subs;
return Err(error);
}
if remaining {
self.drain_pending_evictions()?;
}
changed = true;
}
Ok(if changed {
CommandEffects::interactive()
} else {
CommandEffects::none()
})
}
pub fn window_state(&self, base: &WindowBase) -> WindowState {
let mut units = Vec::new();
let mut pending = Vec::new();
for (unit, sub_id) in self.load_window_units(&window_base_key(base)) {
let is_pending = match self.subs.iter().find(|s| s.id == sub_id) {
Some(sub) => {
sub.state != SubState::Active || sub.cursor < 0 || sub.bootstrap_state.is_some()
}
None => true,
};
if is_pending {
pending.push(unit.clone());
}
units.push(unit);
}
WindowState { units, pending }
}
fn load_window_units(&self, base_key: &str) -> Vec<(String, String)> {
let mut stmt = match self
.conn
.prepare("SELECT unit, sub_id FROM _syncular_windows WHERE base = ?1 ORDER BY unit ASC")
{
Ok(stmt) => stmt,
Err(_) => return Vec::new(),
};
let rows = stmt.query_map(rusqlite::params![base_key], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
});
match rows {
Ok(rows) => rows.filter_map(Result::ok).collect(),
Err(_) => Vec::new(),
}
}
fn load_registered_window_units(&self) -> Vec<(String, String, String)> {
let mut stmt = match self.conn.prepare(
"SELECT windows.base, windows.unit, subscriptions.tbl
FROM _syncular_windows AS windows
JOIN _syncular_subscriptions AS subscriptions
ON subscriptions.id = windows.sub_id
ORDER BY windows.base, windows.unit",
) {
Ok(stmt) => stmt,
Err(_) => return Vec::new(),
};
let rows = stmt.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
))
});
match rows {
Ok(rows) => rows.filter_map(Result::ok).collect(),
Err(_) => Vec::new(),
}
}
fn window_unit_by_sub_id(&self, sub_id: &str) -> Option<(String, String)> {
self.conn
.query_row(
"SELECT base, unit FROM _syncular_windows WHERE sub_id = ?1 LIMIT 1",
rusqlite::params![sub_id],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.ok()
}
fn insert_window_unit(&self, base_key: &str, unit: &str, sub_id: &str) -> Result<(), String> {
self.conn
.execute(
"INSERT OR REPLACE INTO _syncular_windows(base, unit, sub_id) VALUES (?1, ?2, ?3)",
rusqlite::params![base_key, unit, sub_id],
)
.map(|_| ())
.map_err(|error| error.to_string())
}
fn delete_window_unit(&self, base_key: &str, unit: &str) -> Result<(), String> {
self.conn
.execute(
"DELETE FROM _syncular_windows WHERE base = ?1 AND unit = ?2",
rusqlite::params![base_key, unit],
)
.map(|_| ())
.map_err(|error| error.to_string())
}
fn evict_unit(
&mut self,
base_key: &str,
base: &WindowBase,
unit: &str,
sub_id: &str,
) -> Result<bool, String> {
let effective = self
.subs
.iter()
.find(|s| s.id == sub_id)
.and_then(|s| s.effective.clone())
.unwrap_or_else(|| unit_scopes(base, unit));
let pinned = self.pinned_row_ids(&base.table);
let (remaining, deferred) = self.evict_scope_rows(&base.table, &effective, &pinned)?;
self.delete_window_unit(base_key, unit)?;
self.conn
.execute(
"DELETE FROM _syncular_subscriptions WHERE id = ?1",
[sub_id],
)
.map_err(|error| error.to_string())?;
self.subs.retain(|sub| sub.id != sub_id);
if deferred {
self.save_pending_evict(sub_id, &base.table, &effective)?;
} else {
self.delete_pending_evict(sub_id)?;
}
Ok(remaining)
}
fn evict_scope_rows(
&self,
table_name: &str,
effective: &[(String, Vec<String>)],
pinned: &std::collections::HashSet<String>,
) -> Result<(bool, bool), String> {
if effective.is_empty() {
return Ok((false, false));
}
let table = self.schema.table(table_name).ok_or("sync.unknown_table")?;
let mut clauses = Vec::new();
let mut params: Vec<SqlValue> = Vec::new();
for (variable, values) in effective {
let column = table.scope_column(variable).ok_or("sync.scope_revoked")?;
if values.is_empty() {
return Ok((false, false));
}
let holes = values
.iter()
.map(|v| {
params.push(SqlValue::Text(v.clone()));
"?"
})
.collect::<Vec<_>>()
.join(", ");
clauses.push(format!("{} IN ({holes})", quote_ident(column)));
}
let scope = clauses.join(" AND ");
let scope_params = params.len();
let pk = quote_ident(&table.primary_key);
if !pinned.is_empty() {
let holes = pinned
.iter()
.map(|id| {
params.push(SqlValue::Text(id.clone()));
"?"
})
.collect::<Vec<_>>()
.join(", ");
clauses.push(format!("{pk} NOT IN ({holes})"));
}
let predicate = clauses.join(" AND ");
let base = base_table(table_name);
let visible = visible_table(table_name);
let mut ids: Vec<SqlValue> = Vec::new();
for source in [&base, &visible] {
let mut stmt = self
.conn
.prepare_cached(&format!(
"SELECT {pk} FROM {source} WHERE {predicate} LIMIT 1024"
))
.map_err(|e| e.to_string())?;
ids = stmt
.query_map(rusqlite::params_from_iter(¶ms), |row| row.get(0))
.map_err(|e| e.to_string())?
.collect::<Result<_, _>>()
.map_err(|e| e.to_string())?;
if !ids.is_empty() {
break;
}
}
if !ids.is_empty() {
let holes = vec!["?"; ids.len()].join(", ");
for source in [&base, &visible] {
self.conn
.execute(
&format!("DELETE FROM {source} WHERE {pk} IN ({holes})"),
rusqlite::params_from_iter(&ids),
)
.map_err(|e| e.to_string())?;
}
}
let mut remaining = false;
let mut deferred = false;
for source in [&base, &visible] {
remaining |= self
.conn
.query_row(
&format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {predicate})"),
rusqlite::params_from_iter(¶ms),
|row| row.get::<_, bool>(0),
)
.map_err(|e| e.to_string())?;
deferred |= self
.conn
.query_row(
&format!("SELECT EXISTS(SELECT 1 FROM {source} WHERE {scope})"),
rusqlite::params_from_iter(¶ms[..scope_params]),
|row| row.get::<_, bool>(0),
)
.map_err(|e| e.to_string())?;
}
Ok((remaining, deferred))
}
fn drain_pending_evictions(&mut self) -> Result<(), String> {
let pending = self.load_pending_evictions()?;
if pending.is_empty() {
return Ok(());
}
self.rebuild_overlay_if_dirty();
for (sub_id, table_name, effective) in pending {
if self.schema.table(&table_name).is_none() {
self.delete_pending_evict(&sub_id)?;
continue;
}
loop {
std::thread::yield_now();
self.begin_observation("syncular_evict")?;
let pinned = self.pinned_row_ids(&table_name);
let result = self.evict_scope_rows(&table_name, &effective, &pinned);
let (remaining, deferred) = match result {
Ok(result) => result,
Err(error) => {
self.rollback_observation("syncular_evict");
return Err(error);
}
};
if !deferred {
if let Err(error) = self.delete_pending_evict(&sub_id) {
self.rollback_observation("syncular_evict");
return Err(error);
}
}
let mut batch = ChangeAccumulator::default();
self.record_scope_map(&mut batch, &table_name, &effective);
if let Err(error) = self.finish_observation("syncular_evict", batch) {
self.rollback_observation("syncular_evict");
return Err(error);
}
if !remaining {
break;
}
}
}
Ok(())
}
fn pinned_row_ids(&self, table: &str) -> std::collections::HashSet<String> {
let mut pinned = std::collections::HashSet::new();
for commit in &self.outbox {
for op in &commit.ops {
if op.table == table {
pinned.insert(op.row_id.clone());
}
}
}
pinned
}
fn save_pending_evict(
&self,
sub_id: &str,
table: &str,
effective: &[(String, Vec<String>)],
) -> Result<(), String> {
self.conn.execute(
"INSERT OR REPLACE INTO _syncular_window_pending_evict(sub_id, tbl, effective_scopes) VALUES (?1, ?2, ?3)",
rusqlite::params![sub_id, table, scope_map_to_json(effective).to_string()],
).map(|_| ()).map_err(|error| error.to_string())
}
fn delete_pending_evict(&self, sub_id: &str) -> Result<(), String> {
self.conn
.execute(
"DELETE FROM _syncular_window_pending_evict WHERE sub_id = ?1",
rusqlite::params![sub_id],
)
.map(|_| ())
.map_err(|error| error.to_string())
}
fn load_pending_evictions(&self) -> Result<Vec<PendingEvict>, String> {
let mut stmt = self
.conn
.prepare("SELECT sub_id, tbl, effective_scopes FROM _syncular_window_pending_evict")
.map_err(|error| error.to_string())?;
let rows = stmt
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
))
})
.map_err(|error| error.to_string())?;
rows.map(|row| {
let (sub_id, table, json) = row.map_err(|error| error.to_string())?;
let value: Value = serde_json::from_str(&json)
.map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
let effective = json_to_scope_map(&value)
.map_err(|_| "sync.local_corrupt: invalid pending eviction scopes".to_owned())?;
Ok((sub_id, table, effective))
})
.collect()
}
pub fn mutate(&mut self, mutations: Vec<Mutation>) -> Result<String, String> {
if mutations.is_empty() {
return Err("a commit must contain at least one operation (§6.1)".to_owned());
}
let mut ops = Vec::with_capacity(mutations.len());
for mutation in mutations {
match mutation {
Mutation::Upsert {
table,
values,
base_version,
} => {
let schema_table = self
.schema
.table(&table)
.ok_or_else(|| format!("unknown table {table:?}"))?;
let values = normalize_values_casing(schema_table, values)?;
let row_id = render_row_id_json(values.get(&schema_table.primary_key))?;
let values = full_row_values(schema_table, values)?;
self.validate_author_encode(schema_table, &row_id, &values)?;
ops.push(OutboxOp {
upsert: true,
table,
row_id,
base_version,
values: Some(values),
});
}
Mutation::Delete {
table,
row_id,
base_version,
} => {
if self.schema.table(&table).is_none() {
return Err(format!("unknown table {table:?}"));
}
ops.push(OutboxOp {
upsert: false,
table,
row_id,
base_version,
values: None,
});
}
}
}
self.record_outbox_commit(ops)
}
fn record_outbox_commit(&mut self, ops: Vec<OutboxOp>) -> Result<String, String> {
let commit = OutboxCommit {
client_commit_id: uuid::Uuid::new_v4().to_string(),
ops,
};
self.begin_observation("syncular_mutation")?;
let mut batch = ChangeAccumulator::default();
for op in &commit.ops {
let mut precise = self.record_row_scopes(&mut batch, &op.table, &op.row_id, false);
if let Some(values) = &op.values {
if let Some(table) = self.schema.table(&op.table) {
for scope in &table.scope_variables {
if let Some(Value::String(value)) = values.get(&scope.column) {
batch.scope(&op.table, format!("{}:{value}", scope.prefix));
precise = true;
}
}
}
}
if !precise {
batch.table(&op.table);
}
}
if let Err(error) = self.persist_outbox_insert(&commit) {
self.rollback_observation("syncular_mutation");
return Err(error);
}
let was_dirty = self.overlay_dirty.get();
if was_dirty {
self.rebuild_overlay();
}
if let Some(error) = commit
.ops
.iter()
.find_map(|op| self.apply_outbox_op(op).err())
{
self.rollback_observation("syncular_mutation");
self.overlay_dirty.set(was_dirty);
return Err(error);
}
let id = commit.client_commit_id.clone();
self.outbox.push(commit);
batch.status = true;
if let Err(error) = self.finish_observation("syncular_mutation", batch) {
self.rollback_observation("syncular_mutation");
self.outbox.pop();
self.overlay_dirty.set(was_dirty);
return Err(error);
}
Ok(id)
}
pub fn patch(
&mut self,
table: &str,
row_id: &str,
partial: Map<String, Value>,
base_version: Option<i64>,
) -> Result<String, String> {
let schema_table = self
.schema
.table(table)
.ok_or_else(|| format!("unknown table {table:?}"))?;
let partial = normalize_values_casing(schema_table, partial)?;
if let Some(pk) = partial.get(&schema_table.primary_key) {
if pk.as_str() != Some(row_id) {
return Err(format!(
"sync.invalid_request: table {table:?}: patch cannot change the primary key"
));
}
}
for scope in &schema_table.scope_variables {
if !partial.contains_key(&scope.column) || scope.column == schema_table.primary_key {
continue;
}
let Some(column) = schema_table
.columns
.iter()
.find(|column| column.name == scope.column)
else {
continue;
};
let stored = self
.conn
.query_row(
&format!(
"SELECT {} FROM {} WHERE {}",
quote_ident(&scope.column),
visible_table(&schema_table.name),
row_id_predicate(schema_table)
),
rusqlite::params![row_id],
|row| Ok(sql_ref_to_json(column, row.get_ref(0)?)),
)
.ok();
let incoming = json_to_column_value(column, partial.get(&scope.column))
.ok()
.map(|value| column_value_to_json(&value));
let equal = stored.as_ref().is_some_and(|stored| {
stored.get("$bytes").is_none() && incoming.as_ref() == Some(stored)
});
if !equal {
return Err(format!(
"sync.invalid_request: table {table:?}: patch cannot write scope column {:?} (§3.4)",
scope.column
));
}
}
let mut values = partial;
for scope in &schema_table.scope_variables {
values.remove(&scope.column);
}
values.insert(
schema_table.primary_key.clone(),
Value::from(row_id.to_owned()),
);
self.validate_author_encode(schema_table, row_id, &values)?;
self.record_outbox_commit(vec![OutboxOp {
upsert: true,
table: table.to_owned(),
row_id: row_id.to_owned(),
base_version,
values: Some(values),
}])
}
fn stored_key_fallback(&self, table: &TableSchema, row_id: &str) -> Option<Row> {
let selector = self.encryption.key_id_columns.get(&table.name)?;
let index = table
.columns
.iter()
.position(|column| &column.name == selector)?;
let sql = format!(
"SELECT {} FROM {} WHERE {}",
quote_ident(selector),
visible_table(&table.name),
row_id_predicate(table)
);
let key: Option<String> = self
.conn
.query_row(&sql, rusqlite::params![row_id], |row| row.get(0))
.ok()?;
let key = key.filter(|key| !key.is_empty())?;
let mut fallback: Row = vec![None; table.columns.len()];
fallback[index] = Some(ColumnValue::String(key));
Some(fallback)
}
fn validate_author_encode(
&self,
table: &TableSchema,
row_id: &str,
values: &Map<String, Value>,
) -> Result<(), String> {
match encode_sparse_row_json(table, row_id, values, &self.encryption, None) {
Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Ok(()),
result => result.map(|_| ()),
}
}
fn drop_unencodable_outbox(&mut self) -> Result<bool, String> {
if !self
.schema
.tables
.iter()
.any(|table| table.has_encrypted_columns())
{
return Ok(false);
}
let failures = self
.outbox
.iter()
.filter_map(|commit| {
commit
.ops
.iter()
.find_map(|op| {
let values = op.values.as_ref().filter(|_| op.upsert)?;
let table = self.schema.table(&op.table)?;
let fallback = self.stored_key_fallback(table, &op.row_id);
match encode_sparse_row_json(
table,
&op.row_id,
values,
&self.encryption,
fallback.as_ref(),
) {
Err(error) if error.starts_with(ENCRYPT_FAILED_CODE) => Some(error),
_ => None,
}
})
.map(|error| (commit.clone(), error))
})
.collect::<Vec<_>>();
if failures.is_empty() {
return Ok(false);
}
for (commit, message) in failures {
let rejection = RejectionRecord {
client_commit_id: commit.client_commit_id.clone(),
op_index: 0,
code: ENCRYPT_FAILED_CODE.to_owned(),
message,
retryable: false,
details: None,
operation: commit.ops.first().map(CommitOperation::from),
};
self.persist_commit_outcome(
&commit.client_commit_id,
CommitOutcomeStatus::Rejected,
&[CommitOperationOutcome::Error {
rejection: rejection.clone(),
}],
Some(&commit.ops),
)?;
self.delete_outbox_persisted(&commit.client_commit_id)?;
self.outbox
.retain(|candidate| candidate.client_commit_id != commit.client_commit_id);
self.rejections.push(rejection);
}
self.prune_commit_outcomes()?;
self.overlay_dirty.set(true);
self.rebuild_overlay();
Ok(true)
}
pub fn pending_commit_ids(&self) -> Vec<String> {
self.outbox
.iter()
.map(|c| c.client_commit_id.clone())
.collect()
}
pub fn pending_payloads(&self) -> Vec<Vec<u8>> {
let mut payloads = Vec::new();
for commit in &self.outbox {
for op in &commit.ops {
let Some(values) = op.values.as_ref().filter(|_| op.upsert) else {
continue;
};
let Some(table) = self.schema.table(&op.table) else {
continue;
};
let fallback = self.stored_key_fallback(table, &op.row_id);
if let Ok(payload) = encode_sparse_row_json(
table,
&op.row_id,
values,
&self.encryption,
fallback.as_ref(),
) {
payloads.push(payload);
}
}
}
payloads
}
pub fn conflicts(&self) -> &[ConflictRecord] {
&self.conflicts
}
pub fn rejections(&self) -> &[RejectionRecord] {
&self.rejections
}
pub fn resolve_commit_outcome(
&mut self,
input: ResolveCommitOutcomeInput,
) -> Result<CommitOutcome, String> {
let current = self
.commit_outcome(&input.client_commit_id)?
.ok_or_else(|| {
format!(
"sync.outcome_not_found: no durable outcome exists for {:?}",
input.client_commit_id
)
})?;
if current.resolution != CommitOutcomeResolution::Active {
return Ok(current);
}
if input.resolution == CommitOutcomeResolution::Active {
return Err("sync.invalid_request: resolution must leave active state".to_owned());
}
match input.resolution {
CommitOutcomeResolution::Superseded => {
let replacement = input
.replacement_client_commit_id
.as_deref()
.filter(|value| !value.is_empty() && *value != input.client_commit_id)
.ok_or_else(|| {
"sync.invalid_request: superseded outcomes require a distinct replacementClientCommitId"
.to_owned()
})?;
let _ = replacement;
}
_ if input.replacement_client_commit_id.is_some() => {
return Err(
"sync.invalid_request: replacementClientCommitId is valid only for superseded outcomes"
.to_owned(),
);
}
_ => {}
}
let allowed = match current.status {
CommitOutcomeStatus::Conflict => matches!(
input.resolution,
CommitOutcomeResolution::ResolvedKeepServer | CommitOutcomeResolution::Superseded
),
CommitOutcomeStatus::Rejected => {
input.resolution == CommitOutcomeResolution::Superseded
}
CommitOutcomeStatus::Applied | CommitOutcomeStatus::Cached => {
input.resolution == CommitOutcomeResolution::Dismissed
}
};
if !allowed {
return Err(format!(
"sync.invalid_request: resolution {:?} is invalid for {:?} outcome",
input.resolution, current.status
));
}
self.begin_observation("syncular_outcome_resolution")?;
let result = (|| {
self.conn
.execute(
"UPDATE _syncular_commit_outcomes
SET resolution = ?1, resolved_at_ms = ?2,
replacement_client_commit_id = ?3
WHERE client_commit_id = ?4 AND resolution = 'active'",
rusqlite::params![
Self::outcome_resolution_name(input.resolution),
self.clock_now_ms(),
input.replacement_client_commit_id,
input.client_commit_id
],
)
.map_err(|error| error.to_string())?;
let resolved = self
.commit_outcome(¤t.client_commit_id)?
.ok_or_else(|| "sync.outcome_not_found: outcome disappeared".to_owned())?;
self.prune_commit_outcomes()?;
Ok(resolved)
})();
let resolved = match result {
Ok(outcome) => outcome,
Err(error) => {
self.rollback_observation("syncular_outcome_resolution");
return Err(error);
}
};
self.conflicts
.retain(|record| record.client_commit_id != current.client_commit_id);
self.rejections
.retain(|record| record.client_commit_id != current.client_commit_id);
let batch = ChangeAccumulator {
conflicts: current.status == CommitOutcomeStatus::Conflict,
rejections: current.status == CommitOutcomeStatus::Rejected,
outcomes: true,
..ChangeAccumulator::default()
};
if let Err(error) = self.finish_observation("syncular_outcome_resolution", batch) {
self.rollback_observation("syncular_outcome_resolution");
return Err(error);
}
Ok(resolved)
}
pub fn schema_floor(&self) -> Option<&SchemaFloor> {
self.schema_floor.as_ref()
}
pub fn lease_state(&self) -> Option<&LeaseState> {
self.lease_state.as_ref()
}
fn record_lease_error(&mut self, code: &str) {
if code != "sync.auth_lease_required" && code != "sync.auth_lease_revoked" {
return;
}
let mut next = self.lease_state.clone().unwrap_or_default();
next.error_code = Some(code.to_owned());
self.set_lease_state(Some(next));
}
fn set_lease_state(&mut self, next: Option<LeaseState>) {
if self.lease_state == next {
return;
}
if self.begin_observation("syncular_lease").is_err() {
return;
}
self.lease_state = next;
if let Some(lease) = &self.lease_state {
if let Ok(json) = serde_json::to_string(lease) {
self.set_meta(LEASE_STATE_KEY, &json);
}
} else {
self.delete_meta(LEASE_STATE_KEY);
}
let batch = ChangeAccumulator {
status: true,
..ChangeAccumulator::default()
};
if self.finish_observation("syncular_lease", batch).is_err() {
self.rollback_observation("syncular_lease");
}
}
fn set_schema_floor(&mut self, next: Option<SchemaFloor>) {
if self.schema_floor == next {
return;
}
if self.begin_observation("syncular_schema_floor").is_err() {
return;
}
self.schema_floor = next;
self.stopped = self.schema_floor.is_some();
if let Some(floor) = &self.schema_floor {
if let Ok(json) = serde_json::to_string(floor) {
self.set_meta(SCHEMA_FLOOR_KEY, &json);
}
} else {
self.delete_meta(SCHEMA_FLOOR_KEY);
}
let batch = ChangeAccumulator {
status: true,
..ChangeAccumulator::default()
};
if self
.finish_observation("syncular_schema_floor", batch)
.is_err()
{
self.rollback_observation("syncular_schema_floor");
}
}
fn set_upgrading(&mut self, value: bool) {
if self.upgrading == value {
return;
}
if self.begin_observation("syncular_upgrading").is_err() {
return;
}
self.upgrading = value;
let batch = ChangeAccumulator {
status: true,
..ChangeAccumulator::default()
};
if self
.finish_observation("syncular_upgrading", batch)
.is_err()
{
self.rollback_observation("syncular_upgrading");
}
}
pub fn sync_needed(&self) -> bool {
self.sync_needed
}
pub fn subscription_state(&self, id: &str) -> Option<SubscriptionStateView> {
let sub = self.subs.iter().find(|s| s.id == id)?;
Some(SubscriptionStateView {
id: sub.id.clone(),
table: sub.table.clone(),
status: sub.state.name().to_owned(),
cursor: sub.cursor,
has_resume_token: sub.bootstrap_state.is_some(),
effective_scopes: sub.effective.as_ref().map(|e| scope_map_to_json(e)),
reason_code: sub.reason_code.clone(),
})
}
pub fn read_rows(&self, table: &str) -> Result<Vec<RowState>, String> {
let schema_table = self
.schema
.table(table)
.ok_or_else(|| format!("unknown table {table:?}"))?;
let sql = format!(
"SELECT * FROM {} ORDER BY {} ASC",
visible_table(table),
quote_ident(&schema_table.primary_key)
);
let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
let mut rows = stmt.query([]).map_err(|e| e.to_string())?;
let mut out = Vec::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
let mut values = Map::new();
for (i, column) in schema_table.columns.iter().enumerate() {
let value = row.get_ref(i).map_err(|e| e.to_string())?;
values.insert(column.name.clone(), sql_ref_to_json(column, value));
}
let version: i64 = row
.get(schema_table.columns.len())
.map_err(|e| e.to_string())?;
let row_id = match values.get(&schema_table.primary_key) {
Some(Value::String(s)) => s.clone(),
Some(Value::Number(n)) => n.to_string(),
Some(Value::Bool(b)) => b.to_string(),
other => format!("{}", other.cloned().unwrap_or(Value::Null)),
};
out.push(RowState {
row_id,
version,
values,
});
}
Ok(out)
}
#[cfg(feature = "crdt-yjs")]
fn crdt_column_bytes(
&self,
table: &str,
row_id: &str,
column: &str,
) -> Result<Option<Vec<u8>>, String> {
let schema_table = self
.schema
.table(table)
.ok_or_else(|| format!("unknown table {table:?}"))?;
let col = schema_table
.columns
.iter()
.find(|c| c.name == column)
.ok_or_else(|| format!("table {table:?} has no column {column:?}"))?;
if col.ty != ColumnType::Crdt {
return Err(format!("column {column:?} is not a crdt column (§5.10.1)"));
}
let sql = format!(
"SELECT {} FROM {} WHERE {}",
quote_ident(column),
visible_table(table),
row_id_predicate(schema_table)
);
let bytes: Option<Vec<u8>> = self
.conn
.prepare_cached(&sql)
.map_err(|error| error.to_string())?
.query_row(rusqlite::params![row_id], |row| {
row.get::<_, Option<Vec<u8>>>(0)
})
.map_err(|e| match e {
rusqlite::Error::QueryReturnedNoRows => "no such row".to_owned(),
other => other.to_string(),
})?;
Ok(bytes)
}
#[cfg(feature = "crdt-yjs")]
pub fn crdt_text(
&self,
table: &str,
row_id: &str,
column: &str,
name: &str,
) -> Result<String, String> {
let bytes = self
.crdt_column_bytes(table, row_id, column)?
.unwrap_or_default();
crate::crdt::text(&bytes, name)
}
#[cfg(feature = "crdt-yjs")]
pub fn crdt_insert_text(
&mut self,
table: &str,
row_id: &str,
column: &str,
name: &str,
index: u32,
value: &str,
) -> Result<String, String> {
let current = self
.crdt_column_bytes(table, row_id, column)?
.unwrap_or_default();
let update = crate::crdt::insert_text(¤t, name, index, value)?;
self.crdt_push_update(table, row_id, column, &update)
}
#[cfg(feature = "crdt-yjs")]
pub fn crdt_delete_text(
&mut self,
table: &str,
row_id: &str,
column: &str,
name: &str,
index: u32,
len: u32,
) -> Result<String, String> {
let current = self
.crdt_column_bytes(table, row_id, column)?
.unwrap_or_default();
let update = crate::crdt::delete_text(¤t, name, index, len)?;
self.crdt_push_update(table, row_id, column, &update)
}
#[cfg(feature = "crdt-yjs")]
pub fn crdt_apply_update(
&mut self,
table: &str,
row_id: &str,
column: &str,
update: &[u8],
) -> Result<String, String> {
let current = self
.crdt_column_bytes(table, row_id, column)?
.unwrap_or_default();
let next = crate::crdt::apply_update(¤t, update)?;
self.crdt_push_update(table, row_id, column, &next)
}
#[cfg(feature = "crdt-yjs")]
fn crdt_push_update(
&mut self,
table: &str,
row_id: &str,
column: &str,
crdt_bytes: &[u8],
) -> Result<String, String> {
let mut bytes_obj = Map::new();
bytes_obj.insert("$bytes".to_owned(), Value::from(bytes_to_hex(crdt_bytes)));
let mut partial = Map::new();
partial.insert(column.to_owned(), Value::Object(bytes_obj));
self.patch(table, row_id, partial, None)
}
pub fn query(&self, sql: &str, params: &[QueryValue]) -> Result<Vec<QueryRow>, String> {
query_connection(&self.conn, sql, params).map_err(|failure| failure.to_string())
}
#[cfg(feature = "bench-internals")]
#[doc(hidden)]
pub fn benchmark_connection(&mut self) -> &mut Connection {
&mut self.conn
}
#[cfg(feature = "bench-internals")]
#[doc(hidden)]
pub fn benchmark_phases(
&self,
enabled: Option<bool>,
reset: bool,
) -> Result<Option<Value>, String> {
self.benchmark_phases.configure(enabled, reset)?;
Ok(self.benchmark_phases.snapshot())
}
pub fn query_snapshot(
&mut self,
sql: &str,
params: &[QueryValue],
coverage: &[WindowCoverage],
owner: Option<QueryOwner<'_>>,
) -> Result<QuerySnapshot, String> {
let result = snapshot_connection(&self.conn, sql, params, coverage);
if let Some(owner) = owner {
self.record_query_read(owner, result.as_ref().err());
}
result.map_err(|failure| failure.to_string())
}
pub fn record_query_read(&mut self, owner: QueryOwner<'_>, failure: Option<&QueryReadFailure>) {
let position = self
.query_failures
.iter()
.position(|entry| entry.id == owner.id);
let Some(failure) = failure else {
if let Some(index) = position {
self.query_failures.remove(index);
}
return;
};
let tables: Vec<String> = BTreeSet::from_iter(owner.tables.iter().copied())
.into_iter()
.map(str::to_owned)
.collect();
let code = failure.code.unwrap_or("client.query_failed").to_owned();
if position.is_some_and(|index| {
let entry = &self.query_failures[index];
entry.tables == tables && entry.code == code && entry.sqlite_code == failure.sqlite_code
}) {
return;
}
if let Some(index) = position {
self.query_failures.remove(index);
}
self.query_failures.push(DiagnosticQueryFailure {
id: owner.id.to_owned(),
tables,
code,
sqlite_code: failure.sqlite_code,
at_ms: self.clock_now_ms(),
});
if self.query_failures.len() > MAX_DIAGNOSTIC_QUERY_FAILURES {
self.query_failures.remove(0);
}
}
fn build_request(&self, url_capable: bool) -> (Message, RequestMeta) {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::RequestPrepare);
let log_epoch = self.get_meta(LOG_EPOCH_KEY);
let mut frames = vec![Frame::ReqHeader {
client_id: self.client_id.clone(),
schema_version: self.schema.version,
log_epoch: log_epoch.clone(),
}];
let mut pushed_ids = Vec::new();
let mut ops_in_request = 0usize;
let mut deferred_commits = 0usize;
#[cfg(feature = "bench-internals")]
let outbox_phase = self.benchmark_phases.start(Phase::OutboxEncode);
for (index, commit) in self
.outbox
.iter()
.take(if log_epoch.is_some() {
self.outbox.len()
} else {
0
})
.enumerate()
{
if ops_in_request > 0 && ops_in_request + commit.ops.len() > PUSH_OPS_PER_REQUEST {
deferred_commits = self.outbox.len() - index;
break;
}
ops_in_request += commit.ops.len();
let operations = commit
.ops
.iter()
.map(|op| {
let payload = op.values.as_ref().and_then(|values| {
let table = self.schema.table(&op.table)?;
let fallback = self.stored_key_fallback(table, &op.row_id);
encode_sparse_row_json(
table,
&op.row_id,
values,
&self.encryption,
fallback.as_ref(),
)
.ok()
});
ssp2::model::Operation {
table: op.table.clone(),
row_id: op.row_id.clone(),
op: if op.upsert { Op::Upsert } else { Op::Delete },
base_version: op.base_version,
payload,
}
})
.collect();
frames.push(Frame::PushCommit {
client_commit_id: commit.client_commit_id.clone(),
operations,
});
pushed_ids.push(commit.client_commit_id.clone());
}
#[cfg(feature = "bench-internals")]
drop(outbox_phase);
let accept = self.limits.accept.unwrap_or(if url_capable {
DEFAULT_ACCEPT | ACCEPT_SIGNED_URLS
} else {
DEFAULT_ACCEPT
});
frames.push(Frame::PullHeader {
limit_commits: self.limits.limit_commits.unwrap_or(0),
limit_snapshot_rows: self.limits.limit_snapshot_rows.unwrap_or(0),
max_snapshot_pages: self.limits.max_snapshot_pages.unwrap_or(0),
accept,
});
let mut fresh = Vec::new();
for sub in &self.subs {
if sub.state != SubState::Active {
continue;
}
let mut scopes = sub.requested.clone();
sort_scope_map(&mut scopes);
frames.push(Frame::Subscription {
id: sub.id.clone(),
table: sub.table.clone(),
scopes,
params: sub.params.clone().map(RawJson),
cursor: sub.cursor,
bootstrap_state: sub.bootstrap_state.clone().map(RawJson),
});
fresh.push((
sub.id.clone(),
sub.cursor < 0 && sub.bootstrap_state.is_none(),
));
}
let message = Message {
wire_version: WIRE_VERSION,
msg_kind: MsgKind::Request,
frames,
};
(
message,
RequestMeta {
pushed_ids,
fresh,
accept,
deferred_commits,
},
)
}
pub fn progress(&self) -> ProgressObserver {
self.progress.clone()
}
pub fn sync(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
self.progress.start();
self.round_retry_delay_ms = None;
let started_at_ms = self.clock_now_ms();
let outcome = self.sync_inner(transport);
let retry_delay_ms = self.round_retry_delay_ms;
self.progress.update(|p| match &outcome {
SyncOutcome::Ok(report) if report.failed.is_empty() => {
p.state = ProgressState::Complete
}
SyncOutcome::Ok(_) => {
p.state = ProgressState::Failed;
p.error_code = Some("sync.scope_revoked".into());
}
SyncOutcome::Failed { error_code, .. } => {
p.state = ProgressState::Failed;
p.error_code = Some(Self::diagnostic_code(error_code));
p.retry_delay_ms = retry_delay_ms;
}
SyncOutcome::RealtimeUnavailable { .. } => {
p.state = ProgressState::Failed;
p.error_code = Some(REALTIME_UNAVAILABLE_CODE.to_owned());
p.retry_delay_ms = retry_delay_ms;
}
});
let completed_at_ms = self.clock_now_ms();
self.last_round = Some(match &outcome {
SyncOutcome::Ok(report) => DiagnosticLastRound {
status: "succeeded".to_owned(),
started_at_ms,
completed_at_ms,
duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
counters: Some(DiagnosticRoundCounters {
pushed: report.pushed,
applied: report.applied.len(),
rejected: report.rejected.len(),
retryable: report.retryable.len(),
conflicts: report.conflicts,
commits_applied: report.commits_applied,
segment_rows_applied: report.segment_rows_applied,
bootstrapping: report.bootstrapping.len(),
resets: report.resets.len(),
revoked: report.revoked.len(),
failed: report.failed.len(),
deferred_commits: report.deferred_commits,
}),
error_code: None,
},
SyncOutcome::Failed { error_code, .. } => DiagnosticLastRound {
status: "failed".to_owned(),
started_at_ms,
completed_at_ms,
duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
counters: None,
error_code: Some(Self::diagnostic_code(error_code)),
},
SyncOutcome::RealtimeUnavailable { .. } => DiagnosticLastRound {
status: "failed".to_owned(),
started_at_ms,
completed_at_ms,
duration_ms: completed_at_ms.saturating_sub(started_at_ms).max(0),
counters: None,
error_code: Some(REALTIME_UNAVAILABLE_CODE.to_owned()),
},
});
outcome
}
fn sync_inner(&mut self, transport: &mut dyn Transport) -> SyncOutcome {
if self.stopped {
return SyncOutcome::Ok(SyncReport {
schema_floor: self.schema_floor.clone(),
..SyncReport::default()
});
}
self.set_sync_needed(false, false);
if self.get_meta(LOG_EPOCH_KEY).is_some() && self.schema_has_blobs() {
if let Err(TransportError { code, message, .. }) = self.flush_blob_uploads(transport) {
if Self::retryable_failure_code(&code) {
self.schedule_background_retry();
}
return SyncOutcome::Failed {
error_code: code,
message,
};
}
}
if let Err(message) = self.drain_pending_evictions() {
return SyncOutcome::Failed {
error_code: "storage.failed".into(),
message,
};
}
if let Err(message) = self.drop_unencodable_outbox() {
return SyncOutcome::Failed {
error_code: "storage.failed".into(),
message,
};
}
let (message, meta) = self.build_request(transport.supports_url_fetch());
let request_bytes = {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::RequestEncode);
encode_message(&message)
};
let round = if self.realtime_state == RealtimeState::Connected {
transport.realtime_sync(&request_bytes)
} else if self.realtime_policy == RealtimePolicy::Required {
let retry_delay_ms = self.schedule_background_retry();
self.realtime_retry_delay_ms = Some(retry_delay_ms);
return SyncOutcome::RealtimeUnavailable {
state: self.realtime_state,
reason_code: self.realtime_reason_code.clone(),
retry_delay_ms,
};
} else {
transport.sync(&request_bytes)
};
let realtime_round = self.realtime_state == RealtimeState::Connected;
let response_bytes = match round {
Ok(bytes) => bytes,
Err(TransportError { code, message, .. }) => {
if realtime_round {
self.set_realtime_state(RealtimeState::Lost, Some(&code));
}
self.record_lease_error(&code);
if Self::retryable_failure_code(&code) {
let retry_delay_ms = self.schedule_background_retry();
if realtime_round {
self.realtime_retry_delay_ms = Some(retry_delay_ms);
}
}
return SyncOutcome::Failed {
error_code: code,
message,
};
}
};
#[cfg(feature = "bench-internals")]
let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
let response = match decode_message(&response_bytes) {
Ok(message) => message,
Err(error) => {
return SyncOutcome::Failed {
error_code: error.code.as_str().to_owned(),
message: error.detail,
};
}
};
#[cfg(feature = "bench-internals")]
drop(decode_phase);
if response.msg_kind != MsgKind::Response {
return SyncOutcome::Failed {
error_code: "sync.invalid_request".to_owned(),
message: "expected a response message".to_owned(),
};
}
let mut outcome = self.process_response(transport, response, &meta);
if let SyncOutcome::Ok(report) = &mut outcome {
report.deferred_commits = meta.deferred_commits;
}
match &outcome {
SyncOutcome::Ok(_) => self.reset_background_retry(),
SyncOutcome::Failed { error_code, .. } if Self::retryable_failure_code(error_code) => {
self.schedule_background_retry();
}
SyncOutcome::Failed { .. } => {}
SyncOutcome::RealtimeUnavailable { .. } => {}
}
if meta.deferred_commits > 0 {
self.set_sync_needed(true, true);
}
self.previous_version_lifetime_check();
outcome
}
pub fn sync_until_idle(
&mut self,
transport: &mut dyn Transport,
max_rounds: Option<u32>,
) -> SyncOutcome {
let rounds = max_rounds.unwrap_or(20).max(1);
let mut aggregate = SyncReport::default();
let mut spent = 0;
while spent < rounds {
spent += 1;
let before = self
.subs
.iter()
.map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
.collect::<Vec<_>>();
let outcome = self.sync(transport);
match outcome {
SyncOutcome::Failed {
error_code,
message,
} => {
return SyncOutcome::Failed {
error_code,
message,
};
}
SyncOutcome::RealtimeUnavailable { .. } => return outcome,
SyncOutcome::Ok(report) => {
if max_rounds.is_none()
&& report.segment_rows_applied > 0
&& !report.bootstrapping.is_empty()
&& before
!= self
.subs
.iter()
.map(|sub| (sub.id.clone(), sub.bootstrap_state.clone()))
.collect::<Vec<_>>()
{
spent = 0;
}
aggregate.pushed += report.pushed;
aggregate.applied.extend(report.applied.iter().cloned());
aggregate.rejected.extend(report.rejected.iter().cloned());
aggregate.retryable.extend(report.retryable.iter().cloned());
aggregate.conflicts += report.conflicts;
aggregate.commits_applied += report.commits_applied;
aggregate.segment_rows_applied += report.segment_rows_applied;
aggregate.bootstrapping = report.bootstrapping.clone();
aggregate.resets.extend(report.resets.iter().cloned());
aggregate.revoked.extend(report.revoked.iter().cloned());
aggregate.failed.extend(report.failed.iter().cloned());
aggregate.deferred_commits = report.deferred_commits;
if report.schema_floor.is_some() {
aggregate.schema_floor = report.schema_floor.clone();
}
let more = !report.bootstrapping.is_empty()
|| report.commits_applied > 0
|| report.segment_rows_applied > 0
|| !report.resets.is_empty()
|| self.sync_needed;
if !more {
return SyncOutcome::Ok(aggregate);
}
}
}
}
SyncOutcome::Failed {
error_code: "sync.invalid_request".into(),
message: "sync did not reach idle within the round budget".into(),
}
}
fn process_response(
&mut self,
transport: &mut dyn Transport,
response: Message,
meta: &RequestMeta,
) -> SyncOutcome {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::ResponseApply);
let mut report = SyncReport {
pushed: meta.pushed_ids.len() as u32,
..SyncReport::default()
};
let mut rejection_details_by_commit: HashMap<String, BTreeMap<i32, RejectionDetails>> =
HashMap::new();
let pushed_ids = meta
.pushed_ids
.iter()
.map(String::as_str)
.collect::<HashSet<_>>();
let mut last_final_push_result_id: Option<String> = None;
for frame in &response.frames {
match frame {
Frame::PushResultDetails {
client_commit_id,
entries,
} => {
let details = rejection_details_by_commit
.entry(client_commit_id.clone())
.or_default();
for entry in entries {
let parsed = match RejectionDetails::parse(&entry.details.0) {
Ok(value) => value,
Err(message) => {
return SyncOutcome::Failed {
error_code: "sync.invalid_request".to_owned(),
message,
};
}
};
details.insert(entry.op_index, parsed);
}
}
Frame::PushResult {
client_commit_id,
status,
results,
..
} if pushed_ids.contains(client_commit_id.as_str())
&& Self::push_result_is_final(*status, results) =>
{
last_final_push_result_id = Some(client_commit_id.clone());
}
_ => {}
}
}
let mut frames = response.frames.into_iter().peekable();
if response.wire_version < 2 {
return SyncOutcome::Failed {
error_code: "client.invalid_host_response".to_owned(),
message: "server response does not carry wire version 2 log-epoch state".to_owned(),
};
}
match frames.next() {
Some(Frame::RespHeader {
required_schema_version,
latest_schema_version,
log_epoch,
reset_required,
}) => {
let Some(log_epoch) = log_epoch else {
return SyncOutcome::Failed {
error_code: "client.invalid_host_response".to_owned(),
message: "response header omits logEpoch".to_owned(),
};
};
let Some(reset_required) = reset_required else {
return SyncOutcome::Failed {
error_code: "client.invalid_host_response".to_owned(),
message: "response header omits resetRequired".to_owned(),
};
};
if let Some(required) = required_schema_version {
let floor = SchemaFloor {
required_schema_version: Some(required),
latest_schema_version,
};
self.set_schema_floor(Some(floor.clone()));
report.schema_floor = Some(floor);
return SyncOutcome::Ok(report);
}
if reset_required {
if frames.next().is_some() {
return SyncOutcome::Failed {
error_code: "client.invalid_host_response".to_owned(),
message: "log-epoch reset response contains body frames".to_owned(),
};
}
match self.run_log_epoch_reset(&log_epoch) {
Ok(resets) => {
report.resets = resets;
return SyncOutcome::Ok(report);
}
Err(message) => {
return SyncOutcome::Failed {
error_code: "sync.local_corrupt".to_owned(),
message,
};
}
}
}
if self.get_meta(LOG_EPOCH_KEY).as_deref() != Some(log_epoch.as_str()) {
return SyncOutcome::Failed {
error_code: "client.invalid_host_response".to_owned(),
message: "server changed logEpoch without requiring a reset".to_owned(),
};
}
}
_ => {
return SyncOutcome::Failed {
error_code: "sync.invalid_request".to_owned(),
message: "response does not start with RESP_HEADER".to_owned(),
};
}
}
let mut failure: Option<(String, String)> = None;
while let Some(frame) = frames.next() {
match frame {
frame @ Frame::PushResult { .. } => {
let successful = matches!(
&frame,
Frame::PushResult {
status: PushStatus::Applied | PushStatus::Cached,
..
}
);
let mut results = vec![frame];
if successful {
while matches!(
frames.peek(),
Some(Frame::PushResult {
status: PushStatus::Applied | PushStatus::Cached,
..
})
) {
if let Some(next) = frames.next() {
results.push(next);
}
}
}
if self
.handle_push_results(
&results,
&pushed_ids,
&rejection_details_by_commit,
&mut report,
last_final_push_result_id.as_deref(),
)
.is_err()
{
failure = Some((
"client.outcome_persistence_failed".to_owned(),
"local commit outcome could not be persisted".to_owned(),
));
break;
}
}
Frame::PushResultDetails { .. } => {}
Frame::SubStart {
id,
status,
reason_code,
effective_scopes,
bootstrap: _,
} => {
let mut body = Vec::new();
let mut sub_end: Option<(i64, Option<String>)> = None;
for inner in frames.by_ref() {
match inner {
Frame::SubEnd {
next_cursor,
bootstrap_state,
} => {
sub_end = Some((next_cursor, bootstrap_state.map(|r| r.0)));
break;
}
Frame::Unknown { .. } => {}
other => body.push(other),
}
}
if let Err(SectionError::Abort(code, message)) = self.process_section(
transport,
&id,
status,
&reason_code,
effective_scopes,
body,
sub_end,
meta,
&mut report,
) {
failure = Some((code, message));
break;
}
}
Frame::Lease {
lease_id,
expires_at_ms,
} => {
self.set_lease_state(Some(LeaseState {
lease_id: Some(lease_id),
expires_at_ms: Some(expires_at_ms),
error_code: None,
}));
}
Frame::Error { code, message, .. } => {
failure = Some((code, message));
break;
}
Frame::Unknown { .. } => {}
_ => {
failure = Some((
"sync.invalid_request".to_owned(),
"unexpected frame in response".to_owned(),
));
break;
}
}
}
if self.overlay_dirty.get() {
self.rebuild_overlay();
}
if let Some((error_code, message)) = failure {
return SyncOutcome::Failed {
error_code,
message,
};
}
if let Err(message) = self.drain_pending_evictions() {
return SyncOutcome::Failed {
error_code: "storage.failed".into(),
message,
};
}
self.ack_after_pull(transport);
if self.upgrading && report.bootstrapping.is_empty() {
self.set_upgrading(false);
}
SyncOutcome::Ok(report)
}
fn handle_push_results(
&mut self,
frames: &[Frame],
pushed_ids: &HashSet<&str>,
rejection_details_by_commit: &HashMap<String, BTreeMap<i32, RejectionDetails>>,
report: &mut SyncReport,
last_final_push_result_id: Option<&str>,
) -> Result<(), String> {
self.begin_observation("syncular_push_results")?;
let conflict_count = self.conflicts.len();
let rejection_count = self.rejections.len();
let applied_count = report.applied.len();
let rejected_count = report.rejected.len();
let retryable_count = report.retryable.len();
let was_dirty = self.overlay_dirty.get();
let mut removed = Vec::new();
let mut batch = ChangeAccumulator::default();
let persisted = (|| {
for frame in frames {
let Frame::PushResult {
client_commit_id,
status,
results,
..
} = frame
else {
return Err("unexpected acknowledgement frame".to_owned());
};
if !pushed_ids.contains(client_commit_id.as_str()) {
continue;
}
let Some(index) = self
.outbox
.iter()
.position(|commit| commit.client_commit_id == *client_commit_id)
else {
continue;
};
let status = *status;
let rejection_details = rejection_details_by_commit.get(client_commit_id);
let operations = self.outbox[index].ops.clone();
match status {
PushStatus::Applied | PushStatus::Cached => {
let journal_results = results
.iter()
.map(|result| {
let op_index = match result {
OpResult::Applied { op_index }
| OpResult::Conflict { op_index, .. }
| OpResult::Error { op_index, .. } => *op_index,
};
CommitOperationOutcome::Applied { op_index }
})
.collect::<Vec<_>>();
let outcome_status = if status == PushStatus::Applied {
CommitOutcomeStatus::Applied
} else {
CommitOutcomeStatus::Cached
};
let persisted = self
.persist_commit_outcome(
client_commit_id,
outcome_status,
&journal_results,
None,
)
.and_then(|()| self.delete_outbox_persisted(client_commit_id));
persisted?;
self.overlay_dirty.set(true);
batch.status = true;
batch.outcomes = true;
}
PushStatus::Rejected => {
if results.iter().any(|result| {
matches!(
result,
OpResult::Error {
code,
retryable: true,
..
} if code == "sync.idempotency_cache_miss"
)
}) {
report.retryable.push(client_commit_id.to_owned());
continue;
}
let mut journal_results = Vec::with_capacity(results.len());
let mut conflicts = Vec::new();
let mut rejections = Vec::new();
for result in results {
match result {
OpResult::Applied { op_index } => {
journal_results.push(CommitOperationOutcome::Applied {
op_index: *op_index,
});
}
OpResult::Conflict {
op_index,
code,
message,
server_version,
server_row,
conflict_columns: conflict_columns_bitmap,
} => {
let operation = operations
.get(*op_index as usize)
.map(CommitOperation::from);
let (table, row_id) = operation
.as_ref()
.map(|op| (op.table.clone(), op.row_id.clone()))
.unwrap_or_default();
let server_row_json = self
.schema
.table(&table)
.and_then(|t| {
decode_row_bytes(t, server_row, &self.encryption)
.ok()
.map(|row| (t, row))
})
.map(|(t, row)| {
let mut map = Map::new();
for (i, column) in t.columns.iter().enumerate() {
map.insert(
column.name.clone(),
column_value_to_json(
row.get(i).unwrap_or(&None),
),
);
}
map
})
.unwrap_or_default();
let conflict_columns = self
.schema
.table(&table)
.map(|t| {
t.columns
.iter()
.enumerate()
.filter(|(index, _)| {
conflict_columns_bitmap
.get(index / 8)
.is_some_and(|byte| {
byte & (1 << (index % 8)) != 0
})
})
.map(|(_, column)| column.name.clone())
.collect::<Vec<_>>()
})
.unwrap_or_default();
let conflict = ConflictRecord {
client_commit_id: client_commit_id.to_owned(),
op_index: *op_index,
table,
row_id,
code: code.clone(),
message: message.clone(),
server_version: *server_version,
server_row: server_row_json,
conflict_columns,
operation,
};
journal_results.push(CommitOperationOutcome::Conflict {
conflict: conflict.clone(),
});
conflicts.push(conflict);
}
OpResult::Error {
op_index,
code,
message,
retryable,
} => {
let rejection = RejectionRecord {
client_commit_id: client_commit_id.to_owned(),
op_index: *op_index,
code: code.clone(),
message: message.clone(),
retryable: *retryable,
details: rejection_details
.and_then(|details| details.get(op_index))
.cloned(),
operation: operations
.get(*op_index as usize)
.map(CommitOperation::from),
};
journal_results.push(CommitOperationOutcome::Error {
rejection: rejection.clone(),
});
rejections.push(rejection);
}
}
}
let outcome_status = if conflicts.is_empty() {
CommitOutcomeStatus::Rejected
} else {
CommitOutcomeStatus::Conflict
};
let persisted = self
.persist_commit_outcome(
client_commit_id,
outcome_status,
&journal_results,
Some(&operations),
)
.and_then(|()| self.delete_outbox_persisted(client_commit_id));
persisted?;
batch.conflicts = !conflicts.is_empty();
batch.rejections = !rejections.is_empty();
batch.status = true;
batch.outcomes = true;
self.conflicts.extend(conflicts);
self.rejections.extend(rejections);
self.overlay_dirty.set(true);
}
}
removed.push((index, self.outbox.remove(index)));
if batch.status {
for operation in &operations {
if !self.record_row_scopes(
&mut batch,
&operation.table,
&operation.row_id,
false,
) {
batch.table(&operation.table);
}
}
}
match status {
PushStatus::Applied | PushStatus::Cached => {
report.applied.push(client_commit_id.to_owned())
}
PushStatus::Rejected => report.rejected.push(client_commit_id.to_owned()),
}
}
if batch.outcomes && frames.iter().any(|frame| matches!(frame, Frame::PushResult { client_commit_id, .. } if Some(client_commit_id.as_str()) == last_final_push_result_id)) {
self.prune_commit_outcomes()?;
}
self.finish_observation("syncular_push_results", batch)
})();
if let Err(error) = persisted {
self.rollback_observation("syncular_push_results");
for (index, commit) in removed.into_iter().rev() {
self.outbox.insert(index, commit);
}
self.conflicts.truncate(conflict_count);
self.rejections.truncate(rejection_count);
self.overlay_dirty.set(was_dirty);
report.applied.truncate(applied_count);
report.rejected.truncate(rejected_count);
report.retryable.truncate(retryable_count);
return Err(error);
}
report.conflicts += (self.conflicts.len() - conflict_count) as u32;
Ok(())
}
fn push_result_is_final(status: PushStatus, results: &[OpResult]) -> bool {
status != PushStatus::Rejected
|| !results.iter().any(|result| {
matches!(
result,
OpResult::Error {
code,
retryable: true,
..
} if code == "sync.idempotency_cache_miss"
)
})
}
#[allow(clippy::too_many_arguments)]
fn process_section(
&mut self,
transport: &mut dyn Transport,
id: &str,
status: SubStatus,
reason_code: &str,
effective_scopes: Vec<(String, Vec<String>)>,
body: Vec<Frame>,
sub_end: Option<(i64, Option<String>)>,
meta: &RequestMeta,
report: &mut SyncReport,
) -> Result<(), SectionError> {
let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
return Ok(()); };
if status != SubStatus::Active && sub_end.is_none() {
return Err(SectionError::Abort(
"sync.invalid_request".into(),
"subscription section without SUB_END".into(),
));
}
match status {
SubStatus::Revoked => {
self.begin_observation("syncular_revocation")
.map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
let previous = self.subs[sub_index].clone();
let was_dirty = self.overlay_dirty.get();
let previous_outbox = self.outbox.clone();
let rejection_count = self.rejections.len();
let revoked_count = report.revoked.len();
let failed_count = report.failed.len();
let result =
(|| {
let mut batch = ChangeAccumulator::default();
let registered = self.window_unit_by_sub_id(id);
let (table, effective) = {
let sub = &self.subs[sub_index];
(sub.table.clone(), sub.effective.clone().unwrap_or_default())
};
let purged = self.purge_scope_rows(&table, &effective);
match purged {
Ok(()) => {
self.record_scope_map(&mut batch, &table, &effective);
let sub = &mut self.subs[sub_index];
sub.state = SubState::Revoked;
sub.reason_code = Some(if reason_code.is_empty() {
"sync.scope_revoked".to_owned()
} else {
reason_code.to_owned()
});
report.revoked.push(id.to_owned());
let doomed_effective = effective;
let sub_table = table;
self.persist_sub(&self.subs[sub_index].clone()).map_err(
|message| SectionError::Abort("storage.failed".into(), message),
)?;
let dropped = self
.drop_doomed_outbox(&sub_table, &doomed_effective)
.map_err(|message| {
SectionError::Abort("storage.failed".to_owned(), message)
})?;
if dropped {
batch.status = true;
batch.rejections = true;
batch.outcomes = true;
}
self.delete_unreferenced_cached_blobs();
}
Err(()) => {
let sub = &mut self.subs[sub_index];
sub.state = SubState::Failed;
sub.reason_code = Some("sync.scope_revoked".to_owned());
report.failed.push(id.to_owned());
self.persist_sub(&self.subs[sub_index].clone()).map_err(
|message| SectionError::Abort("storage.failed".into(), message),
)?;
}
}
if let Some((base_key, unit)) = registered {
batch.window(&base_key, &self.subs[sub_index].table, &unit);
}
self.rebuild_overlay_if_dirty();
self.finish_observation("syncular_revocation", batch)
.map_err(|message| {
SectionError::Abort("storage.failed".to_owned(), message)
})?;
Ok(())
})();
if result.is_err() {
self.rollback_observation("syncular_revocation");
self.subs[sub_index] = previous;
self.overlay_dirty.set(was_dirty);
self.outbox = previous_outbox;
self.rejections.truncate(rejection_count);
report.revoked.truncate(revoked_count);
report.failed.truncate(failed_count);
}
result
}
SubStatus::Reset => {
self.begin_observation("syncular_reset")
.map_err(|message| SectionError::Abort("storage.failed".to_owned(), message))?;
let previous = self.subs[sub_index].clone();
let was_dirty = self.overlay_dirty.get();
let reset_count = report.resets.len();
let result = (|| {
let mut batch = ChangeAccumulator::default();
let registered = self.window_unit_by_sub_id(id);
let sub = &mut self.subs[sub_index];
sub.cursor = -1;
sub.bootstrap_state = None;
report.resets.push(id.to_owned());
self.persist_sub(&self.subs[sub_index].clone())
.map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
if let Some((base_key, unit)) = registered {
batch.window(&base_key, &self.subs[sub_index].table, &unit);
}
self.finish_observation("syncular_reset", batch)
.map_err(|message| {
SectionError::Abort("storage.failed".to_owned(), message)
})?;
Ok(())
})();
if result.is_err() {
self.rollback_observation("syncular_reset");
self.subs[sub_index] = previous;
self.overlay_dirty.set(was_dirty);
report.resets.truncate(reset_count);
}
result
}
SubStatus::Active => {
let fresh = meta
.fresh
.iter()
.find(|(fid, _)| fid == id)
.map(|(_, fresh)| *fresh)
.unwrap_or(false);
let previous = self.subs[sub_index].clone();
self.subs[sub_index].effective = Some(effective_scopes);
let outcome = self
.apply_section_body(transport, sub_index, body, fresh, meta, report)
.and_then(|()| {
let (cursor, bootstrap_state) = sub_end.ok_or_else(|| {
SectionError::Abort(
"sync.invalid_request".into(),
"subscription section without SUB_END".into(),
)
})?;
self.apply_sub_end(sub_index, cursor, bootstrap_state)
.map_err(|message| {
SectionError::Abort("storage.failed".into(), message)
})
});
match outcome {
Ok(()) => {
if self.subs[sub_index].bootstrap_state.is_some() {
report.bootstrapping.push(id.to_owned());
}
Ok(())
}
Err(SectionError::FailClosed) => {
self.subs[sub_index] = previous.clone();
self.begin_observation("syncular_section_failure")
.map_err(|message| {
SectionError::Abort("storage.failed".into(), message)
})?;
let persisted = (|| {
let mut batch = ChangeAccumulator::default();
if let Some((base_key, unit)) = self.window_unit_by_sub_id(id) {
batch.window(&base_key, &self.subs[sub_index].table, &unit);
}
self.subs[sub_index].state = SubState::Failed;
self.subs[sub_index].reason_code = Some("sync.scope_revoked".into());
self.persist_sub(&self.subs[sub_index])?;
self.finish_observation("syncular_section_failure", batch)
})();
if let Err(message) = persisted {
self.rollback_observation("syncular_section_failure");
self.subs[sub_index] = previous;
return Err(SectionError::Abort("storage.failed".into(), message));
}
report.failed.push(id.to_owned());
Ok(())
}
Err(error) => {
self.subs[sub_index] = previous;
Err(error)
}
}
}
}
}
fn apply_sub_end(
&mut self,
sub_index: usize,
cursor: i64,
bootstrap_state: Option<String>,
) -> Result<(), String> {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::CursorPersist);
let previous = self.subs[sub_index].clone();
self.begin_observation("syncular_sub_end")?;
let persisted = (|| {
let mut batch = ChangeAccumulator::default();
let completed = (previous.cursor < 0 || previous.bootstrap_state.is_some())
&& cursor >= 0
&& bootstrap_state.is_none();
if completed {
if let Some((base_key, unit)) = self.window_unit_by_sub_id(&previous.id) {
batch.window(&base_key, &previous.table, &unit);
}
}
let sub = &mut self.subs[sub_index];
sub.cursor = cursor;
sub.bootstrap_state = bootstrap_state;
sub.synced_once = true;
self.persist_sub(&self.subs[sub_index])?;
self.finish_observation("syncular_sub_end", batch)
})();
if persisted.is_err() {
self.rollback_observation("syncular_sub_end");
self.subs[sub_index] = previous;
}
persisted
}
fn apply_section_body(
&mut self,
transport: &mut dyn Transport,
sub_index: usize,
body: Vec<Frame>,
fresh: bool,
meta: &RequestMeta,
report: &mut SyncReport,
) -> Result<(), SectionError> {
let mut saw_segment = false;
for frame in body {
match frame {
Frame::Commit {
tables, changes, ..
} => {
self.apply_commit_frame(&tables, &changes)
.map_err(|(c, m)| SectionError::Abort(c, m))?;
report.commits_applied += 1;
}
Frame::SegmentInline { payload } => {
let segment = decode_rows_segment(&payload)
.map_err(|e| SectionError::Abort(e.code.as_str().to_owned(), e.detail))?;
self.progress.update(|p| {
p.phase = ProgressPhase::Import;
p.subscription_id = Some(self.subs[sub_index].id.clone());
p.table = Some(self.subs[sub_index].table.clone());
p.segment_id = None;
p.bytes_received = payload.len() as u64;
p.bytes_total = Some(payload.len() as u64);
p.rows_processed = 0;
p.rows_total =
Some(segment.blocks.iter().map(|block| block.len() as u64).sum());
});
let first = !saw_segment;
saw_segment = true;
let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
report.segment_rows_applied += applied;
}
Frame::SegmentRef {
segment_id,
byte_length,
media_type,
table,
row_count,
as_of_commit_seq,
scope_digest,
row_cursor,
next_row_cursor,
url,
url_expires_at_ms,
..
} => {
let advertised = match media_type {
MediaType::Rows => {
meta.accept & ACCEPT_EXTERNAL_ROWS != 0
|| meta.accept & ACCEPT_INLINE_ROWS != 0
}
MediaType::Sqlite => meta.accept & ACCEPT_SQLITE != 0,
};
if !advertised {
return Err(SectionError::Abort(
"sync.invalid_request".to_owned(),
format!(
"SEGMENT_REF mediaType {} was not advertised in accept (§4.2)",
media_type.name()
),
));
}
self.progress.update(|p| {
p.phase = ProgressPhase::Download;
p.subscription_id = Some(self.subs[sub_index].id.clone());
p.table = Some(table.clone());
p.segment_id = Some(segment_id.clone());
p.bytes_received = 0;
p.bytes_total = u64::try_from(byte_length).ok();
p.rows_processed = 0;
p.rows_total = u64::try_from(row_count).ok();
});
let observer = self.progress.clone();
let mut on_progress = |bytes| observer.update(|p| p.bytes_received = bytes);
let bytes = if let Some(url) = url {
if meta.accept & ACCEPT_SIGNED_URLS == 0 {
return Err(SectionError::Abort(
"sync.invalid_request".to_owned(),
"SEGMENT_REF carries a url but accept bit 3 was not advertised (§5.4)"
.to_owned(),
));
}
if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
return Err(SectionError::Abort(
"sync.segment_expired".to_owned(),
format!(
"signed URL for segment {segment_id} expired before fetch — re-pull mints fresh descriptors (§5.4)"
),
));
}
transport
.fetch_url(&url, &mut on_progress)
.map_err(|e| SectionError::Abort(e.code, e.message))?
} else {
let requested_scopes_json =
canonical_scope_json(&self.subs[sub_index].requested);
transport
.download_segment(
&SegmentRequest {
segment_id: segment_id.clone(),
table,
requested_scopes_json,
},
&mut on_progress,
)
.map_err(|e| SectionError::Abort(e.code, e.message))?
};
on_progress(bytes.len() as u64);
let digest = Sha256::digest(&bytes);
let expected = segment_id
.strip_prefix("sha256:")
.unwrap_or(segment_id.as_str());
if bytes_to_hex(&digest) != expected {
self.schedule_background_retry();
return Err(SectionError::Abort(
"sync.invalid_request".to_owned(),
"segment bytes do not match the content address (§5.1)".to_owned(),
));
}
self.progress.update(|p| p.phase = ProgressPhase::Import);
if media_type == MediaType::Sqlite {
if row_cursor.is_some() || next_row_cursor.is_some() {
return Err(SectionError::Abort(
"sync.invalid_request".to_owned(),
"sqlite segments are whole-table: rowCursor/nextRowCursor must be absent (§5.3)"
.to_owned(),
));
}
let first = !saw_segment;
saw_segment = true;
let applied = self.apply_sqlite_segment(
sub_index,
&bytes,
fresh && first,
row_count,
as_of_commit_seq,
&scope_digest,
)?;
report.segment_rows_applied += applied;
} else {
let segment = decode_rows_segment(&bytes).map_err(|e| {
SectionError::Abort(e.code.as_str().to_owned(), e.detail)
})?;
let first = row_cursor.is_none();
saw_segment = true;
let applied = self.apply_segment(sub_index, &segment, fresh && first)?;
report.segment_rows_applied += applied;
}
}
Frame::Unknown { .. } => {}
Frame::Error { code, message, .. } => {
return Err(SectionError::Abort(code, message))
}
_ => {
return Err(SectionError::Abort(
"sync.invalid_request".to_owned(),
"unexpected frame inside a subscription section".to_owned(),
));
}
}
}
Ok(())
}
fn apply_commit_frame(
&mut self,
tables: &[String],
changes: &[ssp2::model::Change],
) -> Result<(), (String, String)> {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::CommitApply);
let was_dirty = self.overlay_dirty.get();
self.begin_observation("syncular_commit")
.map_err(|message| ("storage.failed".into(), message))?;
let applied = (|| {
let mut batch = ChangeAccumulator::default();
self.record_commit_changes(&mut batch, tables, changes);
self.apply_commit_changes(tables, changes)?;
self.rebuild_overlay_if_dirty();
self.finish_observation("syncular_commit", batch)
.map_err(|message| ("storage.failed".into(), message))
})();
if applied.is_err() {
self.rollback_observation("syncular_commit");
self.overlay_dirty.set(was_dirty);
}
applied
}
fn apply_commit_changes(
&mut self,
tables: &[String],
changes: &[ssp2::model::Change],
) -> Result<(), (String, String)> {
let mirror_visible = !self.overlay_dirty.get()
&& (self.outbox.is_empty()
|| changes.iter().all(|change| {
tables
.get(change.table_index as usize)
.and_then(|name| self.schema.table(name))
.is_some_and(|table| !table.indexes.iter().any(|index| index.unique))
}));
for change in changes {
let table_name = tables.get(change.table_index as usize).ok_or_else(|| {
(
"sync.invalid_request".to_owned(),
"change tableIndex out of range".to_owned(),
)
})?;
let table = self.schema.table(table_name).ok_or_else(|| {
(
"sync.schema_mismatch".to_owned(),
format!("change targets unknown table {table_name:?}"),
)
})?;
match change.op {
Op::Upsert => {
let payload = change.row.as_ref().ok_or_else(|| {
(
"sync.invalid_request".to_owned(),
"upsert change without row payload".to_owned(),
)
})?;
let row = {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::RowDecode);
decode_row_bytes(table, payload, &self.encryption)
.map_err(|m| ("sync.invalid_request".to_owned(), m))?
};
let version = change.row_version.unwrap_or(0);
let table_name = table.name.clone();
self.write_base_row(&table_name, &row, version)
.map_err(|m| ("sync.invalid_request".to_owned(), m))?;
if mirror_visible {
self.write_row(&visible_table(&table_name), &table_name, &row, version)
.map_err(|m| ("sync.invalid_request".to_owned(), m))?;
}
}
Op::Delete => {
self.delete_base_row(table_name, &change.row_id)
.map_err(|m| ("sync.invalid_request".to_owned(), m))?;
if mirror_visible {
let sql = format!(
"DELETE FROM {} WHERE {}",
visible_table(table_name),
row_id_predicate(table)
);
self.conn
.prepare_cached(&sql)
.and_then(|mut statement| {
statement.execute(rusqlite::params![change.row_id])
})
.map_err(|error| {
("sync.invalid_request".to_owned(), error.to_string())
})?;
}
}
}
}
if mirror_visible {
if !self.outbox.is_empty() {
let changed_rows: HashSet<(&str, &str)> = changes
.iter()
.map(|change| {
(
tables[change.table_index as usize].as_str(),
change.row_id.as_str(),
)
})
.collect();
self.apply_outbox_ops(
self.outbox
.iter()
.flat_map(|commit| &commit.ops)
.filter(|op| {
changed_rows.contains(&(op.table.as_str(), op.row_id.as_str()))
}),
);
}
self.overlay_dirty.set(false);
}
Ok(())
}
fn apply_segment(
&mut self,
sub_index: usize,
segment: &RowsSegment,
first_fresh_page: bool,
) -> Result<u32, SectionError> {
let (sub_table, effective) = {
let sub = &self.subs[sub_index];
(sub.table.clone(), sub.effective.clone().unwrap_or_default())
};
let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
SectionError::Abort(
"sync.schema_mismatch".to_owned(),
format!("subscription table {sub_table:?} is not in the client schema"),
)
})?;
let matches = segment.table == table.name
&& segment.schema_version == self.schema.version
&& segment.columns.len() == table.wire_columns.len()
&& segment
.columns
.iter()
.zip(table.wire_columns.iter())
.all(|(a, b)| a.name == b.name && a.ty == b.ty && a.nullable == b.nullable);
if !matches {
return Err(SectionError::Abort(
"sync.schema_mismatch".to_owned(),
"segment column table does not match the generated schema (§5.2)".to_owned(),
));
}
let mut applied = 0u32;
let empty = Vec::new();
let blocks = if segment.blocks.is_empty() {
std::slice::from_ref(&empty)
} else {
&segment.blocks
};
for (index, block) in blocks.iter().enumerate() {
let clear = first_fresh_page && index == 0;
if !clear && block.is_empty() {
continue;
}
let was_dirty = self.overlay_dirty.get();
let reconcile_pending = table.indexes.iter().any(|index| index.unique)
&& self
.outbox
.iter()
.flat_map(|commit| &commit.ops)
.any(|op| op.table == table.name);
let mirror_visible = !reconcile_pending;
self.begin_observation("syncular_segment_block")
.map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
let outcome = (|| {
self.rebuild_overlay_if_dirty();
let mut changed_keys = Vec::new();
let mut batch = ChangeAccumulator::default();
if !block.is_empty() || (clear && self.scoped_rows_exist(&table.name, &effective)) {
batch.table(&table.name);
}
if clear {
self.purge_scope_rows(&table.name, &effective)
.map_err(|()| SectionError::FailClosed)?;
}
for (row_index, row) in block.iter().enumerate() {
let decrypted;
let values = if table.has_encrypted_columns() {
let mut values = row.values.clone();
crate::values::decrypt_segment_row(&table, &mut values, &self.encryption)
.map_err(|message| {
SectionError::Abort("client.decrypt_failed".into(), message)
})?;
decrypted = values;
&decrypted
} else {
&row.values
};
self.write_base_row(&table.name, values, row.server_version)
.map_err(|message| {
SectionError::Abort("sync.invalid_request".into(), message)
})?;
if reconcile_pending {
changed_keys.push(
owned_sql_value(RowParam::Cell(&values[table.pk_index])).map_err(
|message| SectionError::Abort("storage.failed".into(), message),
)?,
);
}
if mirror_visible {
self.write_row(
&visible_table(&table.name),
&table.name,
values,
row.server_version,
)
.map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
}
let processed = u64::from(applied) + row_index as u64 + 1;
if processed.is_multiple_of(1024) {
self.progress.update(|p| p.rows_processed = processed);
}
}
if reconcile_pending {
self.reconcile_imported_rows(&table, &mut changed_keys)
.map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
}
self.apply_outbox_ops(
self.outbox
.iter()
.flat_map(|commit| &commit.ops)
.filter(|op| op.table == table.name),
);
self.overlay_dirty.set(false);
self.finish_observation("syncular_segment_block", batch)
.map_err(|message| SectionError::Abort("storage.failed".into(), message))
})();
if let Err(error) = outcome {
self.rollback_observation("syncular_segment_block");
self.overlay_dirty.set(was_dirty);
return Err(error);
}
applied += block.len() as u32;
self.progress
.update(|p| p.rows_processed = u64::from(applied));
std::thread::yield_now();
}
self.progress
.update(|p| p.rows_processed = u64::from(applied));
Ok(applied)
}
fn apply_sqlite_segment(
&mut self,
sub_index: usize,
bytes: &[u8],
first_fresh_page: bool,
row_count: i64,
as_of_commit_seq: i64,
scope_digest: &str,
) -> Result<u32, SectionError> {
let invalid = |detail: &str| {
SectionError::Abort(
"sync.invalid_request".to_owned(),
format!("sqlite segment rejected: {detail} (§5.3)"),
)
};
let (sub_table, effective) = {
let sub = &self.subs[sub_index];
(sub.table.clone(), sub.effective.clone().unwrap_or_default())
};
let table = self.schema.table(&sub_table).cloned().ok_or_else(|| {
SectionError::Abort(
"sync.schema_mismatch".to_owned(),
format!("subscription table {sub_table:?} is not in the client schema"),
)
})?;
let path = std::env::temp_dir().join(format!("syncular-image-{}.db", uuid::Uuid::new_v4()));
std::fs::write(&path, bytes).map_err(|_| invalid("image temp file write failed"))?;
let img = match rusqlite::Connection::open_with_flags(
&path,
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY,
) {
Ok(conn) => conn,
Err(_) => {
let _ = std::fs::remove_file(&path);
return Err(invalid("bytes do not open as a SQLite database"));
}
};
let outcome = self.apply_sqlite_image(
&img,
&table,
first_fresh_page,
&effective,
row_count,
as_of_commit_seq,
scope_digest,
);
drop(img);
let _ = std::fs::remove_file(&path);
outcome
}
#[allow(clippy::too_many_arguments)]
fn apply_sqlite_image(
&mut self,
img: &rusqlite::Connection,
table: &crate::schema::TableSchema,
first_fresh_page: bool,
effective: &[(String, Vec<String>)],
row_count: i64,
as_of_commit_seq: i64,
scope_digest: &str,
) -> Result<u32, SectionError> {
let invalid = |detail: String| {
SectionError::Abort(
"sync.invalid_request".to_owned(),
format!("sqlite segment rejected: {detail} (§5.3)"),
)
};
type MetaRow = (i64, String, i64, i64, String, i64, i64);
let meta: MetaRow = img
.query_row(
"SELECT format, \"table\", \"schemaVersion\", \"asOfCommitSeq\",
\"scopeDigest\", \"rowCount\",
(SELECT count(*) FROM _syncular_segment)
FROM _syncular_segment",
[],
|row| {
Ok((
row.get(0)?,
row.get(1)?,
row.get(2)?,
row.get(3)?,
row.get(4)?,
row.get(5)?,
row.get(6)?,
))
},
)
.map_err(|_| invalid("missing or unreadable _syncular_segment metadata".to_owned()))?;
let (format, meta_table, schema_version, pin, digest, meta_rows, meta_count) = meta;
if meta_count != 1 {
return Err(invalid(format!(
"_syncular_segment must contain exactly one row, found {meta_count}"
)));
}
if format != 1 {
return Err(invalid(format!("format {format}")));
}
if meta_table != table.name {
return Err(invalid(format!("image table {meta_table:?}")));
}
if schema_version != i64::from(self.schema.version) {
return Err(invalid(format!("schemaVersion {schema_version}")));
}
if pin != as_of_commit_seq {
return Err(invalid(format!("asOfCommitSeq {pin}")));
}
if digest != scope_digest {
return Err(invalid("scopeDigest mismatch".to_owned()));
}
if meta_rows != row_count {
return Err(invalid(format!("rowCount {meta_rows}")));
}
let mut names: Vec<String> = Vec::new();
let mut primary_keys: Vec<String> = Vec::new();
{
let mut stmt = img
.prepare(&format!("PRAGMA table_info({})", quote_ident(&table.name)))
.map_err(|_| invalid("image data table missing".to_owned()))?;
let mut rows = stmt
.query([])
.map_err(|_| invalid("image data table unreadable".to_owned()))?;
while let Some(row) = rows
.next()
.map_err(|_| invalid("image data table unreadable".to_owned()))?
{
if row.get::<_, i64>(5).map_err(|e| invalid(e.to_string()))? > 0 {
primary_keys.push(row.get(1).map_err(|e| invalid(e.to_string()))?);
}
names.push(
row.get::<_, String>(1)
.map_err(|_| invalid("image data table unreadable".to_owned()))?,
);
}
}
let mut expected: Vec<&str> = table.columns.iter().map(|c| c.name.as_str()).collect();
expected.push("_syncular_version");
if names.len() != expected.len() || names.iter().zip(expected.iter()).any(|(a, b)| a != b) {
return Err(SectionError::Abort(
"sync.schema_mismatch".to_owned(),
"sqlite segment columns do not match the generated schema (§5.3)".to_owned(),
));
}
if primary_keys != vec![table.primary_key.clone()] {
return Err(invalid(
"image primary key does not match generated schema".into(),
));
}
let actual_count: i64 = img
.query_row(
&format!("SELECT count(*) FROM {}", quote_ident(&table.name)),
[],
|row| row.get(0),
)
.map_err(|e| invalid(e.to_string()))?;
if actual_count != row_count {
return Err(invalid("image row count does not match descriptor".into()));
}
let insert = self.insert_row_sql(&base_table(&table.name), table);
let visible_insert = self.insert_row_sql(&visible_table(&table.name), table);
let column_list = names
.iter()
.map(|n| quote_ident(n))
.collect::<Vec<_>>()
.join(", ");
let mut stmt = img
.prepare(&format!(
"SELECT {column_list} FROM {} ORDER BY {}",
quote_ident(&table.name),
quote_ident(&table.primary_key)
))
.map_err(|e| invalid(e.to_string()))?;
let mut rows = stmt.query([]).map_err(|e| invalid(e.to_string()))?;
let mut applied = 0u32;
loop {
let clear = applied == 0 && first_fresh_page;
let was_dirty = self.overlay_dirty.get();
let reconcile_pending = table.indexes.iter().any(|index| index.unique)
&& self
.outbox
.iter()
.flat_map(|commit| &commit.ops)
.any(|op| op.table == table.name);
let mirror_visible = !reconcile_pending;
self.begin_observation("syncular_image_chunk")
.map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
let result = (|| {
self.rebuild_overlay_if_dirty();
let mut changed_keys = Vec::new();
let mut batch = ChangeAccumulator::default();
if clear {
if self.scoped_rows_exist(&table.name, effective) {
batch.table(&table.name);
}
self.purge_scope_rows(&table.name, effective)
.map_err(|()| SectionError::FailClosed)?;
}
let mut processed = 0u32;
{
let mut ins = self
.conn
.prepare_cached(&insert)
.map_err(|e| invalid(e.to_string()))?;
let mut mirror = if mirror_visible {
Some(
self.conn
.prepare_cached(&visible_insert)
.map_err(|e| invalid(e.to_string()))?,
)
} else {
None
};
for _ in 0..1024 {
let Some(row) = rows.next().map_err(|e| invalid(e.to_string()))? else {
break;
};
for (i, column) in table.columns.iter().enumerate() {
let cell = row.get_ref(i).map_err(|e| invalid(e.to_string()))?;
let param = image_cell_param(column, cell).map_err(invalid)?;
ins.raw_bind_parameter(i + 1, ¶m)
.map_err(|e| invalid(e.to_string()))?;
if let Some(mirror) = &mut mirror {
mirror
.raw_bind_parameter(i + 1, ¶m)
.map_err(|e| invalid(e.to_string()))?;
}
}
let version_index = table.columns.len();
let version: i64 =
row.get(version_index).map_err(|e| invalid(e.to_string()))?;
if version < 1 {
return Err(invalid("image row version must be positive".into()));
}
ins.raw_bind_parameter(version_index + 1, version)
.map_err(|e| invalid(e.to_string()))?;
ins.raw_execute().map_err(|e| invalid(e.to_string()))?;
if let Some(mirror) = &mut mirror {
mirror
.raw_bind_parameter(version_index + 1, version)
.map_err(|e| invalid(e.to_string()))?;
mirror.raw_execute().map_err(|e| invalid(e.to_string()))?;
}
if reconcile_pending {
changed_keys.push(
row.get::<_, SqlValue>(table.pk_index)
.map_err(|error| invalid(error.to_string()))?,
);
}
processed += 1;
}
}
if processed > 0 {
batch.table(&table.name);
self.overlay_dirty.set(true);
}
if reconcile_pending {
self.reconcile_imported_rows(table, &mut changed_keys)
.map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
}
self.apply_outbox_ops(
self.outbox
.iter()
.flat_map(|commit| &commit.ops)
.filter(|op| op.table == table.name),
);
self.overlay_dirty.set(false);
self.finish_observation("syncular_image_chunk", batch)
.map_err(|message| SectionError::Abort("storage.failed".into(), message))?;
Ok(processed)
})();
let processed = match result {
Ok(processed) => processed,
Err(error) => {
self.rollback_observation("syncular_image_chunk");
self.overlay_dirty.set(was_dirty);
return Err(error);
}
};
applied += processed;
self.progress
.update(|p| p.rows_processed = u64::from(applied));
std::thread::yield_now();
if i64::from(applied) == actual_count {
break;
}
}
Ok(applied)
}
fn reconcile_imported_rows(
&self,
table: &TableSchema,
keys: &mut Vec<SqlValue>,
) -> Result<(), String> {
let visible = visible_table(&table.name);
let base = base_table(&table.name);
let pk = quote_ident(&table.primary_key);
let predicate = row_id_predicate(table);
let mut lookup = self.conn.prepare_cached(&format!(
"SELECT {pk} FROM {base} WHERE {predicate} UNION SELECT {pk} FROM {visible} WHERE {predicate}"
)).map_err(|error| error.to_string())?;
let mut deleted = HashSet::new();
for op in self
.outbox
.iter()
.flat_map(|commit| &commit.ops)
.filter(|op| op.table == table.name)
{
if op.upsert {
if let Some(values) = &op.values {
let key = json_to_column_value(
&table.columns[table.pk_index],
values.get(&table.primary_key),
)?;
keys.push(owned_sql_value(RowParam::Cell(&key))?);
}
} else if deleted.insert(&op.row_id) {
keys.extend(
lookup
.query_map([&op.row_id], |row| row.get::<_, SqlValue>(0))
.map_err(|error| error.to_string())?
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?,
);
}
}
for chunk in keys.chunks(400) {
let holes = vec!["?"; chunk.len()].join(",");
self.conn
.prepare_cached(&format!("DELETE FROM {visible} WHERE {pk} IN ({holes})"))
.and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
.map_err(|error| error.to_string())?;
}
let updates = table
.columns
.iter()
.map(|column| column.name.as_str())
.chain(std::iter::once("_syncular_version"))
.filter(|name| *name != table.primary_key)
.map(|name| {
let name = quote_ident(name);
format!("{name}=excluded.{name}")
})
.collect::<Vec<_>>()
.join(",");
for chunk in keys.chunks(400) {
let holes = vec!["?"; chunk.len()].join(",");
self.conn.prepare_cached(&format!("INSERT INTO {visible} SELECT * FROM {base} WHERE {pk} IN ({holes}) ON CONFLICT ({pk}) DO UPDATE SET {updates}"))
.and_then(|mut statement| statement.execute(rusqlite::params_from_iter(chunk)))
.map_err(|error| error.to_string())?;
}
Ok(())
}
fn compile_local_data_purge(
&self,
input: &LocalDataPurgeInput,
) -> Result<(Vec<CompiledLocalDataPurgeTarget>, String), String> {
let invalid = |message: String| format!("sync.invalid_request: {message}");
if input.purge_id.is_empty()
|| input.purge_id.len() > 128
|| !is_local_operation_code_like(&input.purge_id)
{
return Err(invalid(
"local purge purgeId must be a 1–128 character code-like identifier".to_owned(),
));
}
if input.targets.is_empty() || input.targets.len() > MAX_LOCAL_PURGE_TARGETS {
return Err(invalid(format!(
"local purge needs between 1 and {MAX_LOCAL_PURGE_TARGETS} targets"
)));
}
let mut deduplicated: BTreeMap<
String,
(CompiledLocalDataPurgeTarget, LocalDataPurgeTarget),
> = BTreeMap::new();
for target in &input.targets {
let table = self.schema.table(&target.table).ok_or_else(|| {
invalid(format!(
"local purge names unknown table {:?}",
target.table
))
})?;
if target.selectors.is_empty() || target.selectors.len() > MAX_LOCAL_PURGE_SELECTORS {
return Err(invalid(format!(
"local purge target {:?} needs between 1 and {MAX_LOCAL_PURGE_SELECTORS} selectors",
target.table
)));
}
let mut selectors = Vec::with_capacity(target.selectors.len());
let mut canonical_selectors = BTreeMap::new();
for (column_name, raw_values) in &target.selectors {
let Some((column_index, column)) = table
.columns
.iter()
.enumerate()
.find(|(_, column)| column.name == *column_name)
else {
return Err(invalid(format!(
"local purge target {:?} names unknown column {:?}",
target.table, column_name
)));
};
let encrypted = table
.encrypted_columns
.iter()
.any(|candidate| candidate.index == column_index);
if column.ty != ColumnType::String || encrypted {
return Err(invalid(format!(
"local purge selector {:?}.{:?} must be a plaintext string column",
target.table, column_name
)));
}
if raw_values.is_empty() || raw_values.len() > MAX_LOCAL_PURGE_VALUES {
return Err(invalid(format!(
"local purge selector {:?}.{:?} needs between 1 and {MAX_LOCAL_PURGE_VALUES} values",
target.table, column_name
)));
}
let mut values = raw_values.clone();
values.sort();
values.dedup();
if values.iter().any(|value| {
value.is_empty()
|| value.len() > MAX_LOCAL_PURGE_VALUE_LENGTH
|| !is_local_operation_code_like(value)
}) {
return Err(invalid(format!(
"local purge selector values must be 1–{MAX_LOCAL_PURGE_VALUE_LENGTH} character code-like identifiers"
)));
}
selectors.push((column_name.clone(), values.clone()));
canonical_selectors.insert(column_name.clone(), values);
}
let canonical = LocalDataPurgeTarget {
table: target.table.clone(),
selectors: canonical_selectors,
};
let key = serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
deduplicated.insert(
key,
(
CompiledLocalDataPurgeTarget {
table: target.table.clone(),
selectors,
},
canonical,
),
);
}
let targets = deduplicated
.values()
.map(|(compiled, _)| compiled.clone())
.collect::<Vec<_>>();
let canonical = deduplicated
.values()
.map(|(_, target)| target.clone())
.collect::<Vec<_>>();
let canonical_plan =
serde_json::to_string(&canonical).map_err(|error| error.to_string())?;
Ok((targets, canonical_plan))
}
fn local_purge_base_row_ids(
&self,
targets: &[CompiledLocalDataPurgeTarget],
) -> Result<BTreeMap<String, BTreeSet<String>>, String> {
let mut by_table: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
for target in targets {
let table = self
.schema
.table(&target.table)
.ok_or_else(|| format!("sync.invalid_request: unknown table {:?}", target.table))?;
let mut clauses = Vec::with_capacity(target.selectors.len());
let mut params = Vec::new();
for (column, values) in &target.selectors {
clauses.push(format!(
"{} IN ({})",
quote_ident(column),
values.iter().map(|_| "?").collect::<Vec<_>>().join(", ")
));
params.extend(values.iter().cloned().map(SqlValue::Text));
}
let sql = format!(
"SELECT CAST({} AS TEXT) FROM {} WHERE {}",
quote_ident(&table.primary_key),
base_table(&target.table),
clauses.join(" AND ")
);
let mut statement = self.conn.prepare(&sql).map_err(|error| error.to_string())?;
let rows = statement
.query_map(rusqlite::params_from_iter(params), |row| {
row.get::<_, String>(0)
})
.map_err(|error| error.to_string())?;
let ids = by_table.entry(target.table.clone()).or_default();
for row in rows {
ids.insert(row.map_err(|error| error.to_string())?);
}
}
Ok(by_table)
}
fn local_purge_values_match(
target: &CompiledLocalDataPurgeTarget,
values: &Map<String, Value>,
) -> bool {
target.selectors.iter().all(|(column, allowed)| {
values
.get(column)
.and_then(Value::as_str)
.is_some_and(|value| allowed.iter().any(|candidate| candidate == value))
})
}
pub fn purge_local_data(
&mut self,
input: &LocalDataPurgeInput,
) -> Result<LocalDataPurgeResult, String> {
let (targets, canonical_plan) = self.compile_local_data_purge(input)?;
let meta_key = format!("localPurge:{}", input.purge_id);
let applied_plan = self
.conn
.query_row(
"SELECT value FROM _syncular_meta WHERE key = ?1",
rusqlite::params![meta_key],
|row| row.get::<_, String>(0),
)
.optional()
.map_err(|error| error.to_string())?;
if let Some(applied_plan) = applied_plan {
if applied_plan != canonical_plan {
return Err(format!(
"sync.invalid_request: local purge id {:?} was already used with a different plan",
input.purge_id
));
}
return Ok(LocalDataPurgeResult {
already_applied: true,
purged_rows: 0,
dropped_commits: 0,
});
}
let _ = self.drop_previous_version();
let prior_outbox = self.outbox.clone();
let prior_rejection_count = self.rejections.len();
let prior_overlay_dirty = self.overlay_dirty.get();
self.begin_observation("syncular_local_purge")?;
let applied = (|| -> Result<(ChangeAccumulator, LocalDataPurgeResult), String> {
let row_ids = self.local_purge_base_row_ids(&targets)?;
let doomed = self
.outbox
.iter()
.filter(|commit| {
commit.ops.iter().any(|operation| {
let matching_targets = targets
.iter()
.filter(|target| target.table == operation.table)
.collect::<Vec<_>>();
if matching_targets.is_empty() {
return false;
}
if row_ids
.get(&operation.table)
.is_some_and(|ids| ids.contains(&operation.row_id))
{
return true;
}
operation.values.as_ref().is_some_and(|values| {
matching_targets
.iter()
.any(|target| Self::local_purge_values_match(target, values))
})
})
})
.cloned()
.collect::<Vec<_>>();
let doomed_ids = doomed
.iter()
.map(|commit| commit.client_commit_id.clone())
.collect::<BTreeSet<_>>();
let mut batch = ChangeAccumulator::default();
let mut rejections = Vec::new();
for commit in &doomed {
for operation in &commit.ops {
batch.table(&operation.table);
}
let results = commit
.ops
.iter()
.enumerate()
.map(|(op_index, operation)| {
let rejection = RejectionRecord {
client_commit_id: commit.client_commit_id.clone(),
op_index: op_index as i32,
code: "client.local_data_purged".to_owned(),
message: "the commit was dropped by an application-authorized local data purge".to_owned(),
retryable: false,
details: None,
operation: Some(CommitOperation::from(operation)),
};
rejections.push(rejection.clone());
CommitOperationOutcome::Error { rejection }
})
.collect::<Vec<_>>();
self.persist_commit_outcome(
&commit.client_commit_id,
CommitOutcomeStatus::Rejected,
&results,
Some(&commit.ops),
)?;
self.delete_outbox_persisted(&commit.client_commit_id)?;
}
if !doomed.is_empty() {
self.outbox
.retain(|commit| !doomed_ids.contains(&commit.client_commit_id));
self.rejections.extend(rejections);
self.prune_commit_outcomes()?;
self.overlay_dirty.set(true);
batch.status = true;
batch.rejections = true;
batch.outcomes = true;
}
let mut purged_rows = 0usize;
for (table, ids) in &row_ids {
if ids.is_empty() {
continue;
}
batch.table(table);
for row_id in ids {
self.delete_base_row(table, row_id)?;
purged_rows += 1;
}
}
self.rebuild_overlay_if_dirty();
self.delete_unreferenced_cached_blobs();
self.conn
.execute(
"INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
rusqlite::params![meta_key, canonical_plan],
)
.map_err(|error| error.to_string())?;
Ok((
batch,
LocalDataPurgeResult {
already_applied: false,
purged_rows,
dropped_commits: doomed.len(),
},
))
})();
let (batch, result) = match applied {
Ok(value) => value,
Err(error) => {
self.rollback_observation("syncular_local_purge");
self.outbox = prior_outbox;
self.rejections.truncate(prior_rejection_count);
self.overlay_dirty.set(prior_overlay_dirty);
return Err(error);
}
};
if let Err(error) = self.finish_observation("syncular_local_purge", batch) {
self.rollback_observation("syncular_local_purge");
self.outbox = prior_outbox;
self.rejections.truncate(prior_rejection_count);
self.overlay_dirty.set(prior_overlay_dirty);
return Err(error);
}
Ok(result)
}
pub fn rebootstrap_local_data(
&mut self,
input: &LocalDataRebootstrapInput,
) -> Result<LocalDataRebootstrapResult, String> {
if input.rebootstrap_id.is_empty()
|| input.rebootstrap_id.len() > 128
|| !is_local_operation_code_like(&input.rebootstrap_id)
{
return Err(
"sync.invalid_request: local rebootstrap rebootstrapId must be a 1–128 character code-like identifier"
.to_owned(),
);
}
let meta_key = format!("localRebootstrap:{}", input.rebootstrap_id);
if let Some(persisted) = self.get_meta_strict(&meta_key)? {
let (retained_commits, reset_subscriptions) =
decode_local_rebootstrap_receipt(&persisted)?;
return Ok(LocalDataRebootstrapResult {
already_applied: true,
retained_commits,
reset_subscriptions,
});
}
if self.stopped || self.schema_floor.is_some() {
return Err(
"sync.invalid_request: local rebootstrap cannot bypass an active schema-floor stop; update the application first"
.to_owned(),
);
}
let retained_commits = self.outbox.len();
let reset_subscriptions = self.subs.len();
let prior_subs = self.subs.clone();
let prior_upgrading = self.upgrading;
let prior_stopped = self.stopped;
let prior_schema_floor = self.schema_floor.clone();
let prior_overlay_dirty = self.overlay_dirty.get();
let prior_sync_needed = self.sync_needed;
let prior_sync_intents = self.sync_intent_queue.clone();
let receipt = encode_local_rebootstrap_receipt(retained_commits, reset_subscriptions)?;
self.begin_observation("syncular_local_rebootstrap")?;
let mut batch = ChangeAccumulator::default();
let applied = (|| -> Result<(), String> {
self.run_schema_reset_observed(&mut batch, false, false)?;
self.conn
.execute(
"INSERT INTO _syncular_meta(key, value) VALUES (?1, ?2)",
rusqlite::params![meta_key, receipt],
)
.map_err(|error| error.to_string())?;
self.sync_needed = true;
self.sync_intent_queue.push_back(SyncIntent::Interactive);
batch.status = true;
Ok(())
})();
if let Err(error) = applied {
self.rollback_observation("syncular_local_rebootstrap");
self.subs = prior_subs;
self.upgrading = prior_upgrading;
self.stopped = prior_stopped;
self.schema_floor = prior_schema_floor;
self.overlay_dirty.set(prior_overlay_dirty);
self.sync_needed = prior_sync_needed;
self.sync_intent_queue = prior_sync_intents;
return Err(error);
}
if let Err(error) = self.finish_observation("syncular_local_rebootstrap", batch) {
self.rollback_observation("syncular_local_rebootstrap");
self.subs = prior_subs;
self.upgrading = prior_upgrading;
self.stopped = prior_stopped;
self.schema_floor = prior_schema_floor;
self.overlay_dirty.set(prior_overlay_dirty);
self.sync_needed = prior_sync_needed;
self.sync_intent_queue = prior_sync_intents;
return Err(error);
}
Ok(LocalDataRebootstrapResult {
already_applied: false,
retained_commits,
reset_subscriptions,
})
}
fn purge_scope_rows(
&mut self,
table_name: &str,
effective: &[(String, Vec<String>)],
) -> Result<(), ()> {
if effective.is_empty() {
return Ok(());
}
let table = self.schema.table(table_name).ok_or(())?.clone();
let mut clauses = Vec::new();
let mut params: Vec<SqlValue> = Vec::new();
for (variable, values) in effective {
let column = table.scope_column(variable).ok_or(())?;
let placeholders: Vec<String> = values
.iter()
.map(|v| {
params.push(SqlValue::Text(v.clone()));
"?".to_owned()
})
.collect();
clauses.push(format!(
"{} IN ({})",
quote_ident(column),
placeholders.join(", ")
));
}
let sql = format!(
"DELETE FROM {} WHERE {}",
base_table(table_name),
clauses.join(" AND ")
);
self.overlay_dirty.set(true);
self.conn
.execute(&sql, rusqlite::params_from_iter(¶ms))
.map_err(|_| ())?;
self.conn
.execute(
&format!(
"DELETE FROM {} WHERE {}",
visible_table(table_name),
clauses.join(" AND ")
),
rusqlite::params_from_iter(¶ms),
)
.map_err(|_| ())?;
Ok(())
}
fn drop_doomed_outbox(
&mut self,
table_name: &str,
effective: &[(String, Vec<String>)],
) -> Result<bool, String> {
if effective.is_empty() {
return Ok(false);
}
let Some(table) = self.schema.table(table_name).cloned() else {
return Ok(false);
};
let mut mappings: Vec<(&str, &Vec<String>)> = Vec::new();
for (variable, values) in effective {
match table.scope_column(variable) {
Some(column) => mappings.push((column, values)),
None => return Ok(false), }
}
let doomed: Vec<OutboxCommit> = self
.outbox
.iter()
.filter(|commit| {
commit.ops.iter().any(|op| {
op.upsert
&& op.table == table_name
&& op.values.as_ref().is_some_and(|values| {
mappings
.iter()
.all(|(column, allowed)| match values.get(*column) {
Some(Value::String(s)) => allowed.contains(s),
Some(Value::Number(n)) => allowed.contains(&n.to_string()),
_ => false,
})
})
})
})
.cloned()
.collect();
if doomed.is_empty() {
return Ok(false);
}
let mut rejections = Vec::new();
for commit in &doomed {
let results = commit
.ops
.iter()
.enumerate()
.map(|(op_index, operation)| {
let rejection = RejectionRecord {
client_commit_id: commit.client_commit_id.clone(),
op_index: op_index as i32,
code: "sync.scope_revoked".to_owned(),
message: "the commit was dropped because its effective scope was revoked"
.to_owned(),
retryable: false,
details: None,
operation: Some(CommitOperation::from(operation)),
};
rejections.push(rejection.clone());
CommitOperationOutcome::Error { rejection }
})
.collect::<Vec<_>>();
self.persist_commit_outcome(
&commit.client_commit_id,
CommitOutcomeStatus::Rejected,
&results,
Some(&commit.ops),
)?;
self.delete_outbox_persisted(&commit.client_commit_id)?;
}
self.prune_commit_outcomes()?;
let doomed_ids = doomed
.iter()
.map(|commit| commit.client_commit_id.as_str())
.collect::<BTreeSet<_>>();
self.outbox
.retain(|commit| !doomed_ids.contains(commit.client_commit_id.as_str()));
self.rejections.extend(rejections);
self.overlay_dirty.set(true);
Ok(true)
}
pub fn upload_blob(
&mut self,
bytes: &[u8],
media_type: Option<String>,
name: Option<String>,
) -> Result<Value, String> {
let blob_id = blob_id_for(bytes);
let now = self.clock_now_ms();
let tx = self.conn.transaction().map_err(|e| e.to_string())?;
tx.execute(
"INSERT INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,?,?)
ON CONFLICT(blob_id) DO NOTHING",
rusqlite::params![blob_id, bytes, bytes.len() as i64, media_type, now],
)
.map_err(|e| e.to_string())?;
tx.execute(
"INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms) VALUES (?,?,?)",
rusqlite::params![blob_id, media_type, now],
)
.map_err(|e| e.to_string())?;
tx.commit().map_err(|e| e.to_string())?;
self.enforce_blob_cache_cap();
let mut obj = Map::new();
obj.insert("blobId".to_owned(), Value::from(blob_id));
obj.insert("byteLength".to_owned(), Value::from(bytes.len() as i64));
if let Some(mt) = media_type {
obj.insert("mediaType".to_owned(), Value::from(mt));
}
if let Some(n) = name {
obj.insert("name".to_owned(), Value::from(n));
}
Ok(Value::Object(obj))
}
pub fn fetch_blob_bytes(
&mut self,
transport: &mut dyn Transport,
blob_id_or_ref: &str,
) -> Result<FetchedBlob, (String, String)> {
let simple = |m: String| ("client.failed".to_owned(), m);
let blob_id = if blob_id_or_ref.starts_with("sha256:") {
blob_id_or_ref.to_owned()
} else {
let value: Value = serde_json::from_str(blob_id_or_ref)
.map_err(|_| simple("blob ref is not JSON".to_owned()))?;
value
.get("blobId")
.and_then(Value::as_str)
.ok_or_else(|| simple("blob ref has no blobId".to_owned()))?
.to_owned()
};
if let Some(cached) = self.get_cached_blob_bytes(&blob_id).map_err(simple)? {
return Ok(cached);
}
#[cfg(feature = "bench-internals")]
let download_phase = self.benchmark_phases.start(Phase::BlobDownload);
let bytes = match transport
.blob_download(&blob_id)
.map_err(|e| (e.code, e.message))?
{
BlobDownload::Bytes(bytes) => bytes,
BlobDownload::Url {
url,
url_expires_at_ms,
} => {
if url_expires_at_ms.is_some_and(|exp| exp <= self.clock_now_ms()) {
return Err((
"sync.segment_expired".to_owned(),
format!(
"blob url for {blob_id} expired before fetch — re-request mints a fresh url (§5.9.5)"
),
));
}
transport
.fetch_blob_url(&url)
.map_err(|e| (e.code, e.message))?
}
};
#[cfg(feature = "bench-internals")]
drop(download_phase);
#[cfg(feature = "bench-internals")]
let validate_phase = self.benchmark_phases.start(Phase::BlobValidate);
if blob_id_for(&bytes) != blob_id {
return Err(simple(format!(
"blob content address mismatch for {blob_id}"
)));
}
#[cfg(feature = "bench-internals")]
drop(validate_phase);
#[cfg(feature = "bench-internals")]
let insert_phase = self.benchmark_phases.start(Phase::BlobCacheInsert);
let now = self.clock_now_ms();
self.conn
.execute(
"INSERT OR IGNORE INTO _syncular_blobs(blob_id, bytes, byte_length, media_type, created_at_ms) VALUES (?,?,?,NULL,?)",
rusqlite::params![blob_id, bytes, bytes.len() as i64, now],
)
.map_err(|e| simple(e.to_string()))?;
self.conn
.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))
.map_err(|e| simple(e.to_string()))?;
self.enforce_blob_cache_cap();
#[cfg(feature = "bench-internals")]
drop(insert_phase);
self.get_cached_blob_bytes(&blob_id)
.map_err(simple)?
.ok_or_else(|| simple("blob cache write failed".to_owned()))
}
fn get_cached_blob_bytes(&self, blob_id: &str) -> Result<Option<FetchedBlob>, String> {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::BlobCacheRead);
let mut stmt = self
.conn
.prepare("SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?")
.map_err(|e| e.to_string())?;
let mut rows = stmt
.query(rusqlite::params![blob_id])
.map_err(|e| e.to_string())?;
if let Some(row) = rows.next().map_err(|e| e.to_string())? {
let bytes: Vec<u8> = row.get(0).map_err(|e| e.to_string())?;
let byte_length: i64 = row.get(1).map_err(|e| e.to_string())?;
let media_type: Option<String> = row.get(2).map_err(|e| e.to_string())?;
return Ok(Some(FetchedBlob {
blob_id: blob_id.to_owned(),
byte_length,
bytes,
media_type,
}));
}
Ok(None)
}
fn flush_blob_uploads(&mut self, transport: &mut dyn Transport) -> Result<(), TransportError> {
let pending: Vec<(String, Option<String>)> = {
let mut stmt = self
.conn
.prepare(
"SELECT blob_id, media_type FROM (
SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
UNION ALL
SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
) ORDER BY created_at_ms, blob_id",
)
.map_err(|e| TransportError::new("client.failed", e.to_string()))?;
let rows = stmt
.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, Option<String>>(1)?))
})
.map_err(|e| TransportError::new("client.failed", e.to_string()))?;
rows.collect::<Result<_, _>>()
.map_err(|error| match error {
rusqlite::Error::InvalidColumnType(..)
| rusqlite::Error::FromSqlConversionFailure(..) => TransportError::new(
"sync.local_corrupt",
"Pending blob upload metadata is invalid",
),
_ => TransportError::new("client.failed", error.to_string()),
})?
};
for (blob_id, media_type) in pending {
let stored: Option<(SqlValue, SqlValue)> = self
.conn
.query_row(
"SELECT bytes, byte_length FROM _syncular_blobs WHERE blob_id = ?",
rusqlite::params![blob_id],
|row| Ok((row.get(0)?, row.get(1)?)),
)
.optional()
.map_err(|e| TransportError::new("client.failed", e.to_string()))?;
let Some((SqlValue::Blob(bytes), SqlValue::Integer(byte_length))) = stored else {
return Err(TransportError::new(
"sync.local_corrupt",
"Pending blob upload body is missing or corrupt",
));
};
if byte_length != bytes.len() as i64 || blob_id_for(&bytes) != blob_id {
return Err(TransportError::new(
"sync.local_corrupt",
"Pending blob upload body is missing or corrupt",
));
}
self.upload_one(transport, &blob_id, &bytes, media_type.as_deref())?;
self.conn
.execute(
"DELETE FROM _syncular_blob_uploads WHERE blob_id = ?",
rusqlite::params![blob_id],
)
.map_err(|e| TransportError::new("client.failed", e.to_string()))?;
}
Ok(())
}
fn upload_one(
&self,
transport: &mut dyn Transport,
blob_id: &str,
bytes: &[u8],
media_type: Option<&str>,
) -> Result<(), TransportError> {
match transport.blob_upload_grant(blob_id, bytes.len() as u64, media_type)? {
BlobUploadGrant::Present => return Ok(()), BlobUploadGrant::Url {
url,
url_expires_at_ms,
} => {
let live = url_expires_at_ms.is_none_or(|exp| exp > self.clock_now_ms());
if live && transport.blob_put_url(&url, bytes, media_type).is_ok() {
return Ok(());
}
}
BlobUploadGrant::None => {
}
}
transport.blob_upload(blob_id, bytes, media_type)
}
fn visible_blob_ids_sql(&self) -> String {
let mut selects = Vec::new();
for table in &self.schema.tables {
for column in table
.columns
.iter()
.filter(|column| column.ty == ColumnType::BlobRef)
{
let identifier = quote_ident(&column.name);
selects.push(format!(
"SELECT json_extract({identifier}, '$.blobId') AS blob_id FROM {} \
WHERE typeof({identifier}) = 'text' AND json_valid({identifier}) \
AND json_type({identifier}, '$.blobId') = 'text'",
quote_ident(&table.name),
));
}
}
if selects.is_empty() {
"SELECT NULL AS blob_id WHERE 0".to_owned()
} else {
selects.join(" UNION ")
}
}
fn enforce_blob_cache_cap(&self) {
let Some(max_bytes) = self.limits.blob_cache_max_bytes else {
return;
};
let mut total: i64 = self
.conn
.query_row(
"SELECT COALESCE(SUM(byte_length), 0) FROM _syncular_blobs",
[],
|row| row.get(0),
)
.unwrap_or(0);
if total <= max_bytes {
return;
}
let query = format!(
"SELECT blob_id, byte_length FROM _syncular_blobs
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
AND blob_id NOT IN ({})
ORDER BY created_at_ms ASC, blob_id ASC",
self.visible_blob_ids_sql(),
);
let candidates: Vec<(String, i64)> = {
let Ok(mut stmt) = self.conn.prepare(&query) else {
return;
};
let Ok(rows) = stmt.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?))
}) else {
return;
};
rows.filter_map(Result::ok).collect()
};
for (blob_id, byte_length) in candidates {
if total <= max_bytes {
break;
}
let _ = self.conn.execute(
"DELETE FROM _syncular_blobs WHERE blob_id = ?",
rusqlite::params![blob_id],
);
total -= byte_length;
}
}
fn delete_unreferenced_cached_blobs(&mut self) {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::BlobRetention);
if !self.schema_has_blobs() {
return;
}
self.rebuild_overlay_if_dirty();
let _ = self.conn.execute(
&format!(
"DELETE FROM _syncular_blobs
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
AND blob_id NOT IN ({})",
self.visible_blob_ids_sql(),
),
[],
);
}
fn insert_row_sql(&self, full_table: &str, table: &crate::schema::TableSchema) -> String {
if let Some(sql) = self.insert_sql.borrow().get(full_table) {
return sql.clone();
}
let mut columns: Vec<String> = table.columns.iter().map(|c| quote_ident(&c.name)).collect();
columns.push(quote_ident("_syncular_version"));
let placeholders: Vec<&str> = columns.iter().map(|_| "?").collect();
let primary_key = quote_ident(&table.primary_key);
let updates = columns
.iter()
.filter(|column| **column != primary_key)
.map(|column| format!("{column}=excluded.{column}"))
.collect::<Vec<_>>()
.join(", ");
let sql = format!(
"INSERT INTO {full_table} ({}) VALUES ({}) ON CONFLICT ({primary_key}) DO UPDATE SET {updates}",
columns.join(", "),
placeholders.join(", ")
);
self.insert_sql
.borrow_mut()
.insert(full_table.to_owned(), sql.clone());
sql
}
fn write_base_row(&self, table_name: &str, row: &Row, version: i64) -> Result<(), String> {
self.overlay_dirty.set(true);
self.write_row(&base_table(table_name), table_name, row, version)
}
fn write_row(
&self,
full_table: &str,
table_name: &str,
row: &Row,
version: i64,
) -> Result<(), String> {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::RowWrite);
let table = self
.schema
.table(table_name)
.ok_or_else(|| format!("unknown table {table_name:?}"))?;
let sql = self.insert_row_sql(full_table, table);
let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
let params = row
.iter()
.map(RowParam::Cell)
.chain(std::iter::once(RowParam::Version(version)));
stmt.execute(rusqlite::params_from_iter(params))
.map_err(|e| e.to_string())?;
Ok(())
}
fn delete_base_row(&self, table_name: &str, row_id: &str) -> Result<(), String> {
let table = self
.schema
.table(table_name)
.ok_or_else(|| format!("unknown table {table_name:?}"))?;
self.overlay_dirty.set(true);
let sql = format!(
"DELETE FROM {} WHERE {}",
base_table(table_name),
row_id_predicate(table)
);
let mut stmt = self.conn.prepare_cached(&sql).map_err(|e| e.to_string())?;
stmt.execute(rusqlite::params![row_id])
.map_err(|e| e.to_string())?;
Ok(())
}
fn rebuild_overlay_if_dirty(&mut self) {
if self.overlay_dirty.get() {
self.rebuild_overlay();
}
}
fn rebuild_overlay(&mut self) {
#[cfg(feature = "bench-internals")]
let _phase = self.benchmark_phases.start(Phase::OverlayRebuild);
#[cfg(test)]
self.overlay_rebuild_count
.set(self.overlay_rebuild_count.get() + 1);
self.exec("SAVEPOINT syncular_overlay");
for table in self.schema.tables.clone() {
for index in &table.fts_indexes {
let _ = self.drop_fts_triggers(index);
}
let visible = visible_table(&table.name);
let base = base_table(&table.name);
self.exec(&format!("DELETE FROM {visible}"));
self.exec(&format!("INSERT INTO {visible} SELECT * FROM {base}"));
}
for commit in &self.outbox {
self.apply_outbox_ops(&commit.ops);
}
for table in self.schema.tables.clone() {
for index in &table.fts_indexes {
let _ = self.rebuild_fts_projection(&table, index);
let _ = self.create_fts_triggers(&table, index);
}
}
self.exec("RELEASE syncular_overlay");
self.overlay_dirty.set(false);
}
fn apply_outbox_ops<'a>(&self, ops: impl IntoIterator<Item = &'a OutboxOp>) {
#[cfg(feature = "bench-internals")]
let mut phase = self.benchmark_phases.start(Phase::PendingReplay);
for op in ops {
#[cfg(feature = "bench-internals")]
if let Some(phase) = &mut phase {
phase.unit();
}
let _ = self.apply_outbox_op(op);
}
}
fn apply_outbox_op(&self, op: &OutboxOp) -> Result<(), String> {
let Some(table) = self.schema.table(&op.table) else {
return Ok(());
};
let visible = visible_table(&table.name);
if !op.upsert {
let sql = format!("DELETE FROM {visible} WHERE {}", row_id_predicate(table));
self.conn
.prepare_cached(&sql)
.and_then(|mut statement| statement.execute(rusqlite::params![op.row_id]))
.map_err(|e| e.to_string())?;
return Ok(());
}
let Some(values) = op.values.as_ref() else {
return Ok(());
};
let full = table
.columns
.iter()
.all(|column| values.contains_key(&column.name));
let local = self.visible_row(table, &op.row_id);
if local.is_none() && !full {
return Ok(());
}
let mut row: Row = Vec::with_capacity(table.columns.len());
for (index, column) in table.columns.iter().enumerate() {
let incoming = match values.get(&column.name) {
Some(raw) => json_to_column_value(column, Some(raw)),
None => Ok(local
.as_ref()
.and_then(|(values, _)| values.get(index).cloned())
.flatten()),
};
match incoming {
Ok(value) => row.push(value),
Err(_) => return Ok(()),
}
}
let version = local.as_ref().map_or(-1, |(_, version)| *version);
self.write_row(&visible, &table.name, &row, version)
.map_err(|error| {
if error.starts_with("UNIQUE constraint failed:") {
"sync.constraint_violation: local write violates a unique constraint".to_owned()
} else {
error
}
})
}
fn visible_row(&self, table: &TableSchema, row_id: &str) -> Option<(Row, i64)> {
let sql = format!(
"SELECT * FROM {} WHERE {}",
visible_table(&table.name),
row_id_predicate(table)
);
let mut stmt = self.conn.prepare_cached(&sql).ok()?;
let mut rows = stmt.query(rusqlite::params![row_id]).ok()?;
let row = rows.next().ok()??;
let mut values = Vec::with_capacity(table.columns.len());
for (index, column) in table.columns.iter().enumerate() {
let raw = row.get_ref(index).ok()?;
let json = sql_ref_to_json(column, raw);
values.push(json_to_column_value(column, Some(&json)).ok()?);
}
let version: i64 = row.get(table.columns.len()).ok()?;
Some((values, version))
}
fn exec(&self, sql: &str) {
let _ = self.conn.execute_batch(sql);
}
#[must_use]
pub fn realtime_policy(&self) -> RealtimePolicy {
self.realtime_policy
}
#[must_use]
pub fn realtime_state(&self) -> RealtimeState {
self.realtime_state
}
pub fn set_realtime_policy(&mut self, policy: RealtimePolicy) -> Result<(), String> {
if self.realtime_state == RealtimeState::Connected && policy != self.realtime_policy {
return Err(
"sync.invalid_request: disconnect realtime before changing the realtime policy"
.to_owned(),
);
}
self.realtime_policy = policy;
match policy {
RealtimePolicy::Off => self.set_realtime_state(RealtimeState::Disabled, None),
RealtimePolicy::Required | RealtimePolicy::Optional
if self.realtime_state == RealtimeState::Disabled =>
{
self.set_realtime_state(RealtimeState::Disconnected, None);
}
RealtimePolicy::Required | RealtimePolicy::Optional => {}
}
Ok(())
}
fn set_realtime_state(&mut self, state: RealtimeState, reason_code: Option<&str>) {
self.realtime_state = state;
self.realtime_reason_code = reason_code.map(Self::diagnostic_code);
if matches!(state, RealtimeState::Connected | RealtimeState::Disabled) {
self.realtime_reason_code = None;
self.realtime_retry_delay_ms = None;
}
}
#[must_use]
fn realtime_connected(&self) -> bool {
self.realtime_state == RealtimeState::Connected
}
pub fn connect_realtime(&mut self, transport: &mut dyn Transport) -> Result<(), String> {
if self.realtime_policy == RealtimePolicy::Off {
return Err(
"sync.invalid_request: realtime is off; realtime connect is refused".to_owned(),
);
}
if self.realtime_connected() {
return Ok(());
}
self.realtime_state = RealtimeState::Connecting;
match transport.realtime_connect_for_client(&self.client_id) {
Ok(()) => {
self.set_realtime_state(RealtimeState::Connected, None);
Ok(())
}
Err(error) => {
self.set_realtime_state(RealtimeState::Refused, Some(&error.code));
Err(format!("{}: {}", error.code, error.message))
}
}
}
pub fn disconnect_realtime(&mut self, transport: &mut dyn Transport) {
if self.realtime_state == RealtimeState::Disconnected
|| self.realtime_state == RealtimeState::Disabled
{
return;
}
let _ = transport.realtime_close();
self.set_realtime_state(
if self.realtime_policy == RealtimePolicy::Off {
RealtimeState::Disabled
} else {
RealtimeState::Disconnected
},
None,
);
self.presence.clear(); }
pub fn set_presence(
&mut self,
transport: &mut dyn Transport,
scope_key: &str,
doc: Option<&Value>,
) -> Result<(), String> {
if !self.realtime_connected() {
return Err("setPresence requires a connected realtime socket (§8.6)".to_string());
}
let text = encode_presence_publish(scope_key, doc);
transport
.realtime_send(&text)
.map_err(|e| format!("{}: {}", e.code, e.message))
}
pub fn presence(&self, scope_key: &str) -> Vec<PresencePeer> {
self.presence
.get(scope_key)
.map(|peers| peers.values().cloned().collect())
.unwrap_or_default()
}
fn apply_presence(
&mut self,
scope_key: String,
kind: Option<PresenceKind>,
actor_id: Option<String>,
client_id: Option<String>,
doc: Option<Value>,
error: Option<String>,
) {
if error.is_some() {
return;
}
let (Some(kind), Some(actor_id), Some(client_id)) = (kind, actor_id, client_id) else {
return;
};
let peer_key = format!("{actor_id} {client_id}");
match kind {
PresenceKind::Leave => {
if let Some(peers) = self.presence.get_mut(&scope_key) {
peers.remove(&peer_key);
if peers.is_empty() {
self.presence.remove(&scope_key);
}
}
}
_ => {
let doc = match doc {
Some(Value::Object(_)) => doc.unwrap(),
_ => return,
};
self.presence.entry(scope_key).or_default().insert(
peer_key,
PresencePeer {
actor_id,
client_id,
doc,
},
);
}
}
}
pub fn on_realtime_text(&mut self, text: &str) {
match parse_control(text) {
Ok(ControlMessage::Hello { requires_sync, .. }) => {
if requires_sync {
self.set_sync_needed(true, true);
}
}
Ok(ControlMessage::Presence {
scope_key,
kind,
actor_id,
client_id,
doc,
error,
..
}) => {
self.apply_presence(scope_key, kind, actor_id, client_id, doc, error);
}
Ok(ControlMessage::Wake { .. }) => {
self.set_sync_needed(true, true);
}
_ => {}
}
}
pub fn on_realtime_binary(&mut self, transport: &mut dyn Transport, bytes: &[u8]) {
if self.stopped {
return;
}
#[cfg(feature = "bench-internals")]
let decode_phase = self.benchmark_phases.start(Phase::ResponseDecode);
let message = match decode_message(bytes) {
Ok(m) if m.msg_kind == MsgKind::Response => m,
_ => {
self.set_sync_needed(true, true);
return;
}
};
#[cfg(feature = "bench-internals")]
drop(decode_phase);
let mut frames = message.frames.into_iter();
let mut applied_cursor: Option<i64> = None;
let mut any_covered = false;
let mut dropped = false;
while let Some(frame) = frames.next() {
let Frame::SubStart {
id,
status,
effective_scopes,
..
} = frame
else {
continue;
};
let mut body = Vec::new();
let mut next_cursor: Option<i64> = None;
for inner in frames.by_ref() {
match inner {
Frame::SubEnd {
next_cursor: nc, ..
} => {
next_cursor = Some(nc);
break;
}
Frame::Unknown { .. } => {}
other => body.push(other),
}
}
let Some(sub_index) = self.subs.iter().position(|s| s.id == id) else {
dropped = true;
continue;
};
let sub = &self.subs[sub_index];
if status != SubStatus::Active
|| sub.state != SubState::Active
|| sub.bootstrap_state.is_some()
|| !sub.synced_once
{
dropped = true;
continue;
}
if next_cursor.is_some_and(|cursor| cursor <= sub.cursor) {
any_covered = true;
continue;
}
let previous_effective = self.subs[sub_index].effective.clone();
let previous_cursor = self.subs[sub_index].cursor;
self.subs[sub_index].effective = Some(effective_scopes);
let mut failed = false;
for inner in body {
match inner {
Frame::Commit {
tables, changes, ..
} => {
if self.apply_commit_frame(&tables, &changes).is_err() {
failed = true;
break;
}
}
_ => {
failed = true;
break;
}
}
}
let committed = next_cursor.filter(|_| !failed).and_then(|cursor| {
self.apply_sub_end(sub_index, cursor, None)
.ok()
.map(|()| cursor)
});
let Some(next_cursor) = committed else {
self.subs[sub_index].effective = previous_effective;
self.subs[sub_index].cursor = previous_cursor;
dropped = true;
continue;
};
applied_cursor = Some(applied_cursor.map_or(next_cursor, |c| c.max(next_cursor)));
}
if let Some(cursor) = applied_cursor {
let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
let _ = transport.realtime_send(&ack);
} else if !any_covered || dropped {
self.set_sync_needed(true, true);
}
}
fn ack_after_pull(&mut self, transport: &mut dyn Transport) {
if !self.realtime_connected() {
return;
}
let floor = self
.subs
.iter()
.filter(|s| {
s.state == SubState::Active
&& s.bootstrap_state.is_none()
&& s.synced_once
&& s.cursor >= 0
})
.map(|s| s.cursor)
.min();
if let Some(cursor) = floor {
let ack = format!("{{\"type\":\"ack\",\"cursor\":{cursor}}}");
let _ = transport.realtime_send(&ack);
}
}
}
#[cfg(test)]
mod previous_version_wiring_tests {
use super::*;
use serde_json::{json, Value};
fn previous_version_schema(version: i32) -> Value {
json!({
"version": version,
"tables": [{
"name": "tasks",
"primaryKey": "id",
"columns": [
{ "name": "id", "type": "string", "nullable": false },
{ "name": "note", "type": "string", "nullable": true }
],
"scopes": []
}]
})
}
fn previous_version_limits() -> crate::api::ClientLimits {
crate::api::ClientLimits {
previous_version_context: Some(crate::previous_version::PreviousVersionContextConfig {
enabled: true,
..Default::default()
}),
..Default::default()
}
}
fn previous_version_config() -> crate::previous_version::PreviousVersionContextConfig {
previous_version_limits()
.previous_version_context
.expect("configured")
}
fn recapture_previous_version(client: &SyncClient) {
let replica_path = client
.previous_version_replica_path()
.expect("file-backed replica");
let outcome = crate::previous_version::capture_previous_version_from_replica(
&client.conn,
&replica_path,
2,
2,
&previous_version_config(),
1,
)
.expect("capture");
assert!(matches!(
outcome,
crate::previous_version::CaptureOutcome::Captured(_)
));
}
fn subscription(state: SubState, cursor: i64, bootstrap_state: Option<&str>) -> Subscription {
Subscription {
id: "wiring-sub".to_owned(),
table: "tasks".to_owned(),
requested: Vec::new(),
params: None,
cursor,
bootstrap_state: bootstrap_state.map(str::to_owned),
state,
reason_code: None,
effective: None,
synced_once: false,
}
}
#[test]
fn previous_version_read_reasons_come_from_live_client_state() {
use crate::previous_version::{
PreviousVersionReadSpec, PreviousVersionReason, PREVIOUS_VERSION_CONTAINER_SUFFIX,
};
let path = std::env::temp_dir()
.join(format!("syncular-prev-wiring-{}.db", uuid::Uuid::new_v4()))
.to_string_lossy()
.into_owned();
let container = format!("{path}{PREVIOUS_VERSION_CONTAINER_SUFFIX}");
{
let mut setup = SyncClient::open_path(
"wiring".to_owned(),
&previous_version_schema(1),
previous_version_limits(),
&path,
)
.expect("v1 install");
setup
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([("id".to_owned(), json!("t1"))]),
base_version: None,
}])
.expect("seed a row");
}
let mut client = SyncClient::open_path(
"wiring".to_owned(),
&previous_version_schema(2),
previous_version_limits(),
&path,
)
.expect("v2 bump captures");
assert!(std::path::Path::new(&container).exists());
let spec = PreviousVersionReadSpec {
table: "tasks".to_owned(),
..Default::default()
};
recapture_previous_version(&client);
client.lease_state = Some(LeaseState {
lease_id: None,
expires_at_ms: None,
error_code: Some("lease.expired".to_owned()),
});
let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
assert!(!snapshot.available);
assert!(!std::path::Path::new(&container).exists());
recapture_previous_version(&client);
client.set_now_ms(1_000);
client.lease_state = Some(LeaseState {
lease_id: Some("lease".to_owned()),
expires_at_ms: Some(1_000),
error_code: None,
});
let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
assert_eq!(snapshot.reason, Some(PreviousVersionReason::LeaseInactive));
assert!(!std::path::Path::new(&container).exists());
client.lease_state = None;
recapture_previous_version(&client);
client.subs.push(subscription(SubState::Revoked, 0, None));
let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
assert_eq!(snapshot.reason, Some(PreviousVersionReason::ScopeRevoked));
assert!(!std::path::Path::new(&container).exists());
client.subs.clear();
recapture_previous_version(&client);
client
.subs
.push(subscription(SubState::Active, 0, Some("resume")));
assert!(!client.previous_version_coverage_complete());
let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
assert!(snapshot.available);
assert_eq!(snapshot.previous_version, Some(2));
assert_eq!(snapshot.rows.len(), 1);
recapture_previous_version(&client);
client.subs[0].bootstrap_state = None;
assert!(client.previous_version_coverage_complete());
let snapshot = client.previous_version_snapshot(&spec).expect("snapshot");
assert_eq!(
snapshot.reason,
Some(PreviousVersionReason::CoverageComplete)
);
assert!(!std::path::Path::new(&container).exists());
client.subs.clear();
assert!(!client.previous_version_coverage_complete());
drop(client);
let _ = std::fs::remove_file(&path);
}
#[test]
fn previous_version_lifetime_check_drops_on_lease_coverage_and_revocation() {
let path = std::env::temp_dir()
.join(format!(
"syncular-prev-lifetime-{}.db",
uuid::Uuid::new_v4()
))
.to_string_lossy()
.into_owned();
let container = format!(
"{path}{}",
crate::previous_version::PREVIOUS_VERSION_CONTAINER_SUFFIX
);
let exists = || std::path::Path::new(&container).exists();
{
let mut setup = SyncClient::open_path(
"lifetime".to_owned(),
&previous_version_schema(1),
previous_version_limits(),
&path,
)
.expect("v1 install");
setup
.mutate(vec![Mutation::Upsert {
table: "tasks".to_owned(),
values: Map::from_iter([("id".to_owned(), json!("t1"))]),
base_version: None,
}])
.expect("seed a row");
}
let mut client = SyncClient::open_path(
"lifetime".to_owned(),
&previous_version_schema(2),
previous_version_limits(),
&path,
)
.expect("v2 bump captures");
client.set_now_ms(1_000);
assert!(exists());
client.lease_state = Some(LeaseState {
lease_id: None,
expires_at_ms: None,
error_code: Some("lease.expired".to_owned()),
});
client.previous_version_lifetime_check();
assert!(!exists());
client.lease_state = None;
recapture_previous_version(&client);
client.subs.push(subscription(SubState::Active, 0, None));
client.previous_version_lifetime_check();
assert!(!exists());
client.subs.clear();
recapture_previous_version(&client);
client.subs.push(subscription(SubState::Revoked, 0, None));
client.previous_version_lifetime_check();
assert!(!exists());
client.subs.clear();
recapture_previous_version(&client);
client
.subs
.push(subscription(SubState::Active, 0, Some("resume")));
client.previous_version_lifetime_check();
assert!(exists());
client.subs.clear();
drop(client);
let _ = std::fs::remove_file(&path);
let _ = std::fs::remove_file(&container);
let _ = std::fs::remove_file(format!("{container}-wal"));
let _ = std::fs::remove_file(format!("{container}-shm"));
}
}