//! Realtime synchronized Rust state for Firebase Realtime Database.
//!
//! `rtdb-sync` owns local synchronization semantics. A [`Backend`] supplies
//! hydration, realtime events, and writes; the Firebase REST/SSE implementation
//! is provided as [`RtdbBackend`].
use async_trait::async_trait;
use futures_core::Stream;
use futures_util::StreamExt;
use rtdb_rs::{RtdbClient, RtdbEvent};
use serde::{de::DeserializeOwned, Serialize};
use serde_json::{Map, Value};
use std::{
fmt,
fs::{self, File, OpenOptions},
hash::{Hash, Hasher},
io::{BufRead, BufReader, Write as IoWrite},
path::{Path, PathBuf},
pin::Pin,
sync::{
atomic::{AtomicU64, Ordering},
Arc, Mutex,
},
time::{Duration, SystemTime, UNIX_EPOCH},
};
use thiserror::Error;
use tokio::sync::{mpsc, watch};
use tokio_util::sync::CancellationToken;
pub type EventStream = Pin<Box<dyn Stream<Item = Result<Event, SyncError>> + Send>>;
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub enum Event {
Put {
path: String,
data: Value,
},
Patch {
path: String,
data: Map<String, Value>,
},
}
#[derive(Debug, Error, Clone, PartialEq, Eq)]
pub enum SyncError {
#[error("backend: {0}")]
Backend(String),
#[error("invalid event path: {0}")]
InvalidPath(String),
#[error("conversion failed: {0}")]
Conversion(String),
#[error("synchronization cancelled")]
Cancelled,
#[error("conflict: {0}")]
Conflict(String),
#[error("persistence: {0}")]
Persistence(String),
}
pub const PERSISTENCE_FORMAT_VERSION: u32 = 1;
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct DurableSnapshot {
pub format_version: u32,
pub sync_key: String,
pub generation: u64,
pub value: Value,
pub saved_at_ms: u64,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub enum DurableMutationKind {
Put,
Patch,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct DurableMutation {
pub format_version: u32,
pub sync_key: String,
pub id: u64,
pub path: String,
pub kind: DurableMutationKind,
pub payload: Value,
pub generation: u64,
pub created_at_ms: u64,
}
#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct SyncMetadata {
pub format_version: u32,
pub sync_key: String,
pub last_remote_event_at_ms: Option<u64>,
pub last_successful_sync_at_ms: Option<u64>,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct PersistedState {
pub snapshot: Option<DurableSnapshot>,
pub pending: Vec<DurableMutation>,
pub metadata: Option<SyncMetadata>,
}
pub trait PersistenceBackend: Send + Sync + fmt::Debug {
fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError>;
fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError>;
fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError>;
fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError>;
fn compact(&self, sync_key: &str) -> Result<(), SyncError>;
fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError>;
fn clear(&self, sync_key: &str) -> Result<(), SyncError>;
}
#[derive(Debug, Default, Clone)]
pub struct MemoryPersistence {
entries: Arc<Mutex<std::collections::HashMap<String, PersistedState>>>,
}
impl PersistenceBackend for MemoryPersistence {
fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError> {
Ok(self
.entries
.lock()
.map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?
.get(sync_key)
.cloned()
.unwrap_or_default())
}
fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError> {
let mut entries = self
.entries
.lock()
.map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
let entry = entries.entry(snapshot.sync_key.clone()).or_default();
entry.snapshot = Some(snapshot.clone());
Ok(())
}
fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError> {
let mut entries = self
.entries
.lock()
.map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
entries
.entry(mutation.sync_key.clone())
.or_default()
.pending
.push(mutation.clone());
Ok(())
}
fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError> {
let mut entries = self
.entries
.lock()
.map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
if let Some(entry) = entries.get_mut(sync_key) {
entry.pending.retain(|mutation| mutation.id != mutation_id);
}
Ok(())
}
fn compact(&self, _sync_key: &str) -> Result<(), SyncError> {
Ok(())
}
fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError> {
let mut entries = self
.entries
.lock()
.map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
entries
.entry(metadata.sync_key.clone())
.or_default()
.metadata = Some(metadata.clone());
Ok(())
}
fn clear(&self, sync_key: &str) -> Result<(), SyncError> {
self.entries
.lock()
.map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?
.remove(sync_key);
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct FilePersistence {
root: Arc<PathBuf>,
}
impl FilePersistence {
pub fn new(root: impl Into<PathBuf>) -> Result<Self, SyncError> {
let root = root.into();
fs::create_dir_all(&root).map_err(|error| persistence_io(&root, error))?;
restrict_permissions(&root, 0o700)?;
Ok(Self {
root: Arc::new(root),
})
}
fn directory(&self, sync_key: &str) -> Result<PathBuf, SyncError> {
let mut hasher = std::collections::hash_map::DefaultHasher::new();
sync_key.hash(&mut hasher);
let directory = self.root.join(format!("{:016x}", hasher.finish()));
fs::create_dir_all(&directory).map_err(|error| persistence_io(&directory, error))?;
restrict_permissions(&directory, 0o700)?;
Ok(directory)
}
fn path(&self, sync_key: &str, name: &str) -> Result<PathBuf, SyncError> {
Ok(self.directory(sync_key)?.join(name))
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
enum JournalEntry {
Append(DurableMutation),
Acknowledge { sync_key: String, mutation_id: u64 },
}
impl PersistenceBackend for FilePersistence {
fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError> {
let snapshot_path = self.path(sync_key, "snapshot.json")?;
let metadata_path = self.path(sync_key, "metadata.json")?;
let journal_path = self.path(sync_key, "journal.jsonl")?;
let snapshot = if snapshot_path.exists() {
let bytes =
fs::read(&snapshot_path).map_err(|error| persistence_io(&snapshot_path, error))?;
let value: DurableSnapshot = serde_json::from_slice(&bytes)
.map_err(|error| SyncError::Persistence(format!("invalid snapshot: {error}")))?;
validate_key_and_version(&value.format_version, &value.sync_key, sync_key)?;
Some(value)
} else {
None
};
let metadata = if metadata_path.exists() {
let bytes =
fs::read(&metadata_path).map_err(|error| persistence_io(&metadata_path, error))?;
let value: SyncMetadata = serde_json::from_slice(&bytes)
.map_err(|error| SyncError::Persistence(format!("invalid metadata: {error}")))?;
validate_key_and_version(&value.format_version, &value.sync_key, sync_key)?;
Some(value)
} else {
None
};
let mut pending = std::collections::BTreeMap::new();
if journal_path.exists() {
for line in BufReader::new(
File::open(&journal_path).map_err(|error| persistence_io(&journal_path, error))?,
)
.lines()
{
let line = line.map_err(|error| persistence_io(&journal_path, error))?;
let entry: JournalEntry = serde_json::from_str(&line).map_err(|error| {
SyncError::Persistence(format!("invalid journal entry: {error}"))
})?;
match entry {
JournalEntry::Append(mutation) => {
validate_key_and_version(
&mutation.format_version,
&mutation.sync_key,
sync_key,
)?;
pending.insert(mutation.id, mutation);
}
JournalEntry::Acknowledge {
sync_key: entry_key,
mutation_id,
} => {
if entry_key != sync_key {
return Err(SyncError::Persistence(
"journal namespace mismatch".into(),
));
}
pending.remove(&mutation_id);
}
}
}
}
Ok(PersistedState {
snapshot,
pending: pending.into_values().collect(),
metadata,
})
}
fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError> {
validate_key_and_version(
&snapshot.format_version,
&snapshot.sync_key,
&snapshot.sync_key,
)?;
let path = self.path(&snapshot.sync_key, "snapshot.json")?;
let tmp = path.with_extension("json.tmp");
let bytes = serde_json::to_vec(snapshot)
.map_err(|error| SyncError::Persistence(error.to_string()))?;
write_atomic(&path, &tmp, &bytes)
}
fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError> {
validate_key_and_version(
&mutation.format_version,
&mutation.sync_key,
&mutation.sync_key,
)?;
let path = self.path(&mutation.sync_key, "journal.jsonl")?;
append_line(&path, &JournalEntry::Append(mutation.clone()))
}
fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError> {
let path = self.path(sync_key, "journal.jsonl")?;
append_line(
&path,
&JournalEntry::Acknowledge {
sync_key: sync_key.into(),
mutation_id,
},
)
}
fn compact(&self, sync_key: &str) -> Result<(), SyncError> {
let state = self.load(sync_key)?;
let path = self.path(sync_key, "journal.jsonl")?;
let tmp = path.with_extension("jsonl.tmp");
let mut bytes = Vec::new();
for mutation in state.pending {
serde_json::to_writer(&mut bytes, &JournalEntry::Append(mutation))
.map_err(|error| SyncError::Persistence(error.to_string()))?;
bytes.push(b'\n');
}
write_atomic(&path, &tmp, &bytes)
}
fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError> {
validate_key_and_version(
&metadata.format_version,
&metadata.sync_key,
&metadata.sync_key,
)?;
let path = self.path(&metadata.sync_key, "metadata.json")?;
let tmp = path.with_extension("json.tmp");
let bytes = serde_json::to_vec(metadata)
.map_err(|error| SyncError::Persistence(error.to_string()))?;
write_atomic(&path, &tmp, &bytes)
}
fn clear(&self, sync_key: &str) -> Result<(), SyncError> {
let directory = self.directory(sync_key)?;
if directory.exists() {
fs::remove_dir_all(&directory).map_err(|error| persistence_io(&directory, error))?;
}
Ok(())
}
}
fn now_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as u64
}
fn validate_key_and_version(
version: &u32,
entry_key: &str,
expected_key: &str,
) -> Result<(), SyncError> {
if *version != PERSISTENCE_FORMAT_VERSION {
return Err(SyncError::Persistence(format!(
"unsupported persistence format version {version}"
)));
}
if entry_key != expected_key {
return Err(SyncError::Persistence(
"persistence namespace mismatch".into(),
));
}
Ok(())
}
fn persistence_io(path: &Path, error: std::io::Error) -> SyncError {
SyncError::Persistence(format!("{}: {error}", path.display()))
}
fn restrict_permissions(path: &Path, mode: u32) -> Result<(), SyncError> {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(path, fs::Permissions::from_mode(mode))
.map_err(|error| persistence_io(path, error))?;
}
Ok(())
}
fn write_atomic(path: &Path, tmp: &Path, bytes: &[u8]) -> Result<(), SyncError> {
let mut file = OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(tmp)
.map_err(|error| persistence_io(tmp, error))?;
file.write_all(bytes)
.map_err(|error| persistence_io(tmp, error))?;
file.sync_all()
.map_err(|error| persistence_io(tmp, error))?;
restrict_permissions(tmp, 0o600)?;
fs::rename(tmp, path).map_err(|error| persistence_io(path, error))?;
if let Some(parent) = path.parent() {
if let Ok(directory) = File::open(parent) {
let _ = directory.sync_all();
}
}
Ok(())
}
fn append_line<T: serde::Serialize>(path: &Path, entry: &T) -> Result<(), SyncError> {
let mut file = OpenOptions::new()
.create(true)
.append(true)
.open(path)
.map_err(|error| persistence_io(path, error))?;
serde_json::to_writer(&mut file, entry)
.map_err(|error| SyncError::Persistence(error.to_string()))?;
file.write_all(b"\n")
.map_err(|error| persistence_io(path, error))?;
file.sync_all()
.map_err(|error| persistence_io(path, error))?;
restrict_permissions(path, 0o600)
}
async fn persistence_load(
store: Arc<dyn PersistenceBackend>,
sync_key: String,
) -> Result<PersistedState, SyncError> {
tokio::task::spawn_blocking(move || store.load(&sync_key))
.await
.map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
}
async fn persistence_snapshot(
store: Arc<dyn PersistenceBackend>,
snapshot: DurableSnapshot,
) -> Result<(), SyncError> {
tokio::task::spawn_blocking(move || store.store_snapshot(&snapshot))
.await
.map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
}
async fn persistence_append(
store: Arc<dyn PersistenceBackend>,
mutation: DurableMutation,
) -> Result<(), SyncError> {
tokio::task::spawn_blocking(move || store.append_mutation(&mutation))
.await
.map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
}
async fn persistence_ack(
store: Arc<dyn PersistenceBackend>,
sync_key: String,
mutation_id: u64,
) -> Result<(), SyncError> {
tokio::task::spawn_blocking(move || store.acknowledge_mutation(&sync_key, mutation_id))
.await
.map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
}
async fn persistence_compact(
store: Arc<dyn PersistenceBackend>,
sync_key: String,
) -> Result<(), SyncError> {
tokio::task::spawn_blocking(move || store.compact(&sync_key))
.await
.map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
}
async fn persistence_metadata(
store: Arc<dyn PersistenceBackend>,
metadata: SyncMetadata,
) -> Result<(), SyncError> {
tokio::task::spawn_blocking(move || store.store_metadata(&metadata))
.await
.map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
}
#[async_trait]
pub trait Backend: Send + Sync + 'static {
async fn get(&self, path: &str) -> Result<Value, SyncError>;
async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError>;
async fn put(&self, path: &str, value: Value) -> Result<(), SyncError>;
async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError>;
async fn before_reconnect(&self, _attempt: usize) -> Result<(), SyncError> {
Ok(())
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Snapshot {
pub generation: u64,
pub value: Value,
}
impl Snapshot {
pub fn decode<T: DeserializeOwned>(&self) -> Result<T, SyncError> {
serde_json::from_value(self.value.clone()).map_err(|e| SyncError::Conversion(e.to_string()))
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct TypedSnapshot<T> {
pub generation: u64,
pub value: T,
}
pub struct TypedSyncHandle<T> {
inner: SyncHandle,
snapshot: watch::Receiver<Result<TypedSnapshot<T>, SyncError>>,
conversion_task: tokio::task::JoinHandle<()>,
}
impl<T: Clone> TypedSyncHandle<T> {
pub fn snapshot(&self) -> Result<TypedSnapshot<T>, SyncError> {
self.snapshot.borrow().clone()
}
pub fn subscribe(&self) -> watch::Receiver<Result<TypedSnapshot<T>, SyncError>> {
self.snapshot.clone()
}
pub fn status(&self) -> SyncStatus {
self.inner.status()
}
pub fn subscribe_status(&self) -> watch::Receiver<SyncStatus> {
self.inner.subscribe_status()
}
pub fn metrics(&self) -> MetricsSnapshot {
self.inner.metrics()
}
pub async fn put<V: Serialize>(
&self,
path: impl Into<String>,
value: V,
) -> Result<(), SyncError> {
self.inner.put(path, value).await
}
pub async fn patch<V: Serialize>(
&self,
path: impl Into<String>,
value: V,
) -> Result<(), SyncError> {
self.inner.patch(path, value).await
}
pub async fn shutdown(self) {
self.inner.shutdown().await;
let _ = self.conversion_task.await;
}
}
pub fn start_typed<B, T>(
backend: Arc<B>,
path: impl Into<String>,
config: Config,
) -> TypedSyncHandle<T>
where
B: Backend,
T: DeserializeOwned + Clone + Send + Sync + 'static,
{
let inner = start(backend, path, config);
let mut raw = inner.subscribe();
let initial = raw.borrow().decode::<T>().map(|value| TypedSnapshot {
generation: raw.borrow().generation,
value,
});
let (snapshot, receiver) = watch::channel(initial);
let conversion_task = tokio::spawn(async move {
loop {
if raw.changed().await.is_err() {
break;
}
let current = raw.borrow().clone();
let converted = current.decode::<T>().map(|value| TypedSnapshot {
generation: current.generation,
value,
});
let _ = snapshot.send(converted);
}
});
TypedSyncHandle {
inner,
snapshot: receiver,
conversion_task,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RetryPolicy {
Never,
Exponential {
max_attempts: Option<usize>,
base: Duration,
max: Duration,
},
}
impl RetryPolicy {
pub fn delay(&self, attempt: usize, jitter_max: Duration) -> Option<Duration> {
match *self {
RetryPolicy::Never => None,
RetryPolicy::Exponential {
max_attempts,
base,
max,
} => {
if max_attempts.map(|n| attempt > n).unwrap_or(false) {
return None;
}
let factor = 2u32.saturating_pow(attempt.saturating_sub(1) as u32);
let backoff = base.saturating_mul(factor).min(max);
let jitter = if jitter_max.is_zero() {
Duration::ZERO
} else {
let seed = (attempt as u64)
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
Duration::from_nanos(seed % (jitter_max.as_nanos() as u64 + 1))
};
Some(
backoff
.saturating_add(jitter)
.min(max.saturating_add(jitter_max)),
)
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WritePolicy {
Confirmed,
Optimistic,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConflictPolicy {
RemoteWins,
LocalWins,
Reject,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OfflinePolicy {
RejectWhileOffline,
QueueWhileOffline,
QueueWithLimit { max_pending: usize },
}
#[derive(Debug, Clone)]
pub struct Config {
pub retry: RetryPolicy,
pub write_policy: WritePolicy,
pub jitter_max: Duration,
pub conflict_policy: ConflictPolicy,
/// Maximum number of local mutations waiting for the synchronization task.
pub write_queue_capacity: usize,
pub persistence: Option<Arc<dyn PersistenceBackend>>,
pub persistence_key: Option<String>,
pub offline_policy: OfflinePolicy,
}
impl Default for Config {
fn default() -> Self {
Self {
retry: RetryPolicy::Exponential {
max_attempts: None,
base: Duration::from_millis(50),
max: Duration::from_secs(5),
},
write_policy: WritePolicy::Confirmed,
jitter_max: Duration::ZERO,
conflict_policy: ConflictPolicy::RemoteWins,
write_queue_capacity: 64,
persistence: None,
persistence_key: None,
offline_policy: OfflinePolicy::RejectWhileOffline,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MetricsSnapshot {
pub reconnect_attempts: u64,
pub stream_failures: u64,
pub hydration_failures: u64,
pub successful_writes: u64,
pub failed_writes: u64,
pub pending_mutations: u64,
pub persistence_failures: u64,
pub replay_attempts: u64,
pub replay_successes: u64,
pub replay_failures: u64,
pub conflict_count: u64,
}
#[derive(Default)]
struct Metrics {
reconnect_attempts: AtomicU64,
stream_failures: AtomicU64,
hydration_failures: AtomicU64,
successful_writes: AtomicU64,
failed_writes: AtomicU64,
pending_mutations: AtomicU64,
persistence_failures: AtomicU64,
replay_attempts: AtomicU64,
replay_successes: AtomicU64,
replay_failures: AtomicU64,
conflict_count: AtomicU64,
}
impl Metrics {
fn snapshot(&self) -> MetricsSnapshot {
MetricsSnapshot {
reconnect_attempts: self.reconnect_attempts.load(Ordering::Relaxed),
stream_failures: self.stream_failures.load(Ordering::Relaxed),
hydration_failures: self.hydration_failures.load(Ordering::Relaxed),
successful_writes: self.successful_writes.load(Ordering::Relaxed),
failed_writes: self.failed_writes.load(Ordering::Relaxed),
pending_mutations: self.pending_mutations.load(Ordering::Relaxed),
persistence_failures: self.persistence_failures.load(Ordering::Relaxed),
replay_attempts: self.replay_attempts.load(Ordering::Relaxed),
replay_successes: self.replay_successes.load(Ordering::Relaxed),
replay_failures: self.replay_failures.load(Ordering::Relaxed),
conflict_count: self.conflict_count.load(Ordering::Relaxed),
}
}
}
pub struct SyncHandle {
snapshot: watch::Receiver<Snapshot>,
status: watch::Receiver<SyncStatus>,
cancel: CancellationToken,
writes: mpsc::Sender<Write>,
join: tokio::task::JoinHandle<()>,
metrics: Arc<Metrics>,
}
enum Write {
Put(
String,
Value,
tokio::sync::oneshot::Sender<Result<(), SyncError>>,
),
Patch(
String,
Map<String, Value>,
tokio::sync::oneshot::Sender<Result<(), SyncError>>,
),
}
struct PendingMutation {
path: String,
event: Event,
}
impl SyncHandle {
pub fn snapshot(&self) -> Snapshot {
self.snapshot.borrow().clone()
}
pub fn subscribe(&self) -> watch::Receiver<Snapshot> {
self.snapshot.clone()
}
pub fn status(&self) -> SyncStatus {
self.status.borrow().clone()
}
pub fn subscribe_status(&self) -> watch::Receiver<SyncStatus> {
self.status.clone()
}
pub fn metrics(&self) -> MetricsSnapshot {
self.metrics.snapshot()
}
pub async fn put<T: Serialize>(
&self,
path: impl Into<String>,
value: T,
) -> Result<(), SyncError> {
let (tx, rx) = tokio::sync::oneshot::channel();
let v = serde_json::to_value(value).map_err(|e| SyncError::Conversion(e.to_string()))?;
self.writes
.send(Write::Put(path.into(), v, tx))
.await
.map_err(|_| SyncError::Cancelled)?;
rx.await.map_err(|_| SyncError::Cancelled)?
}
pub async fn patch<T: Serialize>(
&self,
path: impl Into<String>,
value: T,
) -> Result<(), SyncError> {
let (tx, rx) = tokio::sync::oneshot::channel();
let v = serde_json::to_value(value).map_err(|e| SyncError::Conversion(e.to_string()))?;
let data = v
.as_object()
.cloned()
.ok_or_else(|| SyncError::Conversion("patch must be an object".into()))?;
self.writes
.send(Write::Patch(path.into(), data, tx))
.await
.map_err(|_| SyncError::Cancelled)?;
rx.await.map_err(|_| SyncError::Cancelled)?
}
pub async fn shutdown(self) {
self.cancel.cancel();
let _ = self.join.await;
}
}
pub fn start<B: Backend>(backend: Arc<B>, path: impl Into<String>, config: Config) -> SyncHandle {
let path = path.into();
let persistence = config.persistence.clone();
let sync_key = config
.persistence_key
.clone()
.unwrap_or_else(|| path.clone());
let (snap_tx, snap_rx) = watch::channel(Snapshot {
generation: 0,
value: Value::Null,
});
let (status_tx, status_rx) = watch::channel(SyncStatus::Idle);
let cancel = CancellationToken::new();
let (writes, mut write_rx) = mpsc::channel(config.write_queue_capacity.max(1));
let child = cancel.clone();
let metrics = Arc::new(Metrics::default());
let task_metrics = metrics.clone();
let join = tokio::spawn(async move {
if matches!(
config.offline_policy,
OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
) && persistence.is_none()
{
let _ = status_tx.send(SyncStatus::Failed(SyncError::Persistence(
"offline queue requires a persistence backend".into(),
)));
return;
}
#[allow(unused_assignments)]
let mut state: Option<Value> = None;
let mut generation = 0;
let mut attempt = 0usize;
let mut pending = Vec::new();
let mut durable_pending = Vec::new();
let mut restored_base = None;
if let Some(store) = persistence.as_ref() {
match persistence_load(store.clone(), sync_key.clone()).await {
Ok(restored) => {
if let Some(snapshot) = restored.snapshot {
state = Some(snapshot.value);
restored_base = state.clone();
generation = snapshot.generation;
let _ = snap_tx.send(Snapshot {
generation,
value: state.as_ref().unwrap().clone(),
});
let _ = status_tx.send(SyncStatus::RestoredStale);
}
for mutation in restored.pending {
match durable_event(&mutation) {
Ok(event) => {
pending.push(PendingMutation {
path: mutation.path.clone(),
event,
});
durable_pending.push(mutation);
}
Err(error) => {
task_metrics
.persistence_failures
.fetch_add(1, Ordering::Relaxed);
let _ = status_tx.send(SyncStatus::Failed(error));
return;
}
}
}
task_metrics
.pending_mutations
.store(durable_pending.len() as u64, Ordering::Relaxed);
}
Err(error) => {
task_metrics
.persistence_failures
.fetch_add(1, Ordering::Relaxed);
let _ = status_tx.send(SyncStatus::Failed(error));
return;
}
}
}
let _ = status_tx.send(SyncStatus::Hydrating);
loop {
let hydrated = tokio::select! {
_ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
result = backend.get(&path) => result
};
match hydrated {
Ok(value) => {
state = Some(value);
if let Some(base) = restored_base.take() {
let remote = state.as_ref().expect("hydrated").clone();
if let Err(error) = resolve_restored_conflicts(
&base,
&remote,
config.conflict_policy,
&mut pending,
&mut durable_pending,
persistence.as_ref(),
&sync_key,
&task_metrics,
)
.await
{
if matches!(error, SyncError::Conflict(_)) {
task_metrics.conflict_count.fetch_add(1, Ordering::Relaxed);
let _ = status_tx.send(SyncStatus::Conflict(error));
} else {
task_metrics
.persistence_failures
.fetch_add(1, Ordering::Relaxed);
let _ = status_tx.send(SyncStatus::Failed(error));
}
return;
}
task_metrics
.pending_mutations
.store(durable_pending.len() as u64, Ordering::Relaxed);
}
let state_ref = state.as_ref().expect("hydration set state");
generation += 1;
let _ = snap_tx.send(Snapshot {
generation,
value: state_ref.clone(),
});
if let Some(store) = persistence.as_ref() {
if let Err(error) = persistence_snapshot(
store.clone(),
DurableSnapshot {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.clone(),
generation,
value: state_ref.clone(),
saved_at_ms: now_ms(),
},
)
.await
{
task_metrics
.persistence_failures
.fetch_add(1, Ordering::Relaxed);
let _ = status_tx.send(SyncStatus::Failed(error));
return;
}
if let Err(error) = persistence_metadata(
store.clone(),
SyncMetadata {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.clone(),
last_remote_event_at_ms: None,
last_successful_sync_at_ms: Some(now_ms()),
},
)
.await
{
task_metrics
.persistence_failures
.fetch_add(1, Ordering::Relaxed);
let _ = status_tx.send(SyncStatus::Failed(error));
return;
}
}
if !durable_pending.is_empty() {
let _ = status_tx.send(SyncStatus::Replaying);
if replay_pending(
&*backend,
&path,
state.as_mut().unwrap(),
&mut generation,
&snap_tx,
&mut pending,
&mut durable_pending,
persistence.as_ref(),
&sync_key,
&task_metrics,
)
.await
.is_err()
{
let _ = status_tx.send(SyncStatus::Offline);
continue;
}
}
attempt = 0;
}
Err(e) => {
task_metrics
.hydration_failures
.fetch_add(1, Ordering::Relaxed);
let offline_store = if matches!(
config.offline_policy,
OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
) {
persistence.as_ref()
} else {
None
};
if let Some(store) = offline_store {
let _ = status_tx.send(SyncStatus::Offline);
let delay = config
.retry
.delay(attempt.saturating_add(1), config.jitter_max)
.unwrap_or(Duration::from_secs(1));
tokio::select! {
_ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
Some(write) = write_rx.recv() => {
let result = handle_offline_write(
write,
state.get_or_insert(Value::Null),
&mut generation,
&snap_tx,
&mut pending,
&mut durable_pending,
store,
&sync_key,
config.offline_policy,
).await;
if result.is_ok() { task_metrics.successful_writes.fetch_add(1, Ordering::Relaxed); }
else { task_metrics.failed_writes.fetch_add(1, Ordering::Relaxed); }
task_metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed);
}
_ = tokio::time::sleep(delay) => {
attempt = attempt.saturating_add(1);
task_metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
let _ = backend.before_reconnect(attempt).await;
}
}
continue;
}
if !retry(
&*backend,
&config,
&mut attempt,
&child,
&status_tx,
&task_metrics,
)
.await
{
let _ = status_tx.send(SyncStatus::Failed(e));
return;
}
continue;
}
}
let mut stream = match backend.subscribe(&path).await {
Ok(s) => s,
Err(e) => {
let queue_offline = matches!(
config.offline_policy,
OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
);
if queue_offline {
if let Some(store) = persistence.as_ref() {
if !reconnect_or_queue(
&*backend,
&config,
&mut attempt,
&child,
&status_tx,
&task_metrics,
&mut write_rx,
&mut state,
&mut generation,
&snap_tx,
&mut pending,
&mut durable_pending,
store,
&sync_key,
)
.await
{
let _ = status_tx.send(SyncStatus::Failed(e));
return;
}
continue;
}
}
if !retry(
&*backend,
&config,
&mut attempt,
&child,
&status_tx,
&task_metrics,
)
.await
{
let _ = status_tx.send(SyncStatus::Failed(e));
return;
}
continue;
}
};
let _ = status_tx.send(SyncStatus::Connected);
loop {
tokio::select! {
_ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
Some(write) = write_rx.recv() => { let result = handle_write(&*backend, &path, state.as_mut().expect("hydrated"), &mut generation, &snap_tx, &mut pending, write, config.write_policy, persistence.as_ref(), &sync_key, &mut durable_pending).await; task_metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed); if result.is_ok() { task_metrics.successful_writes.fetch_add(1, Ordering::Relaxed); } else { task_metrics.failed_writes.fetch_add(1, Ordering::Relaxed); let _ = status_tx.send(SyncStatus::Failed(result.clone().unwrap_err())); } },
event = stream.next() => match event {
Some(Ok(event)) => match reconcile_event(&path, state.as_mut().expect("hydrated"), &mut generation, &snap_tx, &mut pending, event, config.conflict_policy) {
Ok(()) => {
if let Some(store) = persistence.as_ref() {
if let Err(error) = persistence_snapshot(store.clone(), DurableSnapshot {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.clone(),
generation,
value: state.as_ref().expect("hydrated").clone(),
saved_at_ms: now_ms(),
}).await {
task_metrics.persistence_failures.fetch_add(1, Ordering::Relaxed);
let _ = status_tx.send(SyncStatus::Failed(error));
return;
}
}
},
Err(e) => { let _ = status_tx.send(SyncStatus::Failed(e)); return; }
},
Some(Err(_)) | None => break
}
}
}
task_metrics.stream_failures.fetch_add(1, Ordering::Relaxed);
let queue_offline = matches!(
config.offline_policy,
OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
);
if queue_offline {
if let Some(store) = persistence.as_ref() {
if !reconnect_or_queue(
&*backend,
&config,
&mut attempt,
&child,
&status_tx,
&task_metrics,
&mut write_rx,
&mut state,
&mut generation,
&snap_tx,
&mut pending,
&mut durable_pending,
store,
&sync_key,
)
.await
{
let _ = status_tx.send(SyncStatus::Stopped);
return;
}
continue;
}
}
if !retry(
&*backend,
&config,
&mut attempt,
&child,
&status_tx,
&task_metrics,
)
.await
{
let _ = status_tx.send(SyncStatus::Stopped);
return;
}
}
});
SyncHandle {
snapshot: snap_rx,
status: status_rx,
cancel,
writes,
join,
metrics,
}
}
async fn retry<B: Backend + ?Sized>(
backend: &B,
config: &Config,
attempt: &mut usize,
cancel: &CancellationToken,
status: &watch::Sender<SyncStatus>,
metrics: &Metrics,
) -> bool {
*attempt += 1;
let Some(delay) = config.retry.delay(*attempt, config.jitter_max) else {
return false;
};
metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
let _ = status.send(SyncStatus::Reconnecting {
attempt: *attempt,
delay,
});
if backend.before_reconnect(*attempt).await.is_err() {
return false;
}
tokio::select! { _ = cancel.cancelled() => false, _ = tokio::time::sleep(delay) => true }
}
fn durable_event(mutation: &DurableMutation) -> Result<Event, SyncError> {
validate_key_and_version(
&mutation.format_version,
&mutation.sync_key,
&mutation.sync_key,
)?;
match mutation.kind {
DurableMutationKind::Put => Ok(Event::Put {
path: mutation.path.clone(),
data: mutation.payload.clone(),
}),
DurableMutationKind::Patch => Ok(Event::Patch {
path: mutation.path.clone(),
data: mutation.payload.as_object().cloned().ok_or_else(|| {
SyncError::Persistence("journal patch payload is not an object".into())
})?,
}),
}
}
fn durable_mutation(
sync_key: &str,
pending: &[DurableMutation],
event: &Event,
generation: u64,
) -> Option<DurableMutation> {
let (path, kind, payload) = match event {
Event::Put { path, data } => (path.clone(), DurableMutationKind::Put, data.clone()),
Event::Patch { path, data } => (
path.clone(),
DurableMutationKind::Patch,
Value::Object(data.clone()),
),
};
Some(DurableMutation {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.into(),
id: pending
.iter()
.map(|entry| entry.id)
.max()
.unwrap_or(0)
.saturating_add(1),
path,
kind,
payload,
generation,
created_at_ms: now_ms(),
})
}
fn event_would_change(root: &str, state: &Value, event: &Event) -> Result<bool, SyncError> {
let mut candidate = state.clone();
apply_event(root, &mut candidate, event.clone())?;
Ok(candidate != *state)
}
#[allow(clippy::too_many_arguments)]
async fn handle_offline_write(
write: Write,
state: &mut Value,
generation: &mut u64,
tx: &watch::Sender<Snapshot>,
pending: &mut Vec<PendingMutation>,
durable_pending: &mut Vec<DurableMutation>,
persistence: &Arc<dyn PersistenceBackend>,
sync_key: &str,
policy: OfflinePolicy,
) -> Result<(), SyncError> {
if let OfflinePolicy::QueueWithLimit { max_pending } = policy {
if durable_pending.len() >= max_pending {
let error = SyncError::Persistence("offline mutation queue is full".into());
match write {
Write::Put(_, _, done) | Write::Patch(_, _, done) => {
let _ = done.send(Err(error.clone()));
}
}
return Err(error);
}
}
let (path, kind, payload, done) = match write {
Write::Put(path, payload, done) => (path, DurableMutationKind::Put, payload, done),
Write::Patch(path, payload, done) => (
path,
DurableMutationKind::Patch,
Value::Object(payload),
done,
),
};
let event = match kind {
DurableMutationKind::Put => Event::Put {
path: path.clone(),
data: payload.clone(),
},
DurableMutationKind::Patch => Event::Patch {
path: path.clone(),
data: payload.as_object().cloned().unwrap_or_default(),
},
};
let mut next = state.clone();
if matches!(
policy,
OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
) {
apply_event("", &mut next, event.clone())?;
}
let mutation = DurableMutation {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.into(),
id: durable_pending
.iter()
.map(|entry| entry.id)
.max()
.unwrap_or(0)
.saturating_add(1),
path: path.clone(),
kind,
payload,
generation: generation.saturating_add(1),
created_at_ms: now_ms(),
};
if let Err(error) = persistence_append(persistence.clone(), mutation.clone()).await {
let _ = done.send(Err(error.clone()));
return Err(error);
}
*state = next;
*generation = mutation.generation;
let _ = tx.send(Snapshot {
generation: *generation,
value: state.clone(),
});
pending.push(PendingMutation { path, event });
durable_pending.push(mutation);
if let Err(error) = persistence_snapshot(
persistence.clone(),
DurableSnapshot {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.into(),
generation: *generation,
value: state.clone(),
saved_at_ms: now_ms(),
},
)
.await
{
let _ = done.send(Err(error.clone()));
return Err(error);
}
let _ = done.send(Ok(()));
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn replay_pending<B: Backend + ?Sized>(
backend: &B,
root: &str,
state: &mut Value,
generation: &mut u64,
tx: &watch::Sender<Snapshot>,
pending: &mut Vec<PendingMutation>,
durable_pending: &mut Vec<DurableMutation>,
persistence: Option<&Arc<dyn PersistenceBackend>>,
sync_key: &str,
metrics: &Metrics,
) -> Result<(), SyncError> {
for mutation in durable_pending.clone() {
metrics.replay_attempts.fetch_add(1, Ordering::Relaxed);
let remote_path = join(root, &mutation.path)?;
let event = durable_event(&mutation)?;
let result = match mutation.kind {
DurableMutationKind::Put => backend.put(&remote_path, mutation.payload.clone()).await,
DurableMutationKind::Patch => {
backend
.patch(
&remote_path,
mutation.payload.as_object().cloned().ok_or_else(|| {
SyncError::Persistence("journal patch payload is not an object".into())
})?,
)
.await
}
};
if let Err(error) = result {
metrics.replay_failures.fetch_add(1, Ordering::Relaxed);
return Err(error);
}
apply_event(root, state, event.clone())?;
*generation += 1;
let _ = tx.send(Snapshot {
generation: *generation,
value: state.clone(),
});
if let Some(store) = persistence {
persistence_snapshot(
store.clone(),
DurableSnapshot {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.into(),
generation: *generation,
value: state.clone(),
saved_at_ms: now_ms(),
},
)
.await?;
persistence_ack(store.clone(), sync_key.into(), mutation.id).await?;
}
if let Some(index) = durable_pending
.iter()
.position(|entry| entry.id == mutation.id)
{
durable_pending.remove(index);
}
if let Some(index) = pending
.iter()
.position(|entry| equivalent_event(&entry.event, &event))
{
pending.remove(index);
}
metrics.replay_successes.fetch_add(1, Ordering::Relaxed);
}
if let Some(store) = persistence {
persistence_compact(store.clone(), sync_key.into()).await?;
}
metrics
.pending_mutations
.store(durable_pending.len() as u64, Ordering::Relaxed);
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn resolve_restored_conflicts(
base: &Value,
remote: &Value,
policy: ConflictPolicy,
pending: &mut Vec<PendingMutation>,
durable_pending: &mut Vec<DurableMutation>,
persistence: Option<&Arc<dyn PersistenceBackend>>,
sync_key: &str,
metrics: &Metrics,
) -> Result<(), SyncError> {
let mut conflicted = false;
for mutation in durable_pending.clone() {
let event = durable_event(&mutation)?;
if base != remote && event_would_change("", base, &event)? {
conflicted = true;
match policy {
ConflictPolicy::RemoteWins => {
if let Some(store) = persistence {
persistence_ack(store.clone(), sync_key.into(), mutation.id).await?;
}
durable_pending.retain(|entry| entry.id != mutation.id);
if let Some(index) = pending
.iter()
.position(|entry| equivalent_event(&entry.event, &event))
{
pending.remove(index);
}
}
ConflictPolicy::LocalWins => {}
ConflictPolicy::Reject => {
return Err(SyncError::Conflict(format!(
"restored mutation conflicts at {}",
mutation.path
)))
}
}
}
}
if conflicted {
metrics.conflict_count.fetch_add(1, Ordering::Relaxed);
if let Some(store) = persistence {
persistence_compact(store.clone(), sync_key.into()).await?;
}
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn reconnect_or_queue<B: Backend + ?Sized>(
backend: &B,
config: &Config,
attempt: &mut usize,
cancel: &CancellationToken,
status: &watch::Sender<SyncStatus>,
metrics: &Metrics,
write_rx: &mut mpsc::Receiver<Write>,
state: &mut Option<Value>,
generation: &mut u64,
snapshot_tx: &watch::Sender<Snapshot>,
pending: &mut Vec<PendingMutation>,
durable_pending: &mut Vec<DurableMutation>,
persistence: &Arc<dyn PersistenceBackend>,
sync_key: &str,
) -> bool {
loop {
*attempt = attempt.saturating_add(1);
let delay = config
.retry
.delay(*attempt, config.jitter_max)
.unwrap_or(Duration::from_secs(1));
metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
let _ = status.send(SyncStatus::Offline);
tokio::select! {
_ = cancel.cancelled() => return false,
Some(write) = write_rx.recv() => {
let result = handle_offline_write(
write,
state.get_or_insert(Value::Null),
generation,
snapshot_tx,
pending,
durable_pending,
persistence,
sync_key,
config.offline_policy,
).await;
if result.is_ok() {
metrics.successful_writes.fetch_add(1, Ordering::Relaxed);
} else {
metrics.failed_writes.fetch_add(1, Ordering::Relaxed);
}
metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed);
}
_ = tokio::time::sleep(delay) => {
if backend.before_reconnect(*attempt).await.is_ok() {
return true;
}
}
}
}
}
#[allow(clippy::too_many_arguments)]
async fn handle_write<B: Backend + ?Sized>(
backend: &B,
root: &str,
state: &mut Value,
generation: &mut u64,
tx: &watch::Sender<Snapshot>,
pending: &mut Vec<PendingMutation>,
write: Write,
policy: WritePolicy,
persistence: Option<&Arc<dyn PersistenceBackend>>,
sync_key: &str,
durable_pending: &mut Vec<DurableMutation>,
) -> Result<(), SyncError> {
match write {
Write::Put(path, value, done) => {
let old = state.clone();
let backend_path = join(root, &path)?;
let event = Event::Put {
path: path.clone(),
data: value.clone(),
};
let durable = durable_mutation(sync_key, durable_pending, &event, *generation + 1);
if let Some(store) = persistence {
let mutation = durable.clone().unwrap();
if let Err(error) = persistence_append(Arc::clone(store), mutation.clone()).await {
let _ = done.send(Err(error.clone()));
return Err(error);
}
durable_pending.push(mutation);
}
if policy == WritePolicy::Optimistic {
let mut next = state.clone();
apply_event(root, &mut next, event.clone())?;
*state = next;
*generation += 1;
let _ = tx.send(Snapshot {
generation: *generation,
value: state.clone(),
});
if let Some(store) = persistence {
if let Err(error) = persistence_snapshot(
Arc::clone(store),
DurableSnapshot {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.into(),
generation: *generation,
value: state.clone(),
saved_at_ms: now_ms(),
},
)
.await
{
let _ = done.send(Err(error.clone()));
return Err(error);
}
}
}
pending.push(PendingMutation {
path: path.clone(),
event: event.clone(),
});
let result = backend.put(&backend_path, value).await;
if result.is_ok() {
if let (Some(store), Some(mutation)) = (persistence, durable) {
if let Err(error) =
persistence_ack(Arc::clone(store), sync_key.into(), mutation.id).await
{
let _ = done.send(Err(error.clone()));
return Err(error);
}
if let Err(error) =
persistence_compact(Arc::clone(store), sync_key.into()).await
{
let _ = done.send(Err(error.clone()));
return Err(error);
}
durable_pending.retain(|entry| entry.id != mutation.id);
}
}
if result.is_err() && policy == WritePolicy::Optimistic {
*state = old;
*generation += 1;
let _ = tx.send(Snapshot {
generation: *generation,
value: state.clone(),
});
}
if result.is_err() {
let mut removed = false;
pending.retain(|mutation| {
if !removed && equivalent_event(&mutation.event, &event) {
removed = true;
false
} else {
true
}
});
}
let _ = done.send(result.clone());
result
}
Write::Patch(path, data, done) => {
let old = state.clone();
let backend_path = join(root, &path)?;
let event = Event::Patch {
path: path.clone(),
data: data.clone(),
};
let durable = durable_mutation(sync_key, durable_pending, &event, *generation + 1);
if let Some(store) = persistence {
let mutation = durable.clone().unwrap();
if let Err(error) = persistence_append(Arc::clone(store), mutation.clone()).await {
let _ = done.send(Err(error.clone()));
return Err(error);
}
durable_pending.push(mutation);
}
if policy == WritePolicy::Optimistic {
let mut next = state.clone();
apply_event(root, &mut next, event.clone())?;
*state = next;
*generation += 1;
let _ = tx.send(Snapshot {
generation: *generation,
value: state.clone(),
});
if let Some(store) = persistence {
if let Err(error) = persistence_snapshot(
Arc::clone(store),
DurableSnapshot {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: sync_key.into(),
generation: *generation,
value: state.clone(),
saved_at_ms: now_ms(),
},
)
.await
{
let _ = done.send(Err(error.clone()));
return Err(error);
}
}
}
pending.push(PendingMutation {
path: path.clone(),
event: event.clone(),
});
let result = backend.patch(&backend_path, data).await;
if result.is_ok() {
if let (Some(store), Some(mutation)) = (persistence, durable) {
if let Err(error) =
persistence_ack(Arc::clone(store), sync_key.into(), mutation.id).await
{
let _ = done.send(Err(error.clone()));
return Err(error);
}
if let Err(error) =
persistence_compact(Arc::clone(store), sync_key.into()).await
{
let _ = done.send(Err(error.clone()));
return Err(error);
}
durable_pending.retain(|entry| entry.id != mutation.id);
}
}
if result.is_err() && policy == WritePolicy::Optimistic {
*state = old;
*generation += 1;
let _ = tx.send(Snapshot {
generation: *generation,
value: state.clone(),
});
}
if result.is_err() {
let mut removed = false;
pending.retain(|mutation| {
if !removed && equivalent_event(&mutation.event, &event) {
removed = true;
false
} else {
true
}
});
}
let _ = done.send(result.clone());
result
}
}
}
fn reconcile_event(
root: &str,
state: &mut Value,
generation: &mut u64,
tx: &watch::Sender<Snapshot>,
pending: &mut Vec<PendingMutation>,
event: Event,
policy: ConflictPolicy,
) -> Result<(), SyncError> {
if let Some(index) = pending
.iter()
.position(|mutation| mutation.path == event_path(&event))
{
let mutation = &pending[index];
if equivalent_event(&mutation.event, &event) {
pending.remove(index);
return Ok(());
}
match policy {
ConflictPolicy::LocalWins => return Ok(()),
ConflictPolicy::Reject => {
return Err(SyncError::Conflict(format!(
"remote event conflicts at {}",
event_path(&event)
)))
}
ConflictPolicy::RemoteWins => {
pending.remove(index);
}
}
}
apply_event(root, state, event)?;
*generation += 1;
let _ = tx.send(Snapshot {
generation: *generation,
value: state.clone(),
});
Ok(())
}
fn event_path(event: &Event) -> &str {
match event {
Event::Put { path, .. } | Event::Patch { path, .. } => path,
}
}
fn equivalent_event(left: &Event, right: &Event) -> bool {
match (left, right) {
(Event::Put { path: a, data: b }, Event::Put { path: c, data: d }) => a == c && b == d,
(Event::Patch { path: a, data: b }, Event::Patch { path: c, data: d }) => a == c && b == d,
_ => false,
}
}
fn join(root: &str, path: &str) -> Result<String, SyncError> {
let a = root.trim_matches('/');
let b = path.trim_matches('/');
if !b.is_empty() && b.split('/').any(|x| x.is_empty() || x == "." || x == "..") {
return Err(SyncError::InvalidPath(path.into()));
}
Ok(if b.is_empty() {
a.into()
} else if a.is_empty() {
b.into()
} else {
format!("{a}/{b}")
})
}
fn segments(path: &str) -> Result<Vec<&str>, SyncError> {
let p = path.trim_matches('/');
if p.is_empty() {
Ok(vec![])
} else {
let out: Vec<_> = p.split('/').collect();
if out.iter().any(|x| x.is_empty() || *x == "." || *x == "..") {
Err(SyncError::InvalidPath(path.into()))
} else {
Ok(out)
}
}
}
pub fn apply_event(root: &str, state: &mut Value, event: Event) -> Result<(), SyncError> {
let relative = match &event {
Event::Put { path, .. } | Event::Patch { path, .. } => path.clone(),
};
let _ = join(root, &relative)?;
let parts = segments(&relative)?;
let target = get_or_create(state, &parts);
match event {
Event::Put { data, .. } => {
if data.is_null() && !parts.is_empty() {
remove_at(state, &parts);
} else {
*target = data;
}
}
Event::Patch { data, .. } => {
if data.is_empty() {
return Ok(());
}
if !target.is_object() {
*target = Value::Object(Map::new());
}
let obj = target.as_object_mut().unwrap();
for (key, value) in data {
let keys = segments(&key)?;
set_at(obj, &keys, value);
}
}
}
Ok(())
}
fn get_or_create<'a>(value: &'a mut Value, parts: &[&str]) -> &'a mut Value {
let mut current = value;
for part in parts {
if !current.is_object() {
*current = Value::Object(Map::new());
}
current = current
.as_object_mut()
.unwrap()
.entry(*part)
.or_insert(Value::Null);
}
current
}
fn set_at(obj: &mut Map<String, Value>, parts: &[&str], value: Value) {
if parts.is_empty() {
return;
}
if parts.len() == 1 {
if value.is_null() {
obj.remove(parts[0]);
} else {
obj.insert(parts[0].into(), value);
}
} else {
let child = obj
.entry(parts[0])
.or_insert_with(|| Value::Object(Map::new()));
if !child.is_object() {
*child = Value::Object(Map::new());
}
set_at(child.as_object_mut().unwrap(), &parts[1..], value);
}
}
fn remove_at(value: &mut Value, parts: &[&str]) {
if parts.is_empty() {
*value = Value::Null;
return;
}
if let Some(obj) = value.as_object_mut() {
if parts.len() == 1 {
obj.remove(parts[0]);
} else if let Some(child) = obj.get_mut(parts[0]) {
remove_at(child, &parts[1..]);
}
}
}
/// Marker used while the 0.1.0 synchronization API is being designed.
///
/// No stability guarantee is attached to this pre-release scaffold.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SyncStatus {
/// No synchronization task has started.
Idle,
/// Initial state is being hydrated.
Hydrating,
/// A durable snapshot is available, but remote freshness is unconfirmed.
RestoredStale,
/// The remote backend is unavailable while local state remains usable.
Offline,
/// Durable local mutations are being replayed.
Replaying,
/// A restored local mutation conflicts with newly hydrated remote state.
Conflict(SyncError),
/// A realtime stream is active.
Connected,
/// The stream or hydration request is being retried.
Reconnecting { attempt: usize, delay: Duration },
/// Synchronization has stopped.
Stopped,
/// A non-recoverable backend, conversion, or event error occurred.
Failed(SyncError),
}
impl fmt::Display for SyncStatus {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self:?}")
}
}
/// Backend adapter that delegates REST, SSE, authentication parameters, and
/// emulator namespaces to the upstream `rtdb-rs` transport.
#[derive(Clone)]
pub struct RtdbBackend {
client: Arc<RtdbClient>,
base_url: String,
token: String,
namespace: Option<String>,
}
/// Typed transport adapter. `rtdb-typed` converts complete PUT events and
/// models; partial PATCH payloads intentionally remain JSON until the local
/// state has applied them.
pub struct TypedBackend<T> {
client: Arc<rtdb_typed::TypedClient>,
marker: std::marker::PhantomData<fn() -> T>,
}
impl<T> Clone for TypedBackend<T> {
fn clone(&self) -> Self {
Self {
client: self.client.clone(),
marker: std::marker::PhantomData,
}
}
}
impl<T> TypedBackend<T>
where
T: Serialize + DeserializeOwned + Send + Sync + 'static,
{
pub fn new(client: rtdb_typed::TypedClient) -> Self {
Self {
client: Arc::new(client),
marker: std::marker::PhantomData,
}
}
pub fn from_parts(base_url: impl Into<String>, token: impl Into<String>) -> Self {
Self::new(rtdb_typed::TypedClient::from_parts(base_url, token))
}
}
#[async_trait]
impl<T> Backend for TypedBackend<T>
where
T: Serialize + DeserializeOwned + Send + Sync + 'static,
{
async fn get(&self, path: &str) -> Result<Value, SyncError> {
self.client
.get::<T>(path)
.await
.map_err(RtdbBackend::error)
.and_then(|value| {
serde_json::to_value(value)
.map_err(|error| SyncError::Conversion(error.to_string()))
})
}
async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError> {
let stream = self
.client
.query::<T>(path)
.stream()
.await
.map_err(RtdbBackend::error)?;
let stream = stream.filter_map(|event| async move {
match event {
Ok(rtdb_typed::TypedEvent::Put { path, data }) => Some(
serde_json::to_value(data)
.map(|data| Event::Put { path, data })
.map_err(|error| SyncError::Conversion(error.to_string())),
),
Ok(rtdb_typed::TypedEvent::Patch { path, data }) => Some(Ok(Event::Patch {
path,
data: data.as_object().cloned().unwrap_or_default(),
})),
Ok(rtdb_typed::TypedEvent::KeepAlive) => None,
Ok(rtdb_typed::TypedEvent::Cancel) => {
Some(Err(SyncError::Backend("Firebase stream cancelled".into())))
}
Err(error) => Some(Err(RtdbBackend::error(error))),
}
});
Ok(Box::pin(stream))
}
async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
self.client
.inner()
.put(path, &value)
.await
.map(|_| ())
.map_err(RtdbBackend::error)
}
async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
self.client
.inner()
.patch(path, &Value::Object(value))
.await
.map(|_| ())
.map_err(RtdbBackend::error)
}
}
impl RtdbBackend {
pub fn new(base_url: impl Into<String>, token: impl Into<String>) -> Self {
let base_url = base_url.into();
let token = token.into();
Self {
client: Arc::new(RtdbClient::new(&base_url, &token)),
base_url,
token,
namespace: None,
}
}
pub fn with_namespace(mut self, namespace: impl Into<String>) -> Self {
let namespace = namespace.into();
self.client = Arc::new(
RtdbClient::new(&self.base_url, &self.token).with_namespace(namespace.clone()),
);
self.namespace = Some(namespace);
self
}
pub fn typed_client(&self) -> rtdb_typed::TypedClient {
let client = RtdbClient::new(&self.base_url, &self.token);
let client = match &self.namespace {
Some(namespace) => client.with_namespace(namespace.clone()),
None => client,
};
rtdb_typed::TypedClient::new(client)
}
fn error(error: impl fmt::Debug) -> SyncError {
SyncError::Backend(format!("{error:?}"))
}
}
#[async_trait]
impl Backend for RtdbBackend {
async fn get(&self, path: &str) -> Result<Value, SyncError> {
self.client.get(path).await.map_err(Self::error)
}
async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError> {
let stream = self.client.stream(path).await.map_err(Self::error)?;
let stream = stream.filter_map(|event| async move {
match event {
Ok(RtdbEvent::Put { path, data }) => Some(Ok(Event::Put { path, data })),
Ok(RtdbEvent::Patch { path, data }) => Some(Ok(Event::Patch {
path,
data: data.as_object().cloned().unwrap_or_default(),
})),
Ok(RtdbEvent::KeepAlive) => None,
Ok(RtdbEvent::Cancel) => {
Some(Err(SyncError::Backend("Firebase stream cancelled".into())))
}
Err(error) => Some(Err(Self::error(error))),
}
});
Ok(Box::pin(stream))
}
async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
self.client
.put(path, &value)
.await
.map(|_| ())
.map_err(Self::error)
}
async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
self.client
.patch(path, &Value::Object(value))
.await
.map(|_| ())
.map_err(Self::error)
}
}
#[cfg(test)]
mod tests {
use super::*;
use futures_util::stream;
use serde::Deserialize;
use std::sync::Mutex;
#[derive(Clone)]
struct Mock {
value: Arc<Mutex<Value>>,
events: Arc<Mutex<Vec<Event>>>,
writes: Arc<Mutex<Vec<String>>>,
}
struct Flaky {
subscriptions: AtomicU64,
replacements: AtomicU64,
}
struct Cycling {
subscriptions: AtomicU64,
}
#[async_trait]
impl Backend for Cycling {
async fn get(&self, _: &str) -> Result<Value, SyncError> {
Ok(Value::Null)
}
async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
if self.subscriptions.fetch_add(1, Ordering::Relaxed) < 25 {
Ok(Box::pin(stream::empty()))
} else {
Ok(Box::pin(stream::pending()))
}
}
async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
Ok(())
}
async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
Ok(())
}
}
struct Failing;
#[async_trait]
impl Backend for Failing {
async fn get(&self, _: &str) -> Result<Value, SyncError> {
Ok(serde_json::json!({"count": 1}))
}
async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
Ok(Box::pin(stream::pending()))
}
async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
Err(SyncError::Backend("injected write failure".into()))
}
async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
Err(SyncError::Backend("injected write failure".into()))
}
}
#[derive(Clone)]
struct Recoverable {
online: Arc<std::sync::atomic::AtomicBool>,
value: Arc<Mutex<Value>>,
events: tokio::sync::broadcast::Sender<Event>,
}
#[async_trait]
impl Backend for Recoverable {
async fn get(&self, _: &str) -> Result<Value, SyncError> {
if !self.online.load(Ordering::Relaxed) {
return Err(SyncError::Backend("offline".into()));
}
Ok(self.value.lock().unwrap().clone())
}
async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
if !self.online.load(Ordering::Relaxed) {
return Err(SyncError::Backend("offline".into()));
}
let mut receiver = self.events.subscribe();
Ok(Box::pin(async_stream::stream! {
loop { match receiver.recv().await { Ok(event) => yield Ok(event), Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, Err(tokio::sync::broadcast::error::RecvError::Closed) => break } }
}))
}
async fn put(&self, _: &str, value: Value) -> Result<(), SyncError> {
if !self.online.load(Ordering::Relaxed) {
return Err(SyncError::Backend("offline".into()));
}
*self.value.lock().unwrap() = value.clone();
let _ = self.events.send(Event::Put {
path: "".into(),
data: value,
});
Ok(())
}
async fn patch(&self, _: &str, value: Map<String, Value>) -> Result<(), SyncError> {
if !self.online.load(Ordering::Relaxed) {
return Err(SyncError::Backend("offline".into()));
}
apply_event(
"",
&mut self.value.lock().unwrap(),
Event::Patch {
path: "".into(),
data: value.clone(),
},
)?;
let _ = self.events.send(Event::Patch {
path: "".into(),
data: value,
});
Ok(())
}
}
#[derive(Clone)]
struct Live {
value: Arc<Mutex<Value>>,
events: tokio::sync::broadcast::Sender<Event>,
}
#[async_trait]
impl Backend for Live {
async fn get(&self, _: &str) -> Result<Value, SyncError> {
Ok(self.value.lock().unwrap().clone())
}
async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
let mut receiver = self.events.subscribe();
Ok(Box::pin(async_stream::stream! {
loop { match receiver.recv().await { Ok(event) => yield Ok(event), Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, Err(tokio::sync::broadcast::error::RecvError::Closed) => break } }
}))
}
async fn put(&self, _: &str, value: Value) -> Result<(), SyncError> {
*self.value.lock().unwrap() = value.clone();
let _ = self.events.send(Event::Put {
path: "".into(),
data: value,
});
Ok(())
}
async fn patch(&self, _: &str, value: Map<String, Value>) -> Result<(), SyncError> {
let mut current = self.value.lock().unwrap();
apply_event(
"",
&mut current,
Event::Patch {
path: "".into(),
data: value.clone(),
},
)?;
let _ = self.events.send(Event::Patch {
path: "".into(),
data: value,
});
Ok(())
}
}
#[async_trait]
impl Backend for Flaky {
async fn get(&self, _: &str) -> Result<Value, SyncError> {
Ok(Value::Null)
}
async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
if self.subscriptions.fetch_add(1, Ordering::Relaxed) == 0 {
Ok(Box::pin(stream::empty()))
} else {
Ok(Box::pin(stream::pending()))
}
}
async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
Ok(())
}
async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
Ok(())
}
async fn before_reconnect(&self, _: usize) -> Result<(), SyncError> {
self.replacements.fetch_add(1, Ordering::Relaxed);
Ok(())
}
}
#[async_trait]
impl Backend for Mock {
async fn get(&self, _: &str) -> Result<Value, SyncError> {
Ok(self.value.lock().unwrap().clone())
}
async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
let events = std::mem::take(&mut *self.events.lock().unwrap());
Ok(Box::pin(
stream::iter(events.into_iter().map(Ok)).chain(stream::pending()),
))
}
async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
*self.value.lock().unwrap() = value;
self.writes.lock().unwrap().push(format!("put:{path}"));
Ok(())
}
async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
apply_event(
"",
&mut self.value.lock().unwrap(),
Event::Patch {
path: "".into(),
data: value,
},
)?;
self.writes.lock().unwrap().push(format!("patch:{path}"));
Ok(())
}
}
fn mock(value: Value, events: Vec<Event>) -> Mock {
Mock {
value: Arc::new(Mutex::new(value)),
events: Arc::new(Mutex::new(events)),
writes: Arc::new(Mutex::new(Vec::new())),
}
}
#[test]
fn put_patch_and_null_delete_are_deterministic() {
let mut state = serde_json::json!({"a": {"b": 1, "c": 2}});
apply_event(
"root",
&mut state,
Event::Patch {
path: "".into(),
data: serde_json::json!({"a/b": 3, "a/c": null, "new/x": true})
.as_object()
.unwrap()
.clone(),
},
)
.unwrap();
assert_eq!(
state,
serde_json::json!({"a": {"b": 3}, "new": {"x": true}})
);
apply_event(
"root",
&mut state,
Event::Put {
path: "a".into(),
data: Value::Null,
},
)
.unwrap();
assert_eq!(state, serde_json::json!({"new": {"x": true}}));
}
#[tokio::test]
async fn hydrates_notifies_and_stops() {
let backend = mock(
serde_json::json!({"count": 1}),
vec![Event::Patch {
path: "".into(),
data: serde_json::json!({"count": 2}).as_object().unwrap().clone(),
}],
);
let handle = start(
Arc::new(backend),
"items",
Config {
retry: RetryPolicy::Never,
..Config::default()
},
);
let mut snapshots = handle.subscribe();
tokio::time::timeout(Duration::from_secs(1), async {
while snapshots.borrow().value != serde_json::json!({"count": 2}) {
snapshots.changed().await.unwrap();
}
})
.await
.unwrap();
assert_eq!(snapshots.borrow().value, serde_json::json!({"count": 2}));
handle.shutdown().await;
}
#[tokio::test]
async fn typed_snapshot_and_optimistic_rollback_contract() {
#[derive(Debug, Deserialize, PartialEq)]
struct Model {
count: u32,
}
let backend = mock(serde_json::json!({"count": 4}), vec![]);
let handle = start(
Arc::new(backend),
"items",
Config {
retry: RetryPolicy::Never,
write_policy: WritePolicy::Optimistic,
..Config::default()
},
);
let mut status = handle.subscribe_status();
tokio::time::timeout(Duration::from_secs(1), async {
while *status.borrow() != SyncStatus::Connected {
status.changed().await.unwrap();
}
})
.await
.unwrap();
assert_eq!(
handle.snapshot().decode::<Model>().unwrap(),
Model { count: 4 }
);
handle.shutdown().await;
}
#[tokio::test]
async fn typed_handle_surfaces_conversion_errors_after_partial_updates() {
#[derive(Debug, Clone, Deserialize, PartialEq)]
struct Model {
count: u32,
}
let backend = mock(
serde_json::json!({"count": 1}),
vec![
Event::Patch {
path: "".into(),
data: serde_json::json!({"count": 2}).as_object().unwrap().clone(),
},
Event::Put {
path: "".into(),
data: serde_json::json!({"count": "not-a-number"}),
},
],
);
let handle = start_typed::<_, Model>(
Arc::new(backend),
"typed",
Config {
retry: RetryPolicy::Never,
..Config::default()
},
);
let mut snapshots = handle.subscribe();
tokio::time::timeout(Duration::from_secs(1), async {
while handle.status() != SyncStatus::Connected {
snapshots.changed().await.unwrap();
}
})
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(10)).await;
assert!(matches!(
&*snapshots.borrow(),
Err(SyncError::Conversion(_))
));
handle.shutdown().await;
}
#[tokio::test]
async fn firebase_emulator_crud_and_sse_when_configured() {
let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
return;
};
let root = test_root("crud");
let backend = RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb");
backend
.put(&root, serde_json::json!({"generation": 1}))
.await
.unwrap();
assert_eq!(backend.get(&root).await.unwrap()["generation"], 1);
let mut events = backend.subscribe(&root).await.unwrap();
backend
.patch(
&root,
serde_json::json!({"generation": 2})
.as_object()
.unwrap()
.clone(),
)
.await
.unwrap();
let event = tokio::time::timeout(Duration::from_secs(2), async {
loop {
if let Some(Ok(event)) = events.next().await {
let is_target = match &event {
Event::Put { data, .. } => data["generation"] == 2,
Event::Patch { data, .. } => {
data.get("generation") == Some(&serde_json::json!(2))
}
};
if is_target {
break event;
}
}
}
})
.await
.unwrap();
assert!(
matches!(event, Event::Patch { ref data, .. } if data.get("generation") == Some(&serde_json::json!(2)))
|| matches!(event, Event::Put { ref data, .. } if data["generation"] == 2)
);
backend.put(&root, Value::Null).await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn standard_32_path_profile_converges() {
let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
return;
};
let backend = Arc::new(
RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb"),
);
let mut handles = Vec::new();
for path in 0..32 {
let handle = start(
backend.clone(),
test_root(&format!("paths/{path}")),
Config {
retry: RetryPolicy::Never,
..Config::default()
},
);
handles.push((path, handle));
}
for (path, handle) in &handles {
wait_for_status(handle, SyncStatus::Connected).await;
backend
.put(
&test_root(&format!("paths/{path}")),
serde_json::json!({"path": path, "generation": 0}),
)
.await
.unwrap();
}
for generation in 1..=50 {
for path in 0..32 {
let root = test_root(&format!("paths/{path}"));
match generation % 3 {
0 => backend
.put(
&root,
serde_json::json!({"path": path, "generation": generation}),
)
.await
.unwrap(),
1 => backend
.patch(
&root,
serde_json::json!({"generation": generation})
.as_object()
.unwrap()
.clone(),
)
.await
.unwrap(),
_ => {
backend.put(&root, Value::Null).await.unwrap();
backend
.put(
&root,
serde_json::json!({"path": path, "generation": generation}),
)
.await
.unwrap();
}
}
}
}
for (path, handle) in &handles {
let mut updates = handle.subscribe();
tokio::time::timeout(Duration::from_secs(10), async {
while updates.borrow().value["generation"] != 50 {
updates.changed().await.unwrap();
}
})
.await
.unwrap();
assert_eq!(updates.borrow().value["path"], *path);
}
for (_, handle) in handles {
handle.shutdown().await;
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn manual_heavy_64_path_profile_converges() {
if std::env::var("RTDB_SYNC_HEAVY").ok().as_deref() != Some("1") {
return;
}
let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
let backend = Arc::new(
RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb"),
);
let mut handles = Vec::new();
for path in 0..64 {
let root = test_root(&format!("heavy/{path}"));
backend
.put(&root, serde_json::json!({"path": path, "generation": 0}))
.await
.unwrap();
let stream_backend = Arc::new(
RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb"),
);
let handle = start(
stream_backend,
root,
Config {
retry: RetryPolicy::Never,
..Config::default()
},
);
handles.push((path, handle));
}
for (_path, handle) in &handles {
wait_for_status(handle, SyncStatus::Connected).await;
}
for generation in 1..=100 {
let writes = (0..64).map(|path| {
let backend = backend.clone();
async move {
let root = test_root(&format!("heavy/{path}"));
if generation % 3 == 1 {
backend
.patch(
&root,
serde_json::json!({"generation": generation})
.as_object()
.unwrap()
.clone(),
)
.await
} else if generation % 3 == 2 {
backend.put(&root, Value::Null).await?;
backend
.put(
&root,
serde_json::json!({"path": path, "generation": generation}),
)
.await
} else {
backend
.put(
&root,
serde_json::json!({"path": path, "generation": generation}),
)
.await
}
}
});
for result in futures_util::future::join_all(writes).await {
result.unwrap();
}
}
for (path, handle) in &handles {
let mut updates = handle.subscribe();
tokio::time::timeout(Duration::from_secs(30), async {
while updates.borrow().value["generation"] != 100 {
updates.changed().await.unwrap();
}
})
.await
.unwrap();
assert_eq!(updates.borrow().value["path"], *path);
}
for (_, handle) in handles {
handle.shutdown().await;
}
}
#[tokio::test]
async fn emulator_namespaces_are_isolated_when_configured() {
let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
return;
};
let left =
RtdbBackend::new(format!("http://{host}"), "").with_namespace("rtdb-sync-ns-left");
let right =
RtdbBackend::new(format!("http://{host}"), "").with_namespace("rtdb-sync-ns-right");
left.put("isolation", serde_json::json!({"owner": "left"}))
.await
.unwrap();
right
.put("isolation", serde_json::json!({"owner": "right"}))
.await
.unwrap();
assert_eq!(left.get("isolation").await.unwrap()["owner"], "left");
assert_eq!(right.get("isolation").await.unwrap()["owner"], "right");
}
#[tokio::test]
#[ignore = "run through scripts/test-emulator-recovery.sh"]
async fn emulator_restart_rehydrates_and_recovers() {
let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
let ready = std::env::var("RTDB_RECOVERY_READY").expect("ready marker");
let restored = std::env::var("RTDB_RECOVERY_RESTORED").expect("restored marker");
let backend = Arc::new(
RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb"),
);
let mut handles = Vec::new();
let path_count = std::env::var("RTDB_RECOVERY_PATHS")
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(32);
for path in 0..path_count {
let root = test_root(&format!("restart/{path}"));
emulator_put_retry(
&backend,
&root,
serde_json::json!({"path": path, "generation": 1}),
)
.await
.unwrap();
let stream_backend = Arc::new(
RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb"),
);
handles.push((
path,
start(
stream_backend,
root,
Config {
retry: RetryPolicy::Exponential {
max_attempts: Some(20),
base: Duration::from_millis(100),
max: Duration::from_secs(1),
},
..Config::default()
},
),
));
}
for (_, handle) in &handles {
wait_for_status(handle, SyncStatus::Connected).await;
}
std::fs::write(&ready, "connected").unwrap();
while !std::path::Path::new(&restored).exists() {
tokio::time::sleep(Duration::from_millis(50)).await;
}
tokio::time::timeout(Duration::from_secs(30), async {
loop {
if handles
.iter()
.all(|(_, handle)| handle.metrics().reconnect_attempts >= 1)
{
break;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap();
for path in 0..path_count {
emulator_put_retry(
&backend,
&test_root(&format!("restart/{path}")),
serde_json::json!({"path": path, "generation": 2}),
)
.await
.unwrap();
}
for (path, handle) in &handles {
let mut updates = handle.subscribe();
tokio::time::timeout(Duration::from_secs(30), async {
while updates.borrow().value["generation"] != 2 {
updates.changed().await.unwrap();
}
})
.await
.unwrap();
assert_eq!(updates.borrow().value["path"], *path);
assert!(handle.metrics().reconnect_attempts >= 1);
}
for (_, handle) in handles {
handle.shutdown().await;
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn fanout_64_subscribers_converge() {
let events = (1..=250)
.map(|generation| Event::Put {
path: "".into(),
data: serde_json::json!({"generation": generation}),
})
.collect();
let handle = start(
Arc::new(mock(serde_json::json!({"generation": 0}), events)),
"fanout",
Config {
retry: RetryPolicy::Never,
..Config::default()
},
);
let mut subscribers = (0..64).map(|_| handle.subscribe()).collect::<Vec<_>>();
for receiver in &mut subscribers {
tokio::time::timeout(Duration::from_secs(2), async {
while receiver.borrow().value["generation"] != 250 {
receiver.changed().await.unwrap();
}
})
.await
.unwrap();
assert_eq!(receiver.borrow().value["generation"], 250);
}
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn lifecycle_churn_stops_one_hundred_tasks() {
for _ in 0..100 {
let handle = start(
Arc::new(mock(Value::Null, vec![])),
"churn",
Config {
retry: RetryPolicy::Never,
..Config::default()
},
);
handle.shutdown().await;
}
}
#[tokio::test]
async fn reconnect_status_metrics_and_backend_hook_are_observable() {
let backend = Arc::new(Flaky {
subscriptions: AtomicU64::new(0),
replacements: AtomicU64::new(0),
});
let handle = start(
backend.clone(),
"flaky",
Config {
retry: RetryPolicy::Exponential {
max_attempts: Some(3),
base: Duration::from_millis(1),
max: Duration::from_millis(5),
},
jitter_max: Duration::from_millis(1),
..Config::default()
},
);
wait_for_status(&handle, SyncStatus::Connected).await;
assert_eq!(backend.replacements.load(Ordering::Relaxed), 1);
assert_eq!(handle.metrics().stream_failures, 1);
assert_eq!(handle.metrics().reconnect_attempts, 1);
handle.shutdown().await;
}
#[tokio::test]
async fn twenty_five_reconnect_cycles_preserve_state_and_subscribers() {
let backend = Arc::new(Cycling {
subscriptions: AtomicU64::new(0),
});
let handle = start(
backend,
"cycles",
Config {
retry: RetryPolicy::Exponential {
max_attempts: Some(30),
base: Duration::from_millis(1),
max: Duration::from_millis(2),
},
..Config::default()
},
);
wait_for_status(&handle, SyncStatus::Connected).await;
assert_eq!(handle.metrics().stream_failures, 25);
assert_eq!(handle.metrics().reconnect_attempts, 25);
handle.shutdown().await;
}
#[tokio::test]
async fn shutdown_cancels_hydration_backoff_promptly() {
let handle = start(
Arc::new(Failing),
"backoff",
Config {
retry: RetryPolicy::Exponential {
max_attempts: None,
base: Duration::from_secs(30),
max: Duration::from_secs(30),
},
..Config::default()
},
);
tokio::time::timeout(Duration::from_millis(100), handle.shutdown())
.await
.unwrap();
}
#[tokio::test]
async fn optimistic_write_rolls_back_on_backend_failure() {
let handle = start(
Arc::new(Failing),
"failing",
Config {
write_policy: WritePolicy::Optimistic,
retry: RetryPolicy::Never,
..Config::default()
},
);
wait_for_status(&handle, SyncStatus::Connected).await;
assert_eq!(
handle.put("", serde_json::json!({"count": 2})).await,
Err(SyncError::Backend("injected write failure".into()))
);
assert_eq!(handle.snapshot().value, serde_json::json!({"count": 1}));
assert_eq!(handle.metrics().failed_writes, 1);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn concurrent_local_and_remote_writers_converge_without_echo_loop() {
let (events, _) = tokio::sync::broadcast::channel(256);
let backend = Arc::new(Live {
value: Arc::new(Mutex::new(serde_json::json!({"count": 0}))),
events,
});
let handle = start(
backend.clone(),
"writers",
Config {
write_policy: WritePolicy::Optimistic,
conflict_policy: ConflictPolicy::RemoteWins,
retry: RetryPolicy::Never,
..Config::default()
},
);
wait_for_status(&handle, SyncStatus::Connected).await;
for count in 1..=100u64 {
let local = handle.put("", serde_json::json!({"count": count}));
let remote = backend.put("", serde_json::json!({"count": count + 10_000}));
let (local_result, remote_result) = tokio::join!(local, remote);
local_result.unwrap();
remote_result.unwrap();
}
tokio::time::sleep(Duration::from_millis(25)).await;
assert_eq!(
handle.snapshot().value,
backend.get("writers").await.unwrap()
);
assert_eq!(handle.metrics().successful_writes, 100);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn bidirectional_32_path_conflict_profile_converges() {
let mut pairs = Vec::new();
for path in 0..32u64 {
let (events, _) = tokio::sync::broadcast::channel(512);
let backend = Arc::new(Live {
value: Arc::new(Mutex::new(
serde_json::json!({"path": path, "local": 0, "remote": 0}),
)),
events,
});
let handle = start(
backend.clone(),
format!("bidirectional/{path}"),
Config {
write_policy: WritePolicy::Optimistic,
conflict_policy: ConflictPolicy::RemoteWins,
retry: RetryPolicy::Never,
..Config::default()
},
);
pairs.push((path, backend, handle));
}
for (_, _, handle) in &pairs {
wait_for_status(handle, SyncStatus::Connected).await;
}
for generation in 1..=100u64 {
let writes = pairs.iter().map(|(_, backend, handle)| async move {
let local = handle.put("", serde_json::json!({"local": generation}));
let remote = backend.patch(
"",
serde_json::json!({"remote": generation})
.as_object()
.unwrap()
.clone(),
);
let (local_result, remote_result) = tokio::join!(local, remote);
local_result?;
remote_result
});
for result in futures_util::future::join_all(writes).await {
result.unwrap();
}
}
tokio::time::sleep(Duration::from_millis(50)).await;
for (_, backend, handle) in &pairs {
assert_eq!(handle.snapshot().value, backend.get("").await.unwrap());
assert_eq!(handle.metrics().successful_writes, 100);
}
for (_, _, handle) in pairs {
handle.shutdown().await;
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn fan_in_eight_local_producers_preserves_acknowledgements() {
let (events, _) = tokio::sync::broadcast::channel(512);
let backend = Arc::new(Live {
value: Arc::new(Mutex::new(serde_json::json!({"remote": 0}))),
events,
});
let handle = start(
backend.clone(),
"fan-in",
Config {
write_policy: WritePolicy::Optimistic,
retry: RetryPolicy::Never,
..Config::default()
},
);
wait_for_status(&handle, SyncStatus::Connected).await;
let producers = (0..8).map(|producer| {
let handle = &handle;
async move {
for generation in 0..25u64 {
handle
.patch(
"",
serde_json::json!({ (format!("local-{producer}")): generation }),
)
.await
.unwrap();
}
}
});
let _ = futures_util::future::join_all(producers).await;
backend
.patch(
"",
serde_json::json!({"remote": 1})
.as_object()
.unwrap()
.clone(),
)
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(25)).await;
assert_eq!(handle.metrics().successful_writes, 200);
assert_eq!(handle.snapshot().value, backend.get("").await.unwrap());
handle.shutdown().await;
}
async fn wait_for_status(handle: &SyncHandle, expected: SyncStatus) {
let mut status = handle.subscribe_status();
tokio::time::timeout(Duration::from_secs(5), async {
while status.borrow().clone() != expected {
status.changed().await.unwrap();
}
})
.await
.unwrap();
}
fn test_root(label: &str) -> String {
format!("rtdb-sync-tests/{}/{label}", std::process::id())
}
async fn emulator_put_retry(
backend: &RtdbBackend,
path: &str,
value: Value,
) -> Result<(), SyncError> {
let mut last = None;
for _ in 0..300 {
match backend.put(path, value.clone()).await {
Ok(()) => return Ok(()),
Err(error) => {
last = Some(error);
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
}
Err(last.unwrap_or_else(|| SyncError::Backend("emulator did not accept write".into())))
}
#[test]
fn file_persistence_is_versioned_atomic_and_namespace_isolated() {
let root =
std::env::temp_dir().join(format!("rtdb-sync-persistence-{}", std::process::id()));
let _ = fs::remove_dir_all(&root);
let store = FilePersistence::new(&root).unwrap();
let key = "user-a/path";
store
.store_snapshot(&DurableSnapshot {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: key.into(),
generation: 7,
value: serde_json::json!({"ok": true}),
saved_at_ms: now_ms(),
})
.unwrap();
store
.append_mutation(&DurableMutation {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: key.into(),
id: 11,
path: "child".into(),
kind: DurableMutationKind::Patch,
payload: serde_json::json!({"count": 1}),
generation: 8,
created_at_ms: now_ms(),
})
.unwrap();
assert_eq!(store.load(key).unwrap().pending.len(), 1);
assert!(store.load("other/path").unwrap().pending.is_empty());
store.acknowledge_mutation(key, 11).unwrap();
store.compact(key).unwrap();
assert!(store.load(key).unwrap().pending.is_empty());
let snapshot_path = store.path(key, "snapshot.json").unwrap();
fs::write(snapshot_path, b"not-json").unwrap();
assert!(matches!(store.load(key), Err(SyncError::Persistence(_))));
store.clear(key).unwrap();
let _ = fs::remove_dir_all(&root);
}
#[tokio::test]
async fn durable_queue_restores_offline_state_and_replays_after_restart() {
let store = Arc::new(MemoryPersistence::default());
let online = Arc::new(std::sync::atomic::AtomicBool::new(false));
let (events, _) = tokio::sync::broadcast::channel(16);
let backend = Arc::new(Recoverable {
online: online.clone(),
value: Arc::new(Mutex::new(serde_json::json!({"count": 0}))),
events,
});
let config = Config {
persistence: Some(store.clone()),
persistence_key: Some("durable-test".into()),
offline_policy: OfflinePolicy::QueueWithLimit { max_pending: 2 },
write_policy: WritePolicy::Optimistic,
retry: RetryPolicy::Exponential {
max_attempts: None,
base: Duration::from_millis(5),
max: Duration::from_millis(20),
},
..Config::default()
};
let first = start(backend.clone(), "root", config.clone());
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if first.status() == SyncStatus::Offline {
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
first
.put("", serde_json::json!({"count": 1}))
.await
.unwrap();
assert_eq!(first.metrics().pending_mutations, 1);
first.shutdown().await;
assert_eq!(store.load("durable-test").unwrap().pending.len(), 1);
online.store(true, Ordering::Relaxed);
let second = start(backend.clone(), "root", config);
tokio::time::timeout(Duration::from_secs(2), async {
loop {
if backend.get("root").await.unwrap()["count"] == 1
&& second.metrics().pending_mutations == 0
&& second.status() == SyncStatus::Connected
{
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.unwrap();
assert_eq!(store.load("durable-test").unwrap().pending.len(), 0);
second.shutdown().await;
}
#[tokio::test]
async fn offline_queue_limit_is_visible_and_missing_store_is_rejected() {
let backend = Arc::new(Failing);
let no_store = start(
backend.clone(),
"root",
Config {
offline_policy: OfflinePolicy::QueueWhileOffline,
..Config::default()
},
);
tokio::time::timeout(Duration::from_secs(1), async {
loop {
if matches!(
no_store.status(),
SyncStatus::Failed(SyncError::Persistence(_))
) {
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
no_store.shutdown().await;
let store = Arc::new(MemoryPersistence::default());
let online = Arc::new(std::sync::atomic::AtomicBool::new(false));
let (events, _) = tokio::sync::broadcast::channel(4);
let backend = Arc::new(Recoverable {
online,
value: Arc::new(Mutex::new(Value::Null)),
events,
});
let handle = start(
backend,
"root",
Config {
persistence: Some(store),
persistence_key: Some("limit".into()),
offline_policy: OfflinePolicy::QueueWithLimit { max_pending: 1 },
write_policy: WritePolicy::Optimistic,
retry: RetryPolicy::Exponential {
max_attempts: None,
base: Duration::from_millis(5),
max: Duration::from_millis(20),
},
..Config::default()
},
);
tokio::time::timeout(Duration::from_secs(1), async {
while handle.status() != SyncStatus::Offline {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
handle.put("a", 1).await.unwrap();
assert!(matches!(
handle.put("b", 2).await,
Err(SyncError::Persistence(_))
));
handle.shutdown().await;
}
#[tokio::test]
async fn restored_pending_mutation_uses_explicit_remote_wins_policy() {
let store = Arc::new(MemoryPersistence::default());
store
.store_snapshot(&DurableSnapshot {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: "conflict".into(),
generation: 1,
value: serde_json::json!({"count": 1}),
saved_at_ms: now_ms(),
})
.unwrap();
store
.append_mutation(&DurableMutation {
format_version: PERSISTENCE_FORMAT_VERSION,
sync_key: "conflict".into(),
id: 1,
path: "".into(),
kind: DurableMutationKind::Put,
payload: serde_json::json!({"count": 2}),
generation: 2,
created_at_ms: now_ms(),
})
.unwrap();
let (events, _) = tokio::sync::broadcast::channel(4);
let backend = Arc::new(Recoverable {
online: Arc::new(std::sync::atomic::AtomicBool::new(true)),
value: Arc::new(Mutex::new(serde_json::json!({"count": 9}))),
events,
});
let handle = start(
backend.clone(),
"root",
Config {
persistence: Some(store.clone()),
persistence_key: Some("conflict".into()),
conflict_policy: ConflictPolicy::RemoteWins,
..Config::default()
},
);
wait_for_status(&handle, SyncStatus::Connected).await;
assert_eq!(backend.get("root").await.unwrap()["count"], 9);
assert_eq!(handle.metrics().conflict_count, 1);
assert_eq!(handle.metrics().pending_mutations, 0);
assert!(store.load("conflict").unwrap().pending.is_empty());
handle.shutdown().await;
}
fn durable_emulator_config(store: Arc<FilePersistence>) -> Config {
Config {
persistence: Some(store),
persistence_key: Some("durable-emulator-acceptance".into()),
offline_policy: OfflinePolicy::QueueWhileOffline,
write_policy: WritePolicy::Optimistic,
retry: RetryPolicy::Exponential {
max_attempts: None,
base: Duration::from_millis(10),
max: Duration::from_millis(50),
},
..Config::default()
}
}
fn durable_emulator_store() -> Arc<FilePersistence> {
Arc::new(
FilePersistence::new(std::env::var("RTDB_DURABLE_STORE").expect("RTDB_DURABLE_STORE"))
.unwrap(),
)
}
#[tokio::test]
#[ignore = "run through scripts/test-emulator-offline.sh"]
async fn durable_emulator_seed_persists_snapshot() {
let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
let store = durable_emulator_store();
let backend = Arc::new(
RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb"),
);
emulator_put_retry(
&backend,
"durable-acceptance",
serde_json::json!({"count": 1}),
)
.await
.unwrap();
let handle = start(
backend,
"durable-acceptance",
durable_emulator_config(store.clone()),
);
wait_for_status(&handle, SyncStatus::Connected).await;
handle.shutdown().await;
assert!(store
.load("durable-emulator-acceptance")
.unwrap()
.snapshot
.is_some());
}
#[tokio::test]
#[ignore = "run through scripts/test-emulator-offline.sh"]
async fn durable_emulator_queues_while_database_is_down() {
let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
let store = durable_emulator_store();
let backend = Arc::new(
RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb"),
);
let handle = start(
backend,
"durable-acceptance",
durable_emulator_config(store.clone()),
);
tokio::time::timeout(Duration::from_secs(5), async {
loop {
if handle.status() == SyncStatus::Offline {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.unwrap();
handle
.put("", serde_json::json!({"count": 2}))
.await
.unwrap();
if std::env::var_os("RTDB_DURABLE_CRASH_AFTER_QUEUE").is_some() {
std::process::exit(0);
}
handle.shutdown().await;
assert_eq!(
store
.load("durable-emulator-acceptance")
.unwrap()
.pending
.len(),
1
);
}
#[tokio::test]
#[ignore = "run through scripts/test-emulator-offline.sh"]
async fn durable_emulator_replays_after_database_returns() {
let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
let store = durable_emulator_store();
let backend = Arc::new(
RtdbBackend::new(format!("http://{host}"), "")
.with_namespace("demo-rtdb-sync-default-rtdb"),
);
let handle = start(
backend.clone(),
"durable-acceptance",
durable_emulator_config(store.clone()),
);
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if backend.get("durable-acceptance").await.unwrap()["count"] == 2
&& handle.metrics().pending_mutations == 0
&& handle.metrics().replay_successes == 1
{
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.unwrap();
wait_for_status(&handle, SyncStatus::Connected).await;
assert!(store
.load("durable-emulator-acceptance")
.unwrap()
.pending
.is_empty());
handle.shutdown().await;
}
#[test]
fn retry_policy_is_bounded_and_paths_are_safe() {
assert!(segments("a/../b").is_err());
assert!(matches!(RetryPolicy::Never, RetryPolicy::Never));
let policy = RetryPolicy::Exponential {
max_attempts: Some(3),
base: Duration::from_millis(10),
max: Duration::from_millis(40),
};
assert_eq!(
policy.delay(1, Duration::ZERO),
Some(Duration::from_millis(10))
);
assert!(policy.delay(3, Duration::from_millis(5)).unwrap() <= Duration::from_millis(45));
assert_eq!(policy.delay(4, Duration::ZERO), None);
let mut value = Value::Null;
assert!(apply_event(
"root",
&mut value,
Event::Put {
path: "a/../b".into(),
data: Value::Null
}
)
.is_err());
}
#[test]
fn conflict_policy_is_explicit_and_echoes_are_suppressed() {
let (tx, rx) = watch::channel(Snapshot {
generation: 0,
value: serde_json::json!({"count": 2}),
});
let mut state = serde_json::json!({"count": 2});
let mut pending = vec![PendingMutation {
path: "".into(),
event: Event::Put {
path: "".into(),
data: serde_json::json!({"count": 2}),
},
}];
let mut generation = 0;
reconcile_event(
"root",
&mut state,
&mut generation,
&tx,
&mut pending,
Event::Put {
path: "".into(),
data: serde_json::json!({"count": 2}),
},
ConflictPolicy::Reject,
)
.unwrap();
assert_eq!(generation, 0);
pending.push(PendingMutation {
path: "".into(),
event: Event::Put {
path: "".into(),
data: serde_json::json!({"count": 2}),
},
});
reconcile_event(
"root",
&mut state,
&mut generation,
&tx,
&mut pending,
Event::Put {
path: "".into(),
data: serde_json::json!({"count": 3}),
},
ConflictPolicy::LocalWins,
)
.unwrap();
assert_eq!(state["count"], 2);
assert!(reconcile_event(
"root",
&mut state,
&mut generation,
&tx,
&mut pending,
Event::Put {
path: "".into(),
data: serde_json::json!({"count": 4})
},
ConflictPolicy::Reject
)
.is_err());
drop(rx);
}
}