1use async_trait::async_trait;
8use futures_core::Stream;
9use futures_util::StreamExt;
10use rtdb_rs::{RtdbClient, RtdbEvent};
11use serde::{de::DeserializeOwned, Serialize};
12use serde_json::{Map, Value};
13use std::{
14 fmt,
15 fs::{self, File, OpenOptions},
16 hash::{Hash, Hasher},
17 io::{BufRead, BufReader, Write as IoWrite},
18 path::{Path, PathBuf},
19 pin::Pin,
20 sync::{
21 atomic::{AtomicU64, Ordering},
22 Arc, Mutex,
23 },
24 time::{Duration, SystemTime, UNIX_EPOCH},
25};
26use thiserror::Error;
27use tokio::sync::{mpsc, watch};
28use tokio_util::sync::CancellationToken;
29
30pub type EventStream = Pin<Box<dyn Stream<Item = Result<Event, SyncError>> + Send>>;
31
32#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
33pub enum Event {
34 Put {
35 path: String,
36 data: Value,
37 },
38 Patch {
39 path: String,
40 data: Map<String, Value>,
41 },
42}
43
44#[derive(Debug, Error, Clone, PartialEq, Eq)]
45pub enum SyncError {
46 #[error("backend: {0}")]
47 Backend(String),
48 #[error("invalid event path: {0}")]
49 InvalidPath(String),
50 #[error("conversion failed: {0}")]
51 Conversion(String),
52 #[error("synchronization cancelled")]
53 Cancelled,
54 #[error("conflict: {0}")]
55 Conflict(String),
56 #[error("persistence: {0}")]
57 Persistence(String),
58}
59
60pub const PERSISTENCE_FORMAT_VERSION: u32 = 1;
61
62#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
63pub struct DurableSnapshot {
64 pub format_version: u32,
65 pub sync_key: String,
66 pub generation: u64,
67 pub value: Value,
68 pub saved_at_ms: u64,
69}
70
71#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
72pub enum DurableMutationKind {
73 Put,
74 Patch,
75}
76
77#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
78pub struct DurableMutation {
79 pub format_version: u32,
80 pub sync_key: String,
81 pub id: u64,
82 pub path: String,
83 pub kind: DurableMutationKind,
84 pub payload: Value,
85 pub generation: u64,
86 pub created_at_ms: u64,
87}
88
89#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)]
90pub struct SyncMetadata {
91 pub format_version: u32,
92 pub sync_key: String,
93 pub last_remote_event_at_ms: Option<u64>,
94 pub last_successful_sync_at_ms: Option<u64>,
95}
96
97#[derive(Debug, Clone, Default, PartialEq)]
98pub struct PersistedState {
99 pub snapshot: Option<DurableSnapshot>,
100 pub pending: Vec<DurableMutation>,
101 pub metadata: Option<SyncMetadata>,
102}
103
104pub trait PersistenceBackend: Send + Sync + fmt::Debug {
105 fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError>;
106 fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError>;
107 fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError>;
108 fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError>;
109 fn compact(&self, sync_key: &str) -> Result<(), SyncError>;
110 fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError>;
111 fn clear(&self, sync_key: &str) -> Result<(), SyncError>;
112}
113
114#[derive(Debug, Default, Clone)]
115pub struct MemoryPersistence {
116 entries: Arc<Mutex<std::collections::HashMap<String, PersistedState>>>,
117}
118
119impl PersistenceBackend for MemoryPersistence {
120 fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError> {
121 Ok(self
122 .entries
123 .lock()
124 .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?
125 .get(sync_key)
126 .cloned()
127 .unwrap_or_default())
128 }
129 fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError> {
130 let mut entries = self
131 .entries
132 .lock()
133 .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
134 let entry = entries.entry(snapshot.sync_key.clone()).or_default();
135 entry.snapshot = Some(snapshot.clone());
136 Ok(())
137 }
138 fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError> {
139 let mut entries = self
140 .entries
141 .lock()
142 .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
143 entries
144 .entry(mutation.sync_key.clone())
145 .or_default()
146 .pending
147 .push(mutation.clone());
148 Ok(())
149 }
150 fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError> {
151 let mut entries = self
152 .entries
153 .lock()
154 .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
155 if let Some(entry) = entries.get_mut(sync_key) {
156 entry.pending.retain(|mutation| mutation.id != mutation_id);
157 }
158 Ok(())
159 }
160 fn compact(&self, _sync_key: &str) -> Result<(), SyncError> {
161 Ok(())
162 }
163 fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError> {
164 let mut entries = self
165 .entries
166 .lock()
167 .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
168 entries
169 .entry(metadata.sync_key.clone())
170 .or_default()
171 .metadata = Some(metadata.clone());
172 Ok(())
173 }
174 fn clear(&self, sync_key: &str) -> Result<(), SyncError> {
175 self.entries
176 .lock()
177 .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?
178 .remove(sync_key);
179 Ok(())
180 }
181}
182
183#[derive(Debug, Clone)]
184pub struct FilePersistence {
185 root: Arc<PathBuf>,
186}
187
188impl FilePersistence {
189 pub fn new(root: impl Into<PathBuf>) -> Result<Self, SyncError> {
190 let root = root.into();
191 fs::create_dir_all(&root).map_err(|error| persistence_io(&root, error))?;
192 restrict_permissions(&root, 0o700)?;
193 Ok(Self {
194 root: Arc::new(root),
195 })
196 }
197
198 fn directory(&self, sync_key: &str) -> Result<PathBuf, SyncError> {
199 let mut hasher = std::collections::hash_map::DefaultHasher::new();
200 sync_key.hash(&mut hasher);
201 let directory = self.root.join(format!("{:016x}", hasher.finish()));
202 fs::create_dir_all(&directory).map_err(|error| persistence_io(&directory, error))?;
203 restrict_permissions(&directory, 0o700)?;
204 Ok(directory)
205 }
206 fn path(&self, sync_key: &str, name: &str) -> Result<PathBuf, SyncError> {
207 Ok(self.directory(sync_key)?.join(name))
208 }
209}
210
211#[derive(Debug, serde::Serialize, serde::Deserialize)]
212enum JournalEntry {
213 Append(DurableMutation),
214 Acknowledge { sync_key: String, mutation_id: u64 },
215}
216
217impl PersistenceBackend for FilePersistence {
218 fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError> {
219 let snapshot_path = self.path(sync_key, "snapshot.json")?;
220 let metadata_path = self.path(sync_key, "metadata.json")?;
221 let journal_path = self.path(sync_key, "journal.jsonl")?;
222 let snapshot = if snapshot_path.exists() {
223 let bytes =
224 fs::read(&snapshot_path).map_err(|error| persistence_io(&snapshot_path, error))?;
225 let value: DurableSnapshot = serde_json::from_slice(&bytes)
226 .map_err(|error| SyncError::Persistence(format!("invalid snapshot: {error}")))?;
227 validate_key_and_version(&value.format_version, &value.sync_key, sync_key)?;
228 Some(value)
229 } else {
230 None
231 };
232 let metadata = if metadata_path.exists() {
233 let bytes =
234 fs::read(&metadata_path).map_err(|error| persistence_io(&metadata_path, error))?;
235 let value: SyncMetadata = serde_json::from_slice(&bytes)
236 .map_err(|error| SyncError::Persistence(format!("invalid metadata: {error}")))?;
237 validate_key_and_version(&value.format_version, &value.sync_key, sync_key)?;
238 Some(value)
239 } else {
240 None
241 };
242 let mut pending = std::collections::BTreeMap::new();
243 if journal_path.exists() {
244 for line in BufReader::new(
245 File::open(&journal_path).map_err(|error| persistence_io(&journal_path, error))?,
246 )
247 .lines()
248 {
249 let line = line.map_err(|error| persistence_io(&journal_path, error))?;
250 let entry: JournalEntry = serde_json::from_str(&line).map_err(|error| {
251 SyncError::Persistence(format!("invalid journal entry: {error}"))
252 })?;
253 match entry {
254 JournalEntry::Append(mutation) => {
255 validate_key_and_version(
256 &mutation.format_version,
257 &mutation.sync_key,
258 sync_key,
259 )?;
260 pending.insert(mutation.id, mutation);
261 }
262 JournalEntry::Acknowledge {
263 sync_key: entry_key,
264 mutation_id,
265 } => {
266 if entry_key != sync_key {
267 return Err(SyncError::Persistence(
268 "journal namespace mismatch".into(),
269 ));
270 }
271 pending.remove(&mutation_id);
272 }
273 }
274 }
275 }
276 Ok(PersistedState {
277 snapshot,
278 pending: pending.into_values().collect(),
279 metadata,
280 })
281 }
282 fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError> {
283 validate_key_and_version(
284 &snapshot.format_version,
285 &snapshot.sync_key,
286 &snapshot.sync_key,
287 )?;
288 let path = self.path(&snapshot.sync_key, "snapshot.json")?;
289 let tmp = path.with_extension("json.tmp");
290 let bytes = serde_json::to_vec(snapshot)
291 .map_err(|error| SyncError::Persistence(error.to_string()))?;
292 write_atomic(&path, &tmp, &bytes)
293 }
294 fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError> {
295 validate_key_and_version(
296 &mutation.format_version,
297 &mutation.sync_key,
298 &mutation.sync_key,
299 )?;
300 let path = self.path(&mutation.sync_key, "journal.jsonl")?;
301 append_line(&path, &JournalEntry::Append(mutation.clone()))
302 }
303 fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError> {
304 let path = self.path(sync_key, "journal.jsonl")?;
305 append_line(
306 &path,
307 &JournalEntry::Acknowledge {
308 sync_key: sync_key.into(),
309 mutation_id,
310 },
311 )
312 }
313 fn compact(&self, sync_key: &str) -> Result<(), SyncError> {
314 let state = self.load(sync_key)?;
315 let path = self.path(sync_key, "journal.jsonl")?;
316 let tmp = path.with_extension("jsonl.tmp");
317 let mut bytes = Vec::new();
318 for mutation in state.pending {
319 serde_json::to_writer(&mut bytes, &JournalEntry::Append(mutation))
320 .map_err(|error| SyncError::Persistence(error.to_string()))?;
321 bytes.push(b'\n');
322 }
323 write_atomic(&path, &tmp, &bytes)
324 }
325 fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError> {
326 validate_key_and_version(
327 &metadata.format_version,
328 &metadata.sync_key,
329 &metadata.sync_key,
330 )?;
331 let path = self.path(&metadata.sync_key, "metadata.json")?;
332 let tmp = path.with_extension("json.tmp");
333 let bytes = serde_json::to_vec(metadata)
334 .map_err(|error| SyncError::Persistence(error.to_string()))?;
335 write_atomic(&path, &tmp, &bytes)
336 }
337 fn clear(&self, sync_key: &str) -> Result<(), SyncError> {
338 let directory = self.directory(sync_key)?;
339 if directory.exists() {
340 fs::remove_dir_all(&directory).map_err(|error| persistence_io(&directory, error))?;
341 }
342 Ok(())
343 }
344}
345
346fn now_ms() -> u64 {
347 SystemTime::now()
348 .duration_since(UNIX_EPOCH)
349 .unwrap_or_default()
350 .as_millis() as u64
351}
352
353fn validate_key_and_version(
354 version: &u32,
355 entry_key: &str,
356 expected_key: &str,
357) -> Result<(), SyncError> {
358 if *version != PERSISTENCE_FORMAT_VERSION {
359 return Err(SyncError::Persistence(format!(
360 "unsupported persistence format version {version}"
361 )));
362 }
363 if entry_key != expected_key {
364 return Err(SyncError::Persistence(
365 "persistence namespace mismatch".into(),
366 ));
367 }
368 Ok(())
369}
370
371fn persistence_io(path: &Path, error: std::io::Error) -> SyncError {
372 SyncError::Persistence(format!("{}: {error}", path.display()))
373}
374
375fn restrict_permissions(path: &Path, mode: u32) -> Result<(), SyncError> {
376 #[cfg(unix)]
377 {
378 use std::os::unix::fs::PermissionsExt;
379 fs::set_permissions(path, fs::Permissions::from_mode(mode))
380 .map_err(|error| persistence_io(path, error))?;
381 }
382 Ok(())
383}
384
385fn write_atomic(path: &Path, tmp: &Path, bytes: &[u8]) -> Result<(), SyncError> {
386 let mut file = OpenOptions::new()
387 .create(true)
388 .truncate(true)
389 .write(true)
390 .open(tmp)
391 .map_err(|error| persistence_io(tmp, error))?;
392 file.write_all(bytes)
393 .map_err(|error| persistence_io(tmp, error))?;
394 file.sync_all()
395 .map_err(|error| persistence_io(tmp, error))?;
396 restrict_permissions(tmp, 0o600)?;
397 fs::rename(tmp, path).map_err(|error| persistence_io(path, error))?;
398 if let Some(parent) = path.parent() {
399 if let Ok(directory) = File::open(parent) {
400 let _ = directory.sync_all();
401 }
402 }
403 Ok(())
404}
405
406fn append_line<T: serde::Serialize>(path: &Path, entry: &T) -> Result<(), SyncError> {
407 let mut file = OpenOptions::new()
408 .create(true)
409 .append(true)
410 .open(path)
411 .map_err(|error| persistence_io(path, error))?;
412 serde_json::to_writer(&mut file, entry)
413 .map_err(|error| SyncError::Persistence(error.to_string()))?;
414 file.write_all(b"\n")
415 .map_err(|error| persistence_io(path, error))?;
416 file.sync_all()
417 .map_err(|error| persistence_io(path, error))?;
418 restrict_permissions(path, 0o600)
419}
420
421async fn persistence_load(
422 store: Arc<dyn PersistenceBackend>,
423 sync_key: String,
424) -> Result<PersistedState, SyncError> {
425 tokio::task::spawn_blocking(move || store.load(&sync_key))
426 .await
427 .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
428}
429
430async fn persistence_snapshot(
431 store: Arc<dyn PersistenceBackend>,
432 snapshot: DurableSnapshot,
433) -> Result<(), SyncError> {
434 tokio::task::spawn_blocking(move || store.store_snapshot(&snapshot))
435 .await
436 .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
437}
438
439async fn persistence_append(
440 store: Arc<dyn PersistenceBackend>,
441 mutation: DurableMutation,
442) -> Result<(), SyncError> {
443 tokio::task::spawn_blocking(move || store.append_mutation(&mutation))
444 .await
445 .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
446}
447
448async fn persistence_ack(
449 store: Arc<dyn PersistenceBackend>,
450 sync_key: String,
451 mutation_id: u64,
452) -> Result<(), SyncError> {
453 tokio::task::spawn_blocking(move || store.acknowledge_mutation(&sync_key, mutation_id))
454 .await
455 .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
456}
457
458async fn persistence_compact(
459 store: Arc<dyn PersistenceBackend>,
460 sync_key: String,
461) -> Result<(), SyncError> {
462 tokio::task::spawn_blocking(move || store.compact(&sync_key))
463 .await
464 .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
465}
466
467async fn persistence_metadata(
468 store: Arc<dyn PersistenceBackend>,
469 metadata: SyncMetadata,
470) -> Result<(), SyncError> {
471 tokio::task::spawn_blocking(move || store.store_metadata(&metadata))
472 .await
473 .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
474}
475
476#[async_trait]
477pub trait Backend: Send + Sync + 'static {
478 async fn get(&self, path: &str) -> Result<Value, SyncError>;
479 async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError>;
480 async fn put(&self, path: &str, value: Value) -> Result<(), SyncError>;
481 async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError>;
482 async fn before_reconnect(&self, _attempt: usize) -> Result<(), SyncError> {
483 Ok(())
484 }
485}
486
487#[derive(Debug, Clone, PartialEq)]
488pub struct Snapshot {
489 pub generation: u64,
490 pub value: Value,
491}
492
493impl Snapshot {
494 pub fn decode<T: DeserializeOwned>(&self) -> Result<T, SyncError> {
495 serde_json::from_value(self.value.clone()).map_err(|e| SyncError::Conversion(e.to_string()))
496 }
497}
498
499#[derive(Debug, Clone, PartialEq)]
500pub struct TypedSnapshot<T> {
501 pub generation: u64,
502 pub value: T,
503}
504
505pub struct TypedSyncHandle<T> {
506 inner: SyncHandle,
507 snapshot: watch::Receiver<Result<TypedSnapshot<T>, SyncError>>,
508 conversion_task: tokio::task::JoinHandle<()>,
509}
510
511impl<T: Clone> TypedSyncHandle<T> {
512 pub fn snapshot(&self) -> Result<TypedSnapshot<T>, SyncError> {
513 self.snapshot.borrow().clone()
514 }
515 pub fn subscribe(&self) -> watch::Receiver<Result<TypedSnapshot<T>, SyncError>> {
516 self.snapshot.clone()
517 }
518 pub fn status(&self) -> SyncStatus {
519 self.inner.status()
520 }
521 pub fn subscribe_status(&self) -> watch::Receiver<SyncStatus> {
522 self.inner.subscribe_status()
523 }
524 pub fn metrics(&self) -> MetricsSnapshot {
525 self.inner.metrics()
526 }
527 pub async fn put<V: Serialize>(
528 &self,
529 path: impl Into<String>,
530 value: V,
531 ) -> Result<(), SyncError> {
532 self.inner.put(path, value).await
533 }
534 pub async fn patch<V: Serialize>(
535 &self,
536 path: impl Into<String>,
537 value: V,
538 ) -> Result<(), SyncError> {
539 self.inner.patch(path, value).await
540 }
541 pub async fn shutdown(self) {
542 self.inner.shutdown().await;
543 let _ = self.conversion_task.await;
544 }
545}
546
547pub fn start_typed<B, T>(
548 backend: Arc<B>,
549 path: impl Into<String>,
550 config: Config,
551) -> TypedSyncHandle<T>
552where
553 B: Backend,
554 T: DeserializeOwned + Clone + Send + Sync + 'static,
555{
556 let inner = start(backend, path, config);
557 let mut raw = inner.subscribe();
558 let initial = raw.borrow().decode::<T>().map(|value| TypedSnapshot {
559 generation: raw.borrow().generation,
560 value,
561 });
562 let (snapshot, receiver) = watch::channel(initial);
563 let conversion_task = tokio::spawn(async move {
564 loop {
565 if raw.changed().await.is_err() {
566 break;
567 }
568 let current = raw.borrow().clone();
569 let converted = current.decode::<T>().map(|value| TypedSnapshot {
570 generation: current.generation,
571 value,
572 });
573 let _ = snapshot.send(converted);
574 }
575 });
576 TypedSyncHandle {
577 inner,
578 snapshot: receiver,
579 conversion_task,
580 }
581}
582
583#[derive(Debug, Clone, Copy, PartialEq, Eq)]
584pub enum RetryPolicy {
585 Never,
586 Exponential {
587 max_attempts: Option<usize>,
588 base: Duration,
589 max: Duration,
590 },
591}
592
593impl RetryPolicy {
594 pub fn delay(&self, attempt: usize, jitter_max: Duration) -> Option<Duration> {
595 match *self {
596 RetryPolicy::Never => None,
597 RetryPolicy::Exponential {
598 max_attempts,
599 base,
600 max,
601 } => {
602 if max_attempts.map(|n| attempt > n).unwrap_or(false) {
603 return None;
604 }
605 let factor = 2u32.saturating_pow(attempt.saturating_sub(1) as u32);
606 let backoff = base.saturating_mul(factor).min(max);
607 let jitter = if jitter_max.is_zero() {
608 Duration::ZERO
609 } else {
610 let seed = (attempt as u64)
611 .wrapping_mul(6364136223846793005)
612 .wrapping_add(1442695040888963407);
613 Duration::from_nanos(seed % (jitter_max.as_nanos() as u64 + 1))
614 };
615 Some(
616 backoff
617 .saturating_add(jitter)
618 .min(max.saturating_add(jitter_max)),
619 )
620 }
621 }
622 }
623}
624
625#[derive(Debug, Clone, Copy, PartialEq, Eq)]
626pub enum WritePolicy {
627 Confirmed,
628 Optimistic,
629}
630
631#[derive(Debug, Clone, Copy, PartialEq, Eq)]
632pub enum ConflictPolicy {
633 RemoteWins,
634 LocalWins,
635 Reject,
636}
637
638#[derive(Debug, Clone, Copy, PartialEq, Eq)]
639pub enum OfflinePolicy {
640 RejectWhileOffline,
641 QueueWhileOffline,
642 QueueWithLimit { max_pending: usize },
643}
644
645#[derive(Debug, Clone)]
646pub struct Config {
647 pub retry: RetryPolicy,
648 pub write_policy: WritePolicy,
649 pub jitter_max: Duration,
650 pub conflict_policy: ConflictPolicy,
651 pub write_queue_capacity: usize,
653 pub persistence: Option<Arc<dyn PersistenceBackend>>,
654 pub persistence_key: Option<String>,
655 pub offline_policy: OfflinePolicy,
656}
657impl Default for Config {
658 fn default() -> Self {
659 Self {
660 retry: RetryPolicy::Exponential {
661 max_attempts: None,
662 base: Duration::from_millis(50),
663 max: Duration::from_secs(5),
664 },
665 write_policy: WritePolicy::Confirmed,
666 jitter_max: Duration::ZERO,
667 conflict_policy: ConflictPolicy::RemoteWins,
668 write_queue_capacity: 64,
669 persistence: None,
670 persistence_key: None,
671 offline_policy: OfflinePolicy::RejectWhileOffline,
672 }
673 }
674}
675
676#[derive(Debug, Clone, Default, PartialEq, Eq)]
677pub struct MetricsSnapshot {
678 pub reconnect_attempts: u64,
679 pub stream_failures: u64,
680 pub hydration_failures: u64,
681 pub successful_writes: u64,
682 pub failed_writes: u64,
683 pub pending_mutations: u64,
684 pub persistence_failures: u64,
685 pub replay_attempts: u64,
686 pub replay_successes: u64,
687 pub replay_failures: u64,
688 pub conflict_count: u64,
689}
690
691#[derive(Default)]
692struct Metrics {
693 reconnect_attempts: AtomicU64,
694 stream_failures: AtomicU64,
695 hydration_failures: AtomicU64,
696 successful_writes: AtomicU64,
697 failed_writes: AtomicU64,
698 pending_mutations: AtomicU64,
699 persistence_failures: AtomicU64,
700 replay_attempts: AtomicU64,
701 replay_successes: AtomicU64,
702 replay_failures: AtomicU64,
703 conflict_count: AtomicU64,
704}
705impl Metrics {
706 fn snapshot(&self) -> MetricsSnapshot {
707 MetricsSnapshot {
708 reconnect_attempts: self.reconnect_attempts.load(Ordering::Relaxed),
709 stream_failures: self.stream_failures.load(Ordering::Relaxed),
710 hydration_failures: self.hydration_failures.load(Ordering::Relaxed),
711 successful_writes: self.successful_writes.load(Ordering::Relaxed),
712 failed_writes: self.failed_writes.load(Ordering::Relaxed),
713 pending_mutations: self.pending_mutations.load(Ordering::Relaxed),
714 persistence_failures: self.persistence_failures.load(Ordering::Relaxed),
715 replay_attempts: self.replay_attempts.load(Ordering::Relaxed),
716 replay_successes: self.replay_successes.load(Ordering::Relaxed),
717 replay_failures: self.replay_failures.load(Ordering::Relaxed),
718 conflict_count: self.conflict_count.load(Ordering::Relaxed),
719 }
720 }
721}
722
723pub struct SyncHandle {
724 snapshot: watch::Receiver<Snapshot>,
725 status: watch::Receiver<SyncStatus>,
726 cancel: CancellationToken,
727 writes: mpsc::Sender<Write>,
728 join: tokio::task::JoinHandle<()>,
729 metrics: Arc<Metrics>,
730}
731
732enum Write {
733 Put(
734 String,
735 Value,
736 tokio::sync::oneshot::Sender<Result<(), SyncError>>,
737 ),
738 Patch(
739 String,
740 Map<String, Value>,
741 tokio::sync::oneshot::Sender<Result<(), SyncError>>,
742 ),
743}
744
745struct PendingMutation {
746 path: String,
747 event: Event,
748}
749
750impl SyncHandle {
751 pub fn snapshot(&self) -> Snapshot {
752 self.snapshot.borrow().clone()
753 }
754 pub fn subscribe(&self) -> watch::Receiver<Snapshot> {
755 self.snapshot.clone()
756 }
757 pub fn status(&self) -> SyncStatus {
758 self.status.borrow().clone()
759 }
760 pub fn subscribe_status(&self) -> watch::Receiver<SyncStatus> {
761 self.status.clone()
762 }
763 pub fn metrics(&self) -> MetricsSnapshot {
764 self.metrics.snapshot()
765 }
766 pub async fn put<T: Serialize>(
767 &self,
768 path: impl Into<String>,
769 value: T,
770 ) -> Result<(), SyncError> {
771 let (tx, rx) = tokio::sync::oneshot::channel();
772 let v = serde_json::to_value(value).map_err(|e| SyncError::Conversion(e.to_string()))?;
773 self.writes
774 .send(Write::Put(path.into(), v, tx))
775 .await
776 .map_err(|_| SyncError::Cancelled)?;
777 rx.await.map_err(|_| SyncError::Cancelled)?
778 }
779 pub async fn patch<T: Serialize>(
780 &self,
781 path: impl Into<String>,
782 value: T,
783 ) -> Result<(), SyncError> {
784 let (tx, rx) = tokio::sync::oneshot::channel();
785 let v = serde_json::to_value(value).map_err(|e| SyncError::Conversion(e.to_string()))?;
786 let data = v
787 .as_object()
788 .cloned()
789 .ok_or_else(|| SyncError::Conversion("patch must be an object".into()))?;
790 self.writes
791 .send(Write::Patch(path.into(), data, tx))
792 .await
793 .map_err(|_| SyncError::Cancelled)?;
794 rx.await.map_err(|_| SyncError::Cancelled)?
795 }
796 pub async fn shutdown(self) {
797 self.cancel.cancel();
798 let _ = self.join.await;
799 }
800}
801
802pub fn start<B: Backend>(backend: Arc<B>, path: impl Into<String>, config: Config) -> SyncHandle {
803 let path = path.into();
804 let persistence = config.persistence.clone();
805 let sync_key = config
806 .persistence_key
807 .clone()
808 .unwrap_or_else(|| path.clone());
809 let (snap_tx, snap_rx) = watch::channel(Snapshot {
810 generation: 0,
811 value: Value::Null,
812 });
813 let (status_tx, status_rx) = watch::channel(SyncStatus::Idle);
814 let cancel = CancellationToken::new();
815 let (writes, mut write_rx) = mpsc::channel(config.write_queue_capacity.max(1));
816 let child = cancel.clone();
817 let metrics = Arc::new(Metrics::default());
818 let task_metrics = metrics.clone();
819 let join = tokio::spawn(async move {
820 if matches!(
821 config.offline_policy,
822 OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
823 ) && persistence.is_none()
824 {
825 let _ = status_tx.send(SyncStatus::Failed(SyncError::Persistence(
826 "offline queue requires a persistence backend".into(),
827 )));
828 return;
829 }
830 #[allow(unused_assignments)]
831 let mut state: Option<Value> = None;
832 let mut generation = 0;
833 let mut attempt = 0usize;
834 let mut pending = Vec::new();
835 let mut durable_pending = Vec::new();
836 let mut restored_base = None;
837 if let Some(store) = persistence.as_ref() {
838 match persistence_load(store.clone(), sync_key.clone()).await {
839 Ok(restored) => {
840 if let Some(snapshot) = restored.snapshot {
841 state = Some(snapshot.value);
842 restored_base = state.clone();
843 generation = snapshot.generation;
844 let _ = snap_tx.send(Snapshot {
845 generation,
846 value: state.as_ref().unwrap().clone(),
847 });
848 let _ = status_tx.send(SyncStatus::RestoredStale);
849 }
850 for mutation in restored.pending {
851 match durable_event(&mutation) {
852 Ok(event) => {
853 pending.push(PendingMutation {
854 path: mutation.path.clone(),
855 event,
856 });
857 durable_pending.push(mutation);
858 }
859 Err(error) => {
860 task_metrics
861 .persistence_failures
862 .fetch_add(1, Ordering::Relaxed);
863 let _ = status_tx.send(SyncStatus::Failed(error));
864 return;
865 }
866 }
867 }
868 task_metrics
869 .pending_mutations
870 .store(durable_pending.len() as u64, Ordering::Relaxed);
871 }
872 Err(error) => {
873 task_metrics
874 .persistence_failures
875 .fetch_add(1, Ordering::Relaxed);
876 let _ = status_tx.send(SyncStatus::Failed(error));
877 return;
878 }
879 }
880 }
881 let _ = status_tx.send(SyncStatus::Hydrating);
882 loop {
883 let hydrated = tokio::select! {
884 _ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
885 result = backend.get(&path) => result
886 };
887 match hydrated {
888 Ok(value) => {
889 state = Some(value);
890 if let Some(base) = restored_base.take() {
891 let remote = state.as_ref().expect("hydrated").clone();
892 if let Err(error) = resolve_restored_conflicts(
893 &base,
894 &remote,
895 config.conflict_policy,
896 &mut pending,
897 &mut durable_pending,
898 persistence.as_ref(),
899 &sync_key,
900 &task_metrics,
901 )
902 .await
903 {
904 if matches!(error, SyncError::Conflict(_)) {
905 task_metrics.conflict_count.fetch_add(1, Ordering::Relaxed);
906 let _ = status_tx.send(SyncStatus::Conflict(error));
907 } else {
908 task_metrics
909 .persistence_failures
910 .fetch_add(1, Ordering::Relaxed);
911 let _ = status_tx.send(SyncStatus::Failed(error));
912 }
913 return;
914 }
915 task_metrics
916 .pending_mutations
917 .store(durable_pending.len() as u64, Ordering::Relaxed);
918 }
919 let state_ref = state.as_ref().expect("hydration set state");
920 generation += 1;
921 let _ = snap_tx.send(Snapshot {
922 generation,
923 value: state_ref.clone(),
924 });
925 if let Some(store) = persistence.as_ref() {
926 if let Err(error) = persistence_snapshot(
927 store.clone(),
928 DurableSnapshot {
929 format_version: PERSISTENCE_FORMAT_VERSION,
930 sync_key: sync_key.clone(),
931 generation,
932 value: state_ref.clone(),
933 saved_at_ms: now_ms(),
934 },
935 )
936 .await
937 {
938 task_metrics
939 .persistence_failures
940 .fetch_add(1, Ordering::Relaxed);
941 let _ = status_tx.send(SyncStatus::Failed(error));
942 return;
943 }
944 if let Err(error) = persistence_metadata(
945 store.clone(),
946 SyncMetadata {
947 format_version: PERSISTENCE_FORMAT_VERSION,
948 sync_key: sync_key.clone(),
949 last_remote_event_at_ms: None,
950 last_successful_sync_at_ms: Some(now_ms()),
951 },
952 )
953 .await
954 {
955 task_metrics
956 .persistence_failures
957 .fetch_add(1, Ordering::Relaxed);
958 let _ = status_tx.send(SyncStatus::Failed(error));
959 return;
960 }
961 }
962 if !durable_pending.is_empty() {
963 let _ = status_tx.send(SyncStatus::Replaying);
964 if replay_pending(
965 &*backend,
966 &path,
967 state.as_mut().unwrap(),
968 &mut generation,
969 &snap_tx,
970 &mut pending,
971 &mut durable_pending,
972 persistence.as_ref(),
973 &sync_key,
974 &task_metrics,
975 )
976 .await
977 .is_err()
978 {
979 let _ = status_tx.send(SyncStatus::Offline);
980 continue;
981 }
982 }
983 attempt = 0;
984 }
985 Err(e) => {
986 task_metrics
987 .hydration_failures
988 .fetch_add(1, Ordering::Relaxed);
989 let offline_store = if matches!(
990 config.offline_policy,
991 OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
992 ) {
993 persistence.as_ref()
994 } else {
995 None
996 };
997 if let Some(store) = offline_store {
998 let _ = status_tx.send(SyncStatus::Offline);
999 let delay = config
1000 .retry
1001 .delay(attempt.saturating_add(1), config.jitter_max)
1002 .unwrap_or(Duration::from_secs(1));
1003 tokio::select! {
1004 _ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
1005 Some(write) = write_rx.recv() => {
1006 let result = handle_offline_write(
1007 write,
1008 state.get_or_insert(Value::Null),
1009 &mut generation,
1010 &snap_tx,
1011 &mut pending,
1012 &mut durable_pending,
1013 store,
1014 &sync_key,
1015 config.offline_policy,
1016 ).await;
1017 if result.is_ok() { task_metrics.successful_writes.fetch_add(1, Ordering::Relaxed); }
1018 else { task_metrics.failed_writes.fetch_add(1, Ordering::Relaxed); }
1019 task_metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed);
1020 }
1021 _ = tokio::time::sleep(delay) => {
1022 attempt = attempt.saturating_add(1);
1023 task_metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
1024 let _ = backend.before_reconnect(attempt).await;
1025 }
1026 }
1027 continue;
1028 }
1029 if !retry(
1030 &*backend,
1031 &config,
1032 &mut attempt,
1033 &child,
1034 &status_tx,
1035 &task_metrics,
1036 )
1037 .await
1038 {
1039 let _ = status_tx.send(SyncStatus::Failed(e));
1040 return;
1041 }
1042 continue;
1043 }
1044 }
1045 let mut stream = match backend.subscribe(&path).await {
1046 Ok(s) => s,
1047 Err(e) => {
1048 let queue_offline = matches!(
1049 config.offline_policy,
1050 OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
1051 );
1052 if queue_offline {
1053 if let Some(store) = persistence.as_ref() {
1054 if !reconnect_or_queue(
1055 &*backend,
1056 &config,
1057 &mut attempt,
1058 &child,
1059 &status_tx,
1060 &task_metrics,
1061 &mut write_rx,
1062 &mut state,
1063 &mut generation,
1064 &snap_tx,
1065 &mut pending,
1066 &mut durable_pending,
1067 store,
1068 &sync_key,
1069 )
1070 .await
1071 {
1072 let _ = status_tx.send(SyncStatus::Failed(e));
1073 return;
1074 }
1075 continue;
1076 }
1077 }
1078 if !retry(
1079 &*backend,
1080 &config,
1081 &mut attempt,
1082 &child,
1083 &status_tx,
1084 &task_metrics,
1085 )
1086 .await
1087 {
1088 let _ = status_tx.send(SyncStatus::Failed(e));
1089 return;
1090 }
1091 continue;
1092 }
1093 };
1094 let _ = status_tx.send(SyncStatus::Connected);
1095 loop {
1096 tokio::select! {
1097 _ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
1098 Some(write) = write_rx.recv() => { let result = handle_write(&*backend, &path, state.as_mut().expect("hydrated"), &mut generation, &snap_tx, &mut pending, write, config.write_policy, persistence.as_ref(), &sync_key, &mut durable_pending).await; task_metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed); if result.is_ok() { task_metrics.successful_writes.fetch_add(1, Ordering::Relaxed); } else { task_metrics.failed_writes.fetch_add(1, Ordering::Relaxed); let _ = status_tx.send(SyncStatus::Failed(result.clone().unwrap_err())); } },
1099 event = stream.next() => match event {
1100 Some(Ok(event)) => match reconcile_event(&path, state.as_mut().expect("hydrated"), &mut generation, &snap_tx, &mut pending, event, config.conflict_policy) {
1101 Ok(()) => {
1102 if let Some(store) = persistence.as_ref() {
1103 if let Err(error) = persistence_snapshot(store.clone(), DurableSnapshot {
1104 format_version: PERSISTENCE_FORMAT_VERSION,
1105 sync_key: sync_key.clone(),
1106 generation,
1107 value: state.as_ref().expect("hydrated").clone(),
1108 saved_at_ms: now_ms(),
1109 }).await {
1110 task_metrics.persistence_failures.fetch_add(1, Ordering::Relaxed);
1111 let _ = status_tx.send(SyncStatus::Failed(error));
1112 return;
1113 }
1114 }
1115 },
1116 Err(e) => { let _ = status_tx.send(SyncStatus::Failed(e)); return; }
1117 },
1118 Some(Err(_)) | None => break
1119 }
1120 }
1121 }
1122 task_metrics.stream_failures.fetch_add(1, Ordering::Relaxed);
1123 let queue_offline = matches!(
1124 config.offline_policy,
1125 OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
1126 );
1127 if queue_offline {
1128 if let Some(store) = persistence.as_ref() {
1129 if !reconnect_or_queue(
1130 &*backend,
1131 &config,
1132 &mut attempt,
1133 &child,
1134 &status_tx,
1135 &task_metrics,
1136 &mut write_rx,
1137 &mut state,
1138 &mut generation,
1139 &snap_tx,
1140 &mut pending,
1141 &mut durable_pending,
1142 store,
1143 &sync_key,
1144 )
1145 .await
1146 {
1147 let _ = status_tx.send(SyncStatus::Stopped);
1148 return;
1149 }
1150 continue;
1151 }
1152 }
1153 if !retry(
1154 &*backend,
1155 &config,
1156 &mut attempt,
1157 &child,
1158 &status_tx,
1159 &task_metrics,
1160 )
1161 .await
1162 {
1163 let _ = status_tx.send(SyncStatus::Stopped);
1164 return;
1165 }
1166 }
1167 });
1168 SyncHandle {
1169 snapshot: snap_rx,
1170 status: status_rx,
1171 cancel,
1172 writes,
1173 join,
1174 metrics,
1175 }
1176}
1177
1178async fn retry<B: Backend + ?Sized>(
1179 backend: &B,
1180 config: &Config,
1181 attempt: &mut usize,
1182 cancel: &CancellationToken,
1183 status: &watch::Sender<SyncStatus>,
1184 metrics: &Metrics,
1185) -> bool {
1186 *attempt += 1;
1187 let Some(delay) = config.retry.delay(*attempt, config.jitter_max) else {
1188 return false;
1189 };
1190 metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
1191 let _ = status.send(SyncStatus::Reconnecting {
1192 attempt: *attempt,
1193 delay,
1194 });
1195 if backend.before_reconnect(*attempt).await.is_err() {
1196 return false;
1197 }
1198 tokio::select! { _ = cancel.cancelled() => false, _ = tokio::time::sleep(delay) => true }
1199}
1200
1201fn durable_event(mutation: &DurableMutation) -> Result<Event, SyncError> {
1202 validate_key_and_version(
1203 &mutation.format_version,
1204 &mutation.sync_key,
1205 &mutation.sync_key,
1206 )?;
1207 match mutation.kind {
1208 DurableMutationKind::Put => Ok(Event::Put {
1209 path: mutation.path.clone(),
1210 data: mutation.payload.clone(),
1211 }),
1212 DurableMutationKind::Patch => Ok(Event::Patch {
1213 path: mutation.path.clone(),
1214 data: mutation.payload.as_object().cloned().ok_or_else(|| {
1215 SyncError::Persistence("journal patch payload is not an object".into())
1216 })?,
1217 }),
1218 }
1219}
1220
1221fn durable_mutation(
1222 sync_key: &str,
1223 pending: &[DurableMutation],
1224 event: &Event,
1225 generation: u64,
1226) -> Option<DurableMutation> {
1227 let (path, kind, payload) = match event {
1228 Event::Put { path, data } => (path.clone(), DurableMutationKind::Put, data.clone()),
1229 Event::Patch { path, data } => (
1230 path.clone(),
1231 DurableMutationKind::Patch,
1232 Value::Object(data.clone()),
1233 ),
1234 };
1235 Some(DurableMutation {
1236 format_version: PERSISTENCE_FORMAT_VERSION,
1237 sync_key: sync_key.into(),
1238 id: pending
1239 .iter()
1240 .map(|entry| entry.id)
1241 .max()
1242 .unwrap_or(0)
1243 .saturating_add(1),
1244 path,
1245 kind,
1246 payload,
1247 generation,
1248 created_at_ms: now_ms(),
1249 })
1250}
1251
1252fn event_would_change(root: &str, state: &Value, event: &Event) -> Result<bool, SyncError> {
1253 let mut candidate = state.clone();
1254 apply_event(root, &mut candidate, event.clone())?;
1255 Ok(candidate != *state)
1256}
1257
1258#[allow(clippy::too_many_arguments)]
1259async fn handle_offline_write(
1260 write: Write,
1261 state: &mut Value,
1262 generation: &mut u64,
1263 tx: &watch::Sender<Snapshot>,
1264 pending: &mut Vec<PendingMutation>,
1265 durable_pending: &mut Vec<DurableMutation>,
1266 persistence: &Arc<dyn PersistenceBackend>,
1267 sync_key: &str,
1268 policy: OfflinePolicy,
1269) -> Result<(), SyncError> {
1270 if let OfflinePolicy::QueueWithLimit { max_pending } = policy {
1271 if durable_pending.len() >= max_pending {
1272 let error = SyncError::Persistence("offline mutation queue is full".into());
1273 match write {
1274 Write::Put(_, _, done) | Write::Patch(_, _, done) => {
1275 let _ = done.send(Err(error.clone()));
1276 }
1277 }
1278 return Err(error);
1279 }
1280 }
1281 let (path, kind, payload, done) = match write {
1282 Write::Put(path, payload, done) => (path, DurableMutationKind::Put, payload, done),
1283 Write::Patch(path, payload, done) => (
1284 path,
1285 DurableMutationKind::Patch,
1286 Value::Object(payload),
1287 done,
1288 ),
1289 };
1290 let event = match kind {
1291 DurableMutationKind::Put => Event::Put {
1292 path: path.clone(),
1293 data: payload.clone(),
1294 },
1295 DurableMutationKind::Patch => Event::Patch {
1296 path: path.clone(),
1297 data: payload.as_object().cloned().unwrap_or_default(),
1298 },
1299 };
1300 let mut next = state.clone();
1301 if matches!(
1302 policy,
1303 OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
1304 ) {
1305 apply_event("", &mut next, event.clone())?;
1306 }
1307 let mutation = DurableMutation {
1308 format_version: PERSISTENCE_FORMAT_VERSION,
1309 sync_key: sync_key.into(),
1310 id: durable_pending
1311 .iter()
1312 .map(|entry| entry.id)
1313 .max()
1314 .unwrap_or(0)
1315 .saturating_add(1),
1316 path: path.clone(),
1317 kind,
1318 payload,
1319 generation: generation.saturating_add(1),
1320 created_at_ms: now_ms(),
1321 };
1322 if let Err(error) = persistence_append(persistence.clone(), mutation.clone()).await {
1323 let _ = done.send(Err(error.clone()));
1324 return Err(error);
1325 }
1326 *state = next;
1327 *generation = mutation.generation;
1328 let _ = tx.send(Snapshot {
1329 generation: *generation,
1330 value: state.clone(),
1331 });
1332 pending.push(PendingMutation { path, event });
1333 durable_pending.push(mutation);
1334 if let Err(error) = persistence_snapshot(
1335 persistence.clone(),
1336 DurableSnapshot {
1337 format_version: PERSISTENCE_FORMAT_VERSION,
1338 sync_key: sync_key.into(),
1339 generation: *generation,
1340 value: state.clone(),
1341 saved_at_ms: now_ms(),
1342 },
1343 )
1344 .await
1345 {
1346 let _ = done.send(Err(error.clone()));
1347 return Err(error);
1348 }
1349 let _ = done.send(Ok(()));
1350 Ok(())
1351}
1352
1353#[allow(clippy::too_many_arguments)]
1354async fn replay_pending<B: Backend + ?Sized>(
1355 backend: &B,
1356 root: &str,
1357 state: &mut Value,
1358 generation: &mut u64,
1359 tx: &watch::Sender<Snapshot>,
1360 pending: &mut Vec<PendingMutation>,
1361 durable_pending: &mut Vec<DurableMutation>,
1362 persistence: Option<&Arc<dyn PersistenceBackend>>,
1363 sync_key: &str,
1364 metrics: &Metrics,
1365) -> Result<(), SyncError> {
1366 for mutation in durable_pending.clone() {
1367 metrics.replay_attempts.fetch_add(1, Ordering::Relaxed);
1368 let remote_path = join(root, &mutation.path)?;
1369 let event = durable_event(&mutation)?;
1370 let result = match mutation.kind {
1371 DurableMutationKind::Put => backend.put(&remote_path, mutation.payload.clone()).await,
1372 DurableMutationKind::Patch => {
1373 backend
1374 .patch(
1375 &remote_path,
1376 mutation.payload.as_object().cloned().ok_or_else(|| {
1377 SyncError::Persistence("journal patch payload is not an object".into())
1378 })?,
1379 )
1380 .await
1381 }
1382 };
1383 if let Err(error) = result {
1384 metrics.replay_failures.fetch_add(1, Ordering::Relaxed);
1385 return Err(error);
1386 }
1387 apply_event(root, state, event.clone())?;
1388 *generation += 1;
1389 let _ = tx.send(Snapshot {
1390 generation: *generation,
1391 value: state.clone(),
1392 });
1393 if let Some(store) = persistence {
1394 persistence_snapshot(
1395 store.clone(),
1396 DurableSnapshot {
1397 format_version: PERSISTENCE_FORMAT_VERSION,
1398 sync_key: sync_key.into(),
1399 generation: *generation,
1400 value: state.clone(),
1401 saved_at_ms: now_ms(),
1402 },
1403 )
1404 .await?;
1405 persistence_ack(store.clone(), sync_key.into(), mutation.id).await?;
1406 }
1407 if let Some(index) = durable_pending
1408 .iter()
1409 .position(|entry| entry.id == mutation.id)
1410 {
1411 durable_pending.remove(index);
1412 }
1413 if let Some(index) = pending
1414 .iter()
1415 .position(|entry| equivalent_event(&entry.event, &event))
1416 {
1417 pending.remove(index);
1418 }
1419 metrics.replay_successes.fetch_add(1, Ordering::Relaxed);
1420 }
1421 if let Some(store) = persistence {
1422 persistence_compact(store.clone(), sync_key.into()).await?;
1423 }
1424 metrics
1425 .pending_mutations
1426 .store(durable_pending.len() as u64, Ordering::Relaxed);
1427 Ok(())
1428}
1429
1430#[allow(clippy::too_many_arguments)]
1431async fn resolve_restored_conflicts(
1432 base: &Value,
1433 remote: &Value,
1434 policy: ConflictPolicy,
1435 pending: &mut Vec<PendingMutation>,
1436 durable_pending: &mut Vec<DurableMutation>,
1437 persistence: Option<&Arc<dyn PersistenceBackend>>,
1438 sync_key: &str,
1439 metrics: &Metrics,
1440) -> Result<(), SyncError> {
1441 let mut conflicted = false;
1442 for mutation in durable_pending.clone() {
1443 let event = durable_event(&mutation)?;
1444 if base != remote && event_would_change("", base, &event)? {
1445 conflicted = true;
1446 match policy {
1447 ConflictPolicy::RemoteWins => {
1448 if let Some(store) = persistence {
1449 persistence_ack(store.clone(), sync_key.into(), mutation.id).await?;
1450 }
1451 durable_pending.retain(|entry| entry.id != mutation.id);
1452 if let Some(index) = pending
1453 .iter()
1454 .position(|entry| equivalent_event(&entry.event, &event))
1455 {
1456 pending.remove(index);
1457 }
1458 }
1459 ConflictPolicy::LocalWins => {}
1460 ConflictPolicy::Reject => {
1461 return Err(SyncError::Conflict(format!(
1462 "restored mutation conflicts at {}",
1463 mutation.path
1464 )))
1465 }
1466 }
1467 }
1468 }
1469 if conflicted {
1470 metrics.conflict_count.fetch_add(1, Ordering::Relaxed);
1471 if let Some(store) = persistence {
1472 persistence_compact(store.clone(), sync_key.into()).await?;
1473 }
1474 }
1475 Ok(())
1476}
1477
1478#[allow(clippy::too_many_arguments)]
1479async fn reconnect_or_queue<B: Backend + ?Sized>(
1480 backend: &B,
1481 config: &Config,
1482 attempt: &mut usize,
1483 cancel: &CancellationToken,
1484 status: &watch::Sender<SyncStatus>,
1485 metrics: &Metrics,
1486 write_rx: &mut mpsc::Receiver<Write>,
1487 state: &mut Option<Value>,
1488 generation: &mut u64,
1489 snapshot_tx: &watch::Sender<Snapshot>,
1490 pending: &mut Vec<PendingMutation>,
1491 durable_pending: &mut Vec<DurableMutation>,
1492 persistence: &Arc<dyn PersistenceBackend>,
1493 sync_key: &str,
1494) -> bool {
1495 loop {
1496 *attempt = attempt.saturating_add(1);
1497 let delay = config
1498 .retry
1499 .delay(*attempt, config.jitter_max)
1500 .unwrap_or(Duration::from_secs(1));
1501 metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
1502 let _ = status.send(SyncStatus::Offline);
1503 tokio::select! {
1504 _ = cancel.cancelled() => return false,
1505 Some(write) = write_rx.recv() => {
1506 let result = handle_offline_write(
1507 write,
1508 state.get_or_insert(Value::Null),
1509 generation,
1510 snapshot_tx,
1511 pending,
1512 durable_pending,
1513 persistence,
1514 sync_key,
1515 config.offline_policy,
1516 ).await;
1517 if result.is_ok() {
1518 metrics.successful_writes.fetch_add(1, Ordering::Relaxed);
1519 } else {
1520 metrics.failed_writes.fetch_add(1, Ordering::Relaxed);
1521 }
1522 metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed);
1523 }
1524 _ = tokio::time::sleep(delay) => {
1525 if backend.before_reconnect(*attempt).await.is_ok() {
1526 return true;
1527 }
1528 }
1529 }
1530 }
1531}
1532
1533#[allow(clippy::too_many_arguments)]
1534async fn handle_write<B: Backend + ?Sized>(
1535 backend: &B,
1536 root: &str,
1537 state: &mut Value,
1538 generation: &mut u64,
1539 tx: &watch::Sender<Snapshot>,
1540 pending: &mut Vec<PendingMutation>,
1541 write: Write,
1542 policy: WritePolicy,
1543 persistence: Option<&Arc<dyn PersistenceBackend>>,
1544 sync_key: &str,
1545 durable_pending: &mut Vec<DurableMutation>,
1546) -> Result<(), SyncError> {
1547 match write {
1548 Write::Put(path, value, done) => {
1549 let old = state.clone();
1550 let backend_path = join(root, &path)?;
1551 let event = Event::Put {
1552 path: path.clone(),
1553 data: value.clone(),
1554 };
1555 let durable = durable_mutation(sync_key, durable_pending, &event, *generation + 1);
1556 if let Some(store) = persistence {
1557 let mutation = durable.clone().unwrap();
1558 if let Err(error) = persistence_append(Arc::clone(store), mutation.clone()).await {
1559 let _ = done.send(Err(error.clone()));
1560 return Err(error);
1561 }
1562 durable_pending.push(mutation);
1563 }
1564 if policy == WritePolicy::Optimistic {
1565 let mut next = state.clone();
1566 apply_event(root, &mut next, event.clone())?;
1567 *state = next;
1568 *generation += 1;
1569 let _ = tx.send(Snapshot {
1570 generation: *generation,
1571 value: state.clone(),
1572 });
1573 if let Some(store) = persistence {
1574 if let Err(error) = persistence_snapshot(
1575 Arc::clone(store),
1576 DurableSnapshot {
1577 format_version: PERSISTENCE_FORMAT_VERSION,
1578 sync_key: sync_key.into(),
1579 generation: *generation,
1580 value: state.clone(),
1581 saved_at_ms: now_ms(),
1582 },
1583 )
1584 .await
1585 {
1586 let _ = done.send(Err(error.clone()));
1587 return Err(error);
1588 }
1589 }
1590 }
1591 pending.push(PendingMutation {
1592 path: path.clone(),
1593 event: event.clone(),
1594 });
1595 let result = backend.put(&backend_path, value).await;
1596 if result.is_ok() {
1597 if let (Some(store), Some(mutation)) = (persistence, durable) {
1598 if let Err(error) =
1599 persistence_ack(Arc::clone(store), sync_key.into(), mutation.id).await
1600 {
1601 let _ = done.send(Err(error.clone()));
1602 return Err(error);
1603 }
1604 if let Err(error) =
1605 persistence_compact(Arc::clone(store), sync_key.into()).await
1606 {
1607 let _ = done.send(Err(error.clone()));
1608 return Err(error);
1609 }
1610 durable_pending.retain(|entry| entry.id != mutation.id);
1611 }
1612 }
1613 if result.is_err() && policy == WritePolicy::Optimistic {
1614 *state = old;
1615 *generation += 1;
1616 let _ = tx.send(Snapshot {
1617 generation: *generation,
1618 value: state.clone(),
1619 });
1620 }
1621 if result.is_err() {
1622 let mut removed = false;
1623 pending.retain(|mutation| {
1624 if !removed && equivalent_event(&mutation.event, &event) {
1625 removed = true;
1626 false
1627 } else {
1628 true
1629 }
1630 });
1631 }
1632 let _ = done.send(result.clone());
1633 result
1634 }
1635 Write::Patch(path, data, done) => {
1636 let old = state.clone();
1637 let backend_path = join(root, &path)?;
1638 let event = Event::Patch {
1639 path: path.clone(),
1640 data: data.clone(),
1641 };
1642 let durable = durable_mutation(sync_key, durable_pending, &event, *generation + 1);
1643 if let Some(store) = persistence {
1644 let mutation = durable.clone().unwrap();
1645 if let Err(error) = persistence_append(Arc::clone(store), mutation.clone()).await {
1646 let _ = done.send(Err(error.clone()));
1647 return Err(error);
1648 }
1649 durable_pending.push(mutation);
1650 }
1651 if policy == WritePolicy::Optimistic {
1652 let mut next = state.clone();
1653 apply_event(root, &mut next, event.clone())?;
1654 *state = next;
1655 *generation += 1;
1656 let _ = tx.send(Snapshot {
1657 generation: *generation,
1658 value: state.clone(),
1659 });
1660 if let Some(store) = persistence {
1661 if let Err(error) = persistence_snapshot(
1662 Arc::clone(store),
1663 DurableSnapshot {
1664 format_version: PERSISTENCE_FORMAT_VERSION,
1665 sync_key: sync_key.into(),
1666 generation: *generation,
1667 value: state.clone(),
1668 saved_at_ms: now_ms(),
1669 },
1670 )
1671 .await
1672 {
1673 let _ = done.send(Err(error.clone()));
1674 return Err(error);
1675 }
1676 }
1677 }
1678 pending.push(PendingMutation {
1679 path: path.clone(),
1680 event: event.clone(),
1681 });
1682 let result = backend.patch(&backend_path, data).await;
1683 if result.is_ok() {
1684 if let (Some(store), Some(mutation)) = (persistence, durable) {
1685 if let Err(error) =
1686 persistence_ack(Arc::clone(store), sync_key.into(), mutation.id).await
1687 {
1688 let _ = done.send(Err(error.clone()));
1689 return Err(error);
1690 }
1691 if let Err(error) =
1692 persistence_compact(Arc::clone(store), sync_key.into()).await
1693 {
1694 let _ = done.send(Err(error.clone()));
1695 return Err(error);
1696 }
1697 durable_pending.retain(|entry| entry.id != mutation.id);
1698 }
1699 }
1700 if result.is_err() && policy == WritePolicy::Optimistic {
1701 *state = old;
1702 *generation += 1;
1703 let _ = tx.send(Snapshot {
1704 generation: *generation,
1705 value: state.clone(),
1706 });
1707 }
1708 if result.is_err() {
1709 let mut removed = false;
1710 pending.retain(|mutation| {
1711 if !removed && equivalent_event(&mutation.event, &event) {
1712 removed = true;
1713 false
1714 } else {
1715 true
1716 }
1717 });
1718 }
1719 let _ = done.send(result.clone());
1720 result
1721 }
1722 }
1723}
1724
1725fn reconcile_event(
1726 root: &str,
1727 state: &mut Value,
1728 generation: &mut u64,
1729 tx: &watch::Sender<Snapshot>,
1730 pending: &mut Vec<PendingMutation>,
1731 event: Event,
1732 policy: ConflictPolicy,
1733) -> Result<(), SyncError> {
1734 if let Some(index) = pending
1735 .iter()
1736 .position(|mutation| mutation.path == event_path(&event))
1737 {
1738 let mutation = &pending[index];
1739 if equivalent_event(&mutation.event, &event) {
1740 pending.remove(index);
1741 return Ok(());
1742 }
1743 match policy {
1744 ConflictPolicy::LocalWins => return Ok(()),
1745 ConflictPolicy::Reject => {
1746 return Err(SyncError::Conflict(format!(
1747 "remote event conflicts at {}",
1748 event_path(&event)
1749 )))
1750 }
1751 ConflictPolicy::RemoteWins => {
1752 pending.remove(index);
1753 }
1754 }
1755 }
1756 apply_event(root, state, event)?;
1757 *generation += 1;
1758 let _ = tx.send(Snapshot {
1759 generation: *generation,
1760 value: state.clone(),
1761 });
1762 Ok(())
1763}
1764
1765fn event_path(event: &Event) -> &str {
1766 match event {
1767 Event::Put { path, .. } | Event::Patch { path, .. } => path,
1768 }
1769}
1770fn equivalent_event(left: &Event, right: &Event) -> bool {
1771 match (left, right) {
1772 (Event::Put { path: a, data: b }, Event::Put { path: c, data: d }) => a == c && b == d,
1773 (Event::Patch { path: a, data: b }, Event::Patch { path: c, data: d }) => a == c && b == d,
1774 _ => false,
1775 }
1776}
1777
1778fn join(root: &str, path: &str) -> Result<String, SyncError> {
1779 let a = root.trim_matches('/');
1780 let b = path.trim_matches('/');
1781 if !b.is_empty() && b.split('/').any(|x| x.is_empty() || x == "." || x == "..") {
1782 return Err(SyncError::InvalidPath(path.into()));
1783 }
1784 Ok(if b.is_empty() {
1785 a.into()
1786 } else if a.is_empty() {
1787 b.into()
1788 } else {
1789 format!("{a}/{b}")
1790 })
1791}
1792fn segments(path: &str) -> Result<Vec<&str>, SyncError> {
1793 let p = path.trim_matches('/');
1794 if p.is_empty() {
1795 Ok(vec![])
1796 } else {
1797 let out: Vec<_> = p.split('/').collect();
1798 if out.iter().any(|x| x.is_empty() || *x == "." || *x == "..") {
1799 Err(SyncError::InvalidPath(path.into()))
1800 } else {
1801 Ok(out)
1802 }
1803 }
1804}
1805pub fn apply_event(root: &str, state: &mut Value, event: Event) -> Result<(), SyncError> {
1806 let relative = match &event {
1807 Event::Put { path, .. } | Event::Patch { path, .. } => path.clone(),
1808 };
1809 let _ = join(root, &relative)?;
1810 let parts = segments(&relative)?;
1811 let target = get_or_create(state, &parts);
1812 match event {
1813 Event::Put { data, .. } => {
1814 if data.is_null() && !parts.is_empty() {
1815 remove_at(state, &parts);
1816 } else {
1817 *target = data;
1818 }
1819 }
1820 Event::Patch { data, .. } => {
1821 if data.is_empty() {
1822 return Ok(());
1823 }
1824 if !target.is_object() {
1825 *target = Value::Object(Map::new());
1826 }
1827 let obj = target.as_object_mut().unwrap();
1828 for (key, value) in data {
1829 let keys = segments(&key)?;
1830 set_at(obj, &keys, value);
1831 }
1832 }
1833 }
1834 Ok(())
1835}
1836fn get_or_create<'a>(value: &'a mut Value, parts: &[&str]) -> &'a mut Value {
1837 let mut current = value;
1838 for part in parts {
1839 if !current.is_object() {
1840 *current = Value::Object(Map::new());
1841 }
1842 current = current
1843 .as_object_mut()
1844 .unwrap()
1845 .entry(*part)
1846 .or_insert(Value::Null);
1847 }
1848 current
1849}
1850fn set_at(obj: &mut Map<String, Value>, parts: &[&str], value: Value) {
1851 if parts.is_empty() {
1852 return;
1853 }
1854 if parts.len() == 1 {
1855 if value.is_null() {
1856 obj.remove(parts[0]);
1857 } else {
1858 obj.insert(parts[0].into(), value);
1859 }
1860 } else {
1861 let child = obj
1862 .entry(parts[0])
1863 .or_insert_with(|| Value::Object(Map::new()));
1864 if !child.is_object() {
1865 *child = Value::Object(Map::new());
1866 }
1867 set_at(child.as_object_mut().unwrap(), &parts[1..], value);
1868 }
1869}
1870fn remove_at(value: &mut Value, parts: &[&str]) {
1871 if parts.is_empty() {
1872 *value = Value::Null;
1873 return;
1874 }
1875 if let Some(obj) = value.as_object_mut() {
1876 if parts.len() == 1 {
1877 obj.remove(parts[0]);
1878 } else if let Some(child) = obj.get_mut(parts[0]) {
1879 remove_at(child, &parts[1..]);
1880 }
1881 }
1882}
1883
1884#[derive(Debug, Clone, PartialEq, Eq)]
1888pub enum SyncStatus {
1889 Idle,
1891 Hydrating,
1893 RestoredStale,
1895 Offline,
1897 Replaying,
1899 Conflict(SyncError),
1901 Connected,
1903 Reconnecting { attempt: usize, delay: Duration },
1905 Stopped,
1907 Failed(SyncError),
1909}
1910
1911impl fmt::Display for SyncStatus {
1912 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1913 write!(f, "{self:?}")
1914 }
1915}
1916
1917#[derive(Clone)]
1920pub struct RtdbBackend {
1921 client: Arc<RtdbClient>,
1922 base_url: String,
1923 token: String,
1924 namespace: Option<String>,
1925}
1926
1927pub struct TypedBackend<T> {
1931 client: Arc<rtdb_typed::TypedClient>,
1932 marker: std::marker::PhantomData<fn() -> T>,
1933}
1934
1935impl<T> Clone for TypedBackend<T> {
1936 fn clone(&self) -> Self {
1937 Self {
1938 client: self.client.clone(),
1939 marker: std::marker::PhantomData,
1940 }
1941 }
1942}
1943
1944impl<T> TypedBackend<T>
1945where
1946 T: Serialize + DeserializeOwned + Send + Sync + 'static,
1947{
1948 pub fn new(client: rtdb_typed::TypedClient) -> Self {
1949 Self {
1950 client: Arc::new(client),
1951 marker: std::marker::PhantomData,
1952 }
1953 }
1954 pub fn from_parts(base_url: impl Into<String>, token: impl Into<String>) -> Self {
1955 Self::new(rtdb_typed::TypedClient::from_parts(base_url, token))
1956 }
1957}
1958
1959#[async_trait]
1960impl<T> Backend for TypedBackend<T>
1961where
1962 T: Serialize + DeserializeOwned + Send + Sync + 'static,
1963{
1964 async fn get(&self, path: &str) -> Result<Value, SyncError> {
1965 self.client
1966 .get::<T>(path)
1967 .await
1968 .map_err(RtdbBackend::error)
1969 .and_then(|value| {
1970 serde_json::to_value(value)
1971 .map_err(|error| SyncError::Conversion(error.to_string()))
1972 })
1973 }
1974 async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError> {
1975 let stream = self
1976 .client
1977 .query::<T>(path)
1978 .stream()
1979 .await
1980 .map_err(RtdbBackend::error)?;
1981 let stream = stream.filter_map(|event| async move {
1982 match event {
1983 Ok(rtdb_typed::TypedEvent::Put { path, data }) => Some(
1984 serde_json::to_value(data)
1985 .map(|data| Event::Put { path, data })
1986 .map_err(|error| SyncError::Conversion(error.to_string())),
1987 ),
1988 Ok(rtdb_typed::TypedEvent::Patch { path, data }) => Some(Ok(Event::Patch {
1989 path,
1990 data: data.as_object().cloned().unwrap_or_default(),
1991 })),
1992 Ok(rtdb_typed::TypedEvent::KeepAlive) => None,
1993 Ok(rtdb_typed::TypedEvent::Cancel) => {
1994 Some(Err(SyncError::Backend("Firebase stream cancelled".into())))
1995 }
1996 Err(error) => Some(Err(RtdbBackend::error(error))),
1997 }
1998 });
1999 Ok(Box::pin(stream))
2000 }
2001 async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
2002 self.client
2003 .inner()
2004 .put(path, &value)
2005 .await
2006 .map(|_| ())
2007 .map_err(RtdbBackend::error)
2008 }
2009 async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2010 self.client
2011 .inner()
2012 .patch(path, &Value::Object(value))
2013 .await
2014 .map(|_| ())
2015 .map_err(RtdbBackend::error)
2016 }
2017}
2018
2019impl RtdbBackend {
2020 pub fn new(base_url: impl Into<String>, token: impl Into<String>) -> Self {
2021 let base_url = base_url.into();
2022 let token = token.into();
2023 Self {
2024 client: Arc::new(RtdbClient::new(&base_url, &token)),
2025 base_url,
2026 token,
2027 namespace: None,
2028 }
2029 }
2030 pub fn with_namespace(mut self, namespace: impl Into<String>) -> Self {
2031 let namespace = namespace.into();
2032 self.client = Arc::new(
2033 RtdbClient::new(&self.base_url, &self.token).with_namespace(namespace.clone()),
2034 );
2035 self.namespace = Some(namespace);
2036 self
2037 }
2038 pub fn typed_client(&self) -> rtdb_typed::TypedClient {
2039 let client = RtdbClient::new(&self.base_url, &self.token);
2040 let client = match &self.namespace {
2041 Some(namespace) => client.with_namespace(namespace.clone()),
2042 None => client,
2043 };
2044 rtdb_typed::TypedClient::new(client)
2045 }
2046 fn error(error: impl fmt::Debug) -> SyncError {
2047 SyncError::Backend(format!("{error:?}"))
2048 }
2049}
2050
2051#[async_trait]
2052impl Backend for RtdbBackend {
2053 async fn get(&self, path: &str) -> Result<Value, SyncError> {
2054 self.client.get(path).await.map_err(Self::error)
2055 }
2056 async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError> {
2057 let stream = self.client.stream(path).await.map_err(Self::error)?;
2058 let stream = stream.filter_map(|event| async move {
2059 match event {
2060 Ok(RtdbEvent::Put { path, data }) => Some(Ok(Event::Put { path, data })),
2061 Ok(RtdbEvent::Patch { path, data }) => Some(Ok(Event::Patch {
2062 path,
2063 data: data.as_object().cloned().unwrap_or_default(),
2064 })),
2065 Ok(RtdbEvent::KeepAlive) => None,
2066 Ok(RtdbEvent::Cancel) => {
2067 Some(Err(SyncError::Backend("Firebase stream cancelled".into())))
2068 }
2069 Err(error) => Some(Err(Self::error(error))),
2070 }
2071 });
2072 Ok(Box::pin(stream))
2073 }
2074 async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
2075 self.client
2076 .put(path, &value)
2077 .await
2078 .map(|_| ())
2079 .map_err(Self::error)
2080 }
2081 async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2082 self.client
2083 .patch(path, &Value::Object(value))
2084 .await
2085 .map(|_| ())
2086 .map_err(Self::error)
2087 }
2088}
2089
2090#[cfg(test)]
2091mod tests {
2092 use super::*;
2093 use futures_util::stream;
2094 use serde::Deserialize;
2095 use std::sync::Mutex;
2096
2097 #[derive(Clone)]
2098 struct Mock {
2099 value: Arc<Mutex<Value>>,
2100 events: Arc<Mutex<Vec<Event>>>,
2101 writes: Arc<Mutex<Vec<String>>>,
2102 }
2103
2104 struct Flaky {
2105 subscriptions: AtomicU64,
2106 replacements: AtomicU64,
2107 }
2108
2109 struct Cycling {
2110 subscriptions: AtomicU64,
2111 }
2112 #[async_trait]
2113 impl Backend for Cycling {
2114 async fn get(&self, _: &str) -> Result<Value, SyncError> {
2115 Ok(Value::Null)
2116 }
2117 async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2118 if self.subscriptions.fetch_add(1, Ordering::Relaxed) < 25 {
2119 Ok(Box::pin(stream::empty()))
2120 } else {
2121 Ok(Box::pin(stream::pending()))
2122 }
2123 }
2124 async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
2125 Ok(())
2126 }
2127 async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
2128 Ok(())
2129 }
2130 }
2131
2132 struct Failing;
2133 #[async_trait]
2134 impl Backend for Failing {
2135 async fn get(&self, _: &str) -> Result<Value, SyncError> {
2136 Ok(serde_json::json!({"count": 1}))
2137 }
2138 async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2139 Ok(Box::pin(stream::pending()))
2140 }
2141 async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
2142 Err(SyncError::Backend("injected write failure".into()))
2143 }
2144 async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
2145 Err(SyncError::Backend("injected write failure".into()))
2146 }
2147 }
2148
2149 #[derive(Clone)]
2150 struct Recoverable {
2151 online: Arc<std::sync::atomic::AtomicBool>,
2152 value: Arc<Mutex<Value>>,
2153 events: tokio::sync::broadcast::Sender<Event>,
2154 }
2155
2156 #[async_trait]
2157 impl Backend for Recoverable {
2158 async fn get(&self, _: &str) -> Result<Value, SyncError> {
2159 if !self.online.load(Ordering::Relaxed) {
2160 return Err(SyncError::Backend("offline".into()));
2161 }
2162 Ok(self.value.lock().unwrap().clone())
2163 }
2164 async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2165 if !self.online.load(Ordering::Relaxed) {
2166 return Err(SyncError::Backend("offline".into()));
2167 }
2168 let mut receiver = self.events.subscribe();
2169 Ok(Box::pin(async_stream::stream! {
2170 loop { match receiver.recv().await { Ok(event) => yield Ok(event), Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, Err(tokio::sync::broadcast::error::RecvError::Closed) => break } }
2171 }))
2172 }
2173 async fn put(&self, _: &str, value: Value) -> Result<(), SyncError> {
2174 if !self.online.load(Ordering::Relaxed) {
2175 return Err(SyncError::Backend("offline".into()));
2176 }
2177 *self.value.lock().unwrap() = value.clone();
2178 let _ = self.events.send(Event::Put {
2179 path: "".into(),
2180 data: value,
2181 });
2182 Ok(())
2183 }
2184 async fn patch(&self, _: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2185 if !self.online.load(Ordering::Relaxed) {
2186 return Err(SyncError::Backend("offline".into()));
2187 }
2188 apply_event(
2189 "",
2190 &mut self.value.lock().unwrap(),
2191 Event::Patch {
2192 path: "".into(),
2193 data: value.clone(),
2194 },
2195 )?;
2196 let _ = self.events.send(Event::Patch {
2197 path: "".into(),
2198 data: value,
2199 });
2200 Ok(())
2201 }
2202 }
2203
2204 #[derive(Clone)]
2205 struct Live {
2206 value: Arc<Mutex<Value>>,
2207 events: tokio::sync::broadcast::Sender<Event>,
2208 }
2209 #[async_trait]
2210 impl Backend for Live {
2211 async fn get(&self, _: &str) -> Result<Value, SyncError> {
2212 Ok(self.value.lock().unwrap().clone())
2213 }
2214 async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2215 let mut receiver = self.events.subscribe();
2216 Ok(Box::pin(async_stream::stream! {
2217 loop { match receiver.recv().await { Ok(event) => yield Ok(event), Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, Err(tokio::sync::broadcast::error::RecvError::Closed) => break } }
2218 }))
2219 }
2220 async fn put(&self, _: &str, value: Value) -> Result<(), SyncError> {
2221 *self.value.lock().unwrap() = value.clone();
2222 let _ = self.events.send(Event::Put {
2223 path: "".into(),
2224 data: value,
2225 });
2226 Ok(())
2227 }
2228 async fn patch(&self, _: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2229 let mut current = self.value.lock().unwrap();
2230 apply_event(
2231 "",
2232 &mut current,
2233 Event::Patch {
2234 path: "".into(),
2235 data: value.clone(),
2236 },
2237 )?;
2238 let _ = self.events.send(Event::Patch {
2239 path: "".into(),
2240 data: value,
2241 });
2242 Ok(())
2243 }
2244 }
2245
2246 #[async_trait]
2247 impl Backend for Flaky {
2248 async fn get(&self, _: &str) -> Result<Value, SyncError> {
2249 Ok(Value::Null)
2250 }
2251 async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2252 if self.subscriptions.fetch_add(1, Ordering::Relaxed) == 0 {
2253 Ok(Box::pin(stream::empty()))
2254 } else {
2255 Ok(Box::pin(stream::pending()))
2256 }
2257 }
2258 async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
2259 Ok(())
2260 }
2261 async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
2262 Ok(())
2263 }
2264 async fn before_reconnect(&self, _: usize) -> Result<(), SyncError> {
2265 self.replacements.fetch_add(1, Ordering::Relaxed);
2266 Ok(())
2267 }
2268 }
2269
2270 #[async_trait]
2271 impl Backend for Mock {
2272 async fn get(&self, _: &str) -> Result<Value, SyncError> {
2273 Ok(self.value.lock().unwrap().clone())
2274 }
2275 async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2276 let events = std::mem::take(&mut *self.events.lock().unwrap());
2277 Ok(Box::pin(
2278 stream::iter(events.into_iter().map(Ok)).chain(stream::pending()),
2279 ))
2280 }
2281 async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
2282 *self.value.lock().unwrap() = value;
2283 self.writes.lock().unwrap().push(format!("put:{path}"));
2284 Ok(())
2285 }
2286 async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2287 apply_event(
2288 "",
2289 &mut self.value.lock().unwrap(),
2290 Event::Patch {
2291 path: "".into(),
2292 data: value,
2293 },
2294 )?;
2295 self.writes.lock().unwrap().push(format!("patch:{path}"));
2296 Ok(())
2297 }
2298 }
2299
2300 fn mock(value: Value, events: Vec<Event>) -> Mock {
2301 Mock {
2302 value: Arc::new(Mutex::new(value)),
2303 events: Arc::new(Mutex::new(events)),
2304 writes: Arc::new(Mutex::new(Vec::new())),
2305 }
2306 }
2307
2308 #[test]
2309 fn put_patch_and_null_delete_are_deterministic() {
2310 let mut state = serde_json::json!({"a": {"b": 1, "c": 2}});
2311 apply_event(
2312 "root",
2313 &mut state,
2314 Event::Patch {
2315 path: "".into(),
2316 data: serde_json::json!({"a/b": 3, "a/c": null, "new/x": true})
2317 .as_object()
2318 .unwrap()
2319 .clone(),
2320 },
2321 )
2322 .unwrap();
2323 assert_eq!(
2324 state,
2325 serde_json::json!({"a": {"b": 3}, "new": {"x": true}})
2326 );
2327 apply_event(
2328 "root",
2329 &mut state,
2330 Event::Put {
2331 path: "a".into(),
2332 data: Value::Null,
2333 },
2334 )
2335 .unwrap();
2336 assert_eq!(state, serde_json::json!({"new": {"x": true}}));
2337 }
2338
2339 #[tokio::test]
2340 async fn hydrates_notifies_and_stops() {
2341 let backend = mock(
2342 serde_json::json!({"count": 1}),
2343 vec![Event::Patch {
2344 path: "".into(),
2345 data: serde_json::json!({"count": 2}).as_object().unwrap().clone(),
2346 }],
2347 );
2348 let handle = start(
2349 Arc::new(backend),
2350 "items",
2351 Config {
2352 retry: RetryPolicy::Never,
2353 ..Config::default()
2354 },
2355 );
2356 let mut snapshots = handle.subscribe();
2357 tokio::time::timeout(Duration::from_secs(1), async {
2358 while snapshots.borrow().value != serde_json::json!({"count": 2}) {
2359 snapshots.changed().await.unwrap();
2360 }
2361 })
2362 .await
2363 .unwrap();
2364 assert_eq!(snapshots.borrow().value, serde_json::json!({"count": 2}));
2365 handle.shutdown().await;
2366 }
2367
2368 #[tokio::test]
2369 async fn typed_snapshot_and_optimistic_rollback_contract() {
2370 #[derive(Debug, Deserialize, PartialEq)]
2371 struct Model {
2372 count: u32,
2373 }
2374 let backend = mock(serde_json::json!({"count": 4}), vec![]);
2375 let handle = start(
2376 Arc::new(backend),
2377 "items",
2378 Config {
2379 retry: RetryPolicy::Never,
2380 write_policy: WritePolicy::Optimistic,
2381 ..Config::default()
2382 },
2383 );
2384 let mut status = handle.subscribe_status();
2385 tokio::time::timeout(Duration::from_secs(1), async {
2386 while *status.borrow() != SyncStatus::Connected {
2387 status.changed().await.unwrap();
2388 }
2389 })
2390 .await
2391 .unwrap();
2392 assert_eq!(
2393 handle.snapshot().decode::<Model>().unwrap(),
2394 Model { count: 4 }
2395 );
2396 handle.shutdown().await;
2397 }
2398
2399 #[tokio::test]
2400 async fn typed_handle_surfaces_conversion_errors_after_partial_updates() {
2401 #[derive(Debug, Clone, Deserialize, PartialEq)]
2402 struct Model {
2403 count: u32,
2404 }
2405 let backend = mock(
2406 serde_json::json!({"count": 1}),
2407 vec![
2408 Event::Patch {
2409 path: "".into(),
2410 data: serde_json::json!({"count": 2}).as_object().unwrap().clone(),
2411 },
2412 Event::Put {
2413 path: "".into(),
2414 data: serde_json::json!({"count": "not-a-number"}),
2415 },
2416 ],
2417 );
2418 let handle = start_typed::<_, Model>(
2419 Arc::new(backend),
2420 "typed",
2421 Config {
2422 retry: RetryPolicy::Never,
2423 ..Config::default()
2424 },
2425 );
2426 let mut snapshots = handle.subscribe();
2427 tokio::time::timeout(Duration::from_secs(1), async {
2428 while handle.status() != SyncStatus::Connected {
2429 snapshots.changed().await.unwrap();
2430 }
2431 })
2432 .await
2433 .unwrap();
2434 tokio::time::sleep(Duration::from_millis(10)).await;
2435 assert!(matches!(
2436 &*snapshots.borrow(),
2437 Err(SyncError::Conversion(_))
2438 ));
2439 handle.shutdown().await;
2440 }
2441
2442 #[tokio::test]
2443 async fn firebase_emulator_crud_and_sse_when_configured() {
2444 let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
2445 return;
2446 };
2447 let root = test_root("crud");
2448 let backend = RtdbBackend::new(format!("http://{host}"), "")
2449 .with_namespace("demo-rtdb-sync-default-rtdb");
2450 backend
2451 .put(&root, serde_json::json!({"generation": 1}))
2452 .await
2453 .unwrap();
2454 assert_eq!(backend.get(&root).await.unwrap()["generation"], 1);
2455 let mut events = backend.subscribe(&root).await.unwrap();
2456 backend
2457 .patch(
2458 &root,
2459 serde_json::json!({"generation": 2})
2460 .as_object()
2461 .unwrap()
2462 .clone(),
2463 )
2464 .await
2465 .unwrap();
2466 let event = tokio::time::timeout(Duration::from_secs(2), async {
2467 loop {
2468 if let Some(Ok(event)) = events.next().await {
2469 let is_target = match &event {
2470 Event::Put { data, .. } => data["generation"] == 2,
2471 Event::Patch { data, .. } => {
2472 data.get("generation") == Some(&serde_json::json!(2))
2473 }
2474 };
2475 if is_target {
2476 break event;
2477 }
2478 }
2479 }
2480 })
2481 .await
2482 .unwrap();
2483 assert!(
2484 matches!(event, Event::Patch { ref data, .. } if data.get("generation") == Some(&serde_json::json!(2)))
2485 || matches!(event, Event::Put { ref data, .. } if data["generation"] == 2)
2486 );
2487 backend.put(&root, Value::Null).await.unwrap();
2488 }
2489
2490 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2491 async fn standard_32_path_profile_converges() {
2492 let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
2493 return;
2494 };
2495 let backend = Arc::new(
2496 RtdbBackend::new(format!("http://{host}"), "")
2497 .with_namespace("demo-rtdb-sync-default-rtdb"),
2498 );
2499 let mut handles = Vec::new();
2500 for path in 0..32 {
2501 let handle = start(
2502 backend.clone(),
2503 test_root(&format!("paths/{path}")),
2504 Config {
2505 retry: RetryPolicy::Never,
2506 ..Config::default()
2507 },
2508 );
2509 handles.push((path, handle));
2510 }
2511 for (path, handle) in &handles {
2512 wait_for_status(handle, SyncStatus::Connected).await;
2513 backend
2514 .put(
2515 &test_root(&format!("paths/{path}")),
2516 serde_json::json!({"path": path, "generation": 0}),
2517 )
2518 .await
2519 .unwrap();
2520 }
2521 for generation in 1..=50 {
2522 for path in 0..32 {
2523 let root = test_root(&format!("paths/{path}"));
2524 match generation % 3 {
2525 0 => backend
2526 .put(
2527 &root,
2528 serde_json::json!({"path": path, "generation": generation}),
2529 )
2530 .await
2531 .unwrap(),
2532 1 => backend
2533 .patch(
2534 &root,
2535 serde_json::json!({"generation": generation})
2536 .as_object()
2537 .unwrap()
2538 .clone(),
2539 )
2540 .await
2541 .unwrap(),
2542 _ => {
2543 backend.put(&root, Value::Null).await.unwrap();
2544 backend
2545 .put(
2546 &root,
2547 serde_json::json!({"path": path, "generation": generation}),
2548 )
2549 .await
2550 .unwrap();
2551 }
2552 }
2553 }
2554 }
2555 for (path, handle) in &handles {
2556 let mut updates = handle.subscribe();
2557 tokio::time::timeout(Duration::from_secs(10), async {
2558 while updates.borrow().value["generation"] != 50 {
2559 updates.changed().await.unwrap();
2560 }
2561 })
2562 .await
2563 .unwrap();
2564 assert_eq!(updates.borrow().value["path"], *path);
2565 }
2566 for (_, handle) in handles {
2567 handle.shutdown().await;
2568 }
2569 }
2570
2571 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
2572 async fn manual_heavy_64_path_profile_converges() {
2573 if std::env::var("RTDB_SYNC_HEAVY").ok().as_deref() != Some("1") {
2574 return;
2575 }
2576 let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
2577 let backend = Arc::new(
2578 RtdbBackend::new(format!("http://{host}"), "")
2579 .with_namespace("demo-rtdb-sync-default-rtdb"),
2580 );
2581 let mut handles = Vec::new();
2582 for path in 0..64 {
2583 let root = test_root(&format!("heavy/{path}"));
2584 backend
2585 .put(&root, serde_json::json!({"path": path, "generation": 0}))
2586 .await
2587 .unwrap();
2588 let stream_backend = Arc::new(
2589 RtdbBackend::new(format!("http://{host}"), "")
2590 .with_namespace("demo-rtdb-sync-default-rtdb"),
2591 );
2592 let handle = start(
2593 stream_backend,
2594 root,
2595 Config {
2596 retry: RetryPolicy::Never,
2597 ..Config::default()
2598 },
2599 );
2600 handles.push((path, handle));
2601 }
2602 for (_path, handle) in &handles {
2603 wait_for_status(handle, SyncStatus::Connected).await;
2604 }
2605 for generation in 1..=100 {
2606 let writes = (0..64).map(|path| {
2607 let backend = backend.clone();
2608 async move {
2609 let root = test_root(&format!("heavy/{path}"));
2610 if generation % 3 == 1 {
2611 backend
2612 .patch(
2613 &root,
2614 serde_json::json!({"generation": generation})
2615 .as_object()
2616 .unwrap()
2617 .clone(),
2618 )
2619 .await
2620 } else if generation % 3 == 2 {
2621 backend.put(&root, Value::Null).await?;
2622 backend
2623 .put(
2624 &root,
2625 serde_json::json!({"path": path, "generation": generation}),
2626 )
2627 .await
2628 } else {
2629 backend
2630 .put(
2631 &root,
2632 serde_json::json!({"path": path, "generation": generation}),
2633 )
2634 .await
2635 }
2636 }
2637 });
2638 for result in futures_util::future::join_all(writes).await {
2639 result.unwrap();
2640 }
2641 }
2642 for (path, handle) in &handles {
2643 let mut updates = handle.subscribe();
2644 tokio::time::timeout(Duration::from_secs(30), async {
2645 while updates.borrow().value["generation"] != 100 {
2646 updates.changed().await.unwrap();
2647 }
2648 })
2649 .await
2650 .unwrap();
2651 assert_eq!(updates.borrow().value["path"], *path);
2652 }
2653 for (_, handle) in handles {
2654 handle.shutdown().await;
2655 }
2656 }
2657
2658 #[tokio::test]
2659 async fn emulator_namespaces_are_isolated_when_configured() {
2660 let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
2661 return;
2662 };
2663 let left =
2664 RtdbBackend::new(format!("http://{host}"), "").with_namespace("rtdb-sync-ns-left");
2665 let right =
2666 RtdbBackend::new(format!("http://{host}"), "").with_namespace("rtdb-sync-ns-right");
2667 left.put("isolation", serde_json::json!({"owner": "left"}))
2668 .await
2669 .unwrap();
2670 right
2671 .put("isolation", serde_json::json!({"owner": "right"}))
2672 .await
2673 .unwrap();
2674 assert_eq!(left.get("isolation").await.unwrap()["owner"], "left");
2675 assert_eq!(right.get("isolation").await.unwrap()["owner"], "right");
2676 }
2677
2678 #[tokio::test]
2679 #[ignore = "run through scripts/test-emulator-recovery.sh"]
2680 async fn emulator_restart_rehydrates_and_recovers() {
2681 let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
2682 let ready = std::env::var("RTDB_RECOVERY_READY").expect("ready marker");
2683 let restored = std::env::var("RTDB_RECOVERY_RESTORED").expect("restored marker");
2684 let backend = Arc::new(
2685 RtdbBackend::new(format!("http://{host}"), "")
2686 .with_namespace("demo-rtdb-sync-default-rtdb"),
2687 );
2688 let mut handles = Vec::new();
2689 let path_count = std::env::var("RTDB_RECOVERY_PATHS")
2690 .ok()
2691 .and_then(|value| value.parse::<usize>().ok())
2692 .filter(|value| *value > 0)
2693 .unwrap_or(32);
2694 for path in 0..path_count {
2695 let root = test_root(&format!("restart/{path}"));
2696 emulator_put_retry(
2697 &backend,
2698 &root,
2699 serde_json::json!({"path": path, "generation": 1}),
2700 )
2701 .await
2702 .unwrap();
2703 let stream_backend = Arc::new(
2704 RtdbBackend::new(format!("http://{host}"), "")
2705 .with_namespace("demo-rtdb-sync-default-rtdb"),
2706 );
2707 handles.push((
2708 path,
2709 start(
2710 stream_backend,
2711 root,
2712 Config {
2713 retry: RetryPolicy::Exponential {
2714 max_attempts: Some(20),
2715 base: Duration::from_millis(100),
2716 max: Duration::from_secs(1),
2717 },
2718 ..Config::default()
2719 },
2720 ),
2721 ));
2722 }
2723 for (_, handle) in &handles {
2724 wait_for_status(handle, SyncStatus::Connected).await;
2725 }
2726 std::fs::write(&ready, "connected").unwrap();
2727 while !std::path::Path::new(&restored).exists() {
2728 tokio::time::sleep(Duration::from_millis(50)).await;
2729 }
2730 tokio::time::timeout(Duration::from_secs(30), async {
2731 loop {
2732 if handles
2733 .iter()
2734 .all(|(_, handle)| handle.metrics().reconnect_attempts >= 1)
2735 {
2736 break;
2737 }
2738 tokio::time::sleep(Duration::from_millis(50)).await;
2739 }
2740 })
2741 .await
2742 .unwrap();
2743 for path in 0..path_count {
2744 emulator_put_retry(
2745 &backend,
2746 &test_root(&format!("restart/{path}")),
2747 serde_json::json!({"path": path, "generation": 2}),
2748 )
2749 .await
2750 .unwrap();
2751 }
2752 for (path, handle) in &handles {
2753 let mut updates = handle.subscribe();
2754 tokio::time::timeout(Duration::from_secs(30), async {
2755 while updates.borrow().value["generation"] != 2 {
2756 updates.changed().await.unwrap();
2757 }
2758 })
2759 .await
2760 .unwrap();
2761 assert_eq!(updates.borrow().value["path"], *path);
2762 assert!(handle.metrics().reconnect_attempts >= 1);
2763 }
2764 for (_, handle) in handles {
2765 handle.shutdown().await;
2766 }
2767 }
2768
2769 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2770 async fn fanout_64_subscribers_converge() {
2771 let events = (1..=250)
2772 .map(|generation| Event::Put {
2773 path: "".into(),
2774 data: serde_json::json!({"generation": generation}),
2775 })
2776 .collect();
2777 let handle = start(
2778 Arc::new(mock(serde_json::json!({"generation": 0}), events)),
2779 "fanout",
2780 Config {
2781 retry: RetryPolicy::Never,
2782 ..Config::default()
2783 },
2784 );
2785 let mut subscribers = (0..64).map(|_| handle.subscribe()).collect::<Vec<_>>();
2786 for receiver in &mut subscribers {
2787 tokio::time::timeout(Duration::from_secs(2), async {
2788 while receiver.borrow().value["generation"] != 250 {
2789 receiver.changed().await.unwrap();
2790 }
2791 })
2792 .await
2793 .unwrap();
2794 assert_eq!(receiver.borrow().value["generation"], 250);
2795 }
2796 handle.shutdown().await;
2797 }
2798
2799 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2800 async fn lifecycle_churn_stops_one_hundred_tasks() {
2801 for _ in 0..100 {
2802 let handle = start(
2803 Arc::new(mock(Value::Null, vec![])),
2804 "churn",
2805 Config {
2806 retry: RetryPolicy::Never,
2807 ..Config::default()
2808 },
2809 );
2810 handle.shutdown().await;
2811 }
2812 }
2813
2814 #[tokio::test]
2815 async fn reconnect_status_metrics_and_backend_hook_are_observable() {
2816 let backend = Arc::new(Flaky {
2817 subscriptions: AtomicU64::new(0),
2818 replacements: AtomicU64::new(0),
2819 });
2820 let handle = start(
2821 backend.clone(),
2822 "flaky",
2823 Config {
2824 retry: RetryPolicy::Exponential {
2825 max_attempts: Some(3),
2826 base: Duration::from_millis(1),
2827 max: Duration::from_millis(5),
2828 },
2829 jitter_max: Duration::from_millis(1),
2830 ..Config::default()
2831 },
2832 );
2833 wait_for_status(&handle, SyncStatus::Connected).await;
2834 assert_eq!(backend.replacements.load(Ordering::Relaxed), 1);
2835 assert_eq!(handle.metrics().stream_failures, 1);
2836 assert_eq!(handle.metrics().reconnect_attempts, 1);
2837 handle.shutdown().await;
2838 }
2839
2840 #[tokio::test]
2841 async fn twenty_five_reconnect_cycles_preserve_state_and_subscribers() {
2842 let backend = Arc::new(Cycling {
2843 subscriptions: AtomicU64::new(0),
2844 });
2845 let handle = start(
2846 backend,
2847 "cycles",
2848 Config {
2849 retry: RetryPolicy::Exponential {
2850 max_attempts: Some(30),
2851 base: Duration::from_millis(1),
2852 max: Duration::from_millis(2),
2853 },
2854 ..Config::default()
2855 },
2856 );
2857 wait_for_status(&handle, SyncStatus::Connected).await;
2858 assert_eq!(handle.metrics().stream_failures, 25);
2859 assert_eq!(handle.metrics().reconnect_attempts, 25);
2860 handle.shutdown().await;
2861 }
2862
2863 #[tokio::test]
2864 async fn shutdown_cancels_hydration_backoff_promptly() {
2865 let handle = start(
2866 Arc::new(Failing),
2867 "backoff",
2868 Config {
2869 retry: RetryPolicy::Exponential {
2870 max_attempts: None,
2871 base: Duration::from_secs(30),
2872 max: Duration::from_secs(30),
2873 },
2874 ..Config::default()
2875 },
2876 );
2877 tokio::time::timeout(Duration::from_millis(100), handle.shutdown())
2878 .await
2879 .unwrap();
2880 }
2881
2882 #[tokio::test]
2883 async fn optimistic_write_rolls_back_on_backend_failure() {
2884 let handle = start(
2885 Arc::new(Failing),
2886 "failing",
2887 Config {
2888 write_policy: WritePolicy::Optimistic,
2889 retry: RetryPolicy::Never,
2890 ..Config::default()
2891 },
2892 );
2893 wait_for_status(&handle, SyncStatus::Connected).await;
2894 assert_eq!(
2895 handle.put("", serde_json::json!({"count": 2})).await,
2896 Err(SyncError::Backend("injected write failure".into()))
2897 );
2898 assert_eq!(handle.snapshot().value, serde_json::json!({"count": 1}));
2899 assert_eq!(handle.metrics().failed_writes, 1);
2900 handle.shutdown().await;
2901 }
2902
2903 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2904 async fn concurrent_local_and_remote_writers_converge_without_echo_loop() {
2905 let (events, _) = tokio::sync::broadcast::channel(256);
2906 let backend = Arc::new(Live {
2907 value: Arc::new(Mutex::new(serde_json::json!({"count": 0}))),
2908 events,
2909 });
2910 let handle = start(
2911 backend.clone(),
2912 "writers",
2913 Config {
2914 write_policy: WritePolicy::Optimistic,
2915 conflict_policy: ConflictPolicy::RemoteWins,
2916 retry: RetryPolicy::Never,
2917 ..Config::default()
2918 },
2919 );
2920 wait_for_status(&handle, SyncStatus::Connected).await;
2921 for count in 1..=100u64 {
2922 let local = handle.put("", serde_json::json!({"count": count}));
2923 let remote = backend.put("", serde_json::json!({"count": count + 10_000}));
2924 let (local_result, remote_result) = tokio::join!(local, remote);
2925 local_result.unwrap();
2926 remote_result.unwrap();
2927 }
2928 tokio::time::sleep(Duration::from_millis(25)).await;
2929 assert_eq!(
2930 handle.snapshot().value,
2931 backend.get("writers").await.unwrap()
2932 );
2933 assert_eq!(handle.metrics().successful_writes, 100);
2934 handle.shutdown().await;
2935 }
2936
2937 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
2938 async fn bidirectional_32_path_conflict_profile_converges() {
2939 let mut pairs = Vec::new();
2940 for path in 0..32u64 {
2941 let (events, _) = tokio::sync::broadcast::channel(512);
2942 let backend = Arc::new(Live {
2943 value: Arc::new(Mutex::new(
2944 serde_json::json!({"path": path, "local": 0, "remote": 0}),
2945 )),
2946 events,
2947 });
2948 let handle = start(
2949 backend.clone(),
2950 format!("bidirectional/{path}"),
2951 Config {
2952 write_policy: WritePolicy::Optimistic,
2953 conflict_policy: ConflictPolicy::RemoteWins,
2954 retry: RetryPolicy::Never,
2955 ..Config::default()
2956 },
2957 );
2958 pairs.push((path, backend, handle));
2959 }
2960 for (_, _, handle) in &pairs {
2961 wait_for_status(handle, SyncStatus::Connected).await;
2962 }
2963 for generation in 1..=100u64 {
2964 let writes = pairs.iter().map(|(_, backend, handle)| async move {
2965 let local = handle.put("", serde_json::json!({"local": generation}));
2966 let remote = backend.patch(
2967 "",
2968 serde_json::json!({"remote": generation})
2969 .as_object()
2970 .unwrap()
2971 .clone(),
2972 );
2973 let (local_result, remote_result) = tokio::join!(local, remote);
2974 local_result?;
2975 remote_result
2976 });
2977 for result in futures_util::future::join_all(writes).await {
2978 result.unwrap();
2979 }
2980 }
2981 tokio::time::sleep(Duration::from_millis(50)).await;
2982 for (_, backend, handle) in &pairs {
2983 assert_eq!(handle.snapshot().value, backend.get("").await.unwrap());
2984 assert_eq!(handle.metrics().successful_writes, 100);
2985 }
2986 for (_, _, handle) in pairs {
2987 handle.shutdown().await;
2988 }
2989 }
2990
2991 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
2992 async fn fan_in_eight_local_producers_preserves_acknowledgements() {
2993 let (events, _) = tokio::sync::broadcast::channel(512);
2994 let backend = Arc::new(Live {
2995 value: Arc::new(Mutex::new(serde_json::json!({"remote": 0}))),
2996 events,
2997 });
2998 let handle = start(
2999 backend.clone(),
3000 "fan-in",
3001 Config {
3002 write_policy: WritePolicy::Optimistic,
3003 retry: RetryPolicy::Never,
3004 ..Config::default()
3005 },
3006 );
3007 wait_for_status(&handle, SyncStatus::Connected).await;
3008 let producers = (0..8).map(|producer| {
3009 let handle = &handle;
3010 async move {
3011 for generation in 0..25u64 {
3012 handle
3013 .patch(
3014 "",
3015 serde_json::json!({ (format!("local-{producer}")): generation }),
3016 )
3017 .await
3018 .unwrap();
3019 }
3020 }
3021 });
3022 let _ = futures_util::future::join_all(producers).await;
3023 backend
3024 .patch(
3025 "",
3026 serde_json::json!({"remote": 1})
3027 .as_object()
3028 .unwrap()
3029 .clone(),
3030 )
3031 .await
3032 .unwrap();
3033 tokio::time::sleep(Duration::from_millis(25)).await;
3034 assert_eq!(handle.metrics().successful_writes, 200);
3035 assert_eq!(handle.snapshot().value, backend.get("").await.unwrap());
3036 handle.shutdown().await;
3037 }
3038
3039 async fn wait_for_status(handle: &SyncHandle, expected: SyncStatus) {
3040 let mut status = handle.subscribe_status();
3041 tokio::time::timeout(Duration::from_secs(5), async {
3042 while status.borrow().clone() != expected {
3043 status.changed().await.unwrap();
3044 }
3045 })
3046 .await
3047 .unwrap();
3048 }
3049
3050 fn test_root(label: &str) -> String {
3051 format!("rtdb-sync-tests/{}/{label}", std::process::id())
3052 }
3053
3054 async fn emulator_put_retry(
3055 backend: &RtdbBackend,
3056 path: &str,
3057 value: Value,
3058 ) -> Result<(), SyncError> {
3059 let mut last = None;
3060 for _ in 0..300 {
3061 match backend.put(path, value.clone()).await {
3062 Ok(()) => return Ok(()),
3063 Err(error) => {
3064 last = Some(error);
3065 tokio::time::sleep(Duration::from_millis(100)).await;
3066 }
3067 }
3068 }
3069 Err(last.unwrap_or_else(|| SyncError::Backend("emulator did not accept write".into())))
3070 }
3071
3072 #[test]
3073 fn file_persistence_is_versioned_atomic_and_namespace_isolated() {
3074 let root =
3075 std::env::temp_dir().join(format!("rtdb-sync-persistence-{}", std::process::id()));
3076 let _ = fs::remove_dir_all(&root);
3077 let store = FilePersistence::new(&root).unwrap();
3078 let key = "user-a/path";
3079 store
3080 .store_snapshot(&DurableSnapshot {
3081 format_version: PERSISTENCE_FORMAT_VERSION,
3082 sync_key: key.into(),
3083 generation: 7,
3084 value: serde_json::json!({"ok": true}),
3085 saved_at_ms: now_ms(),
3086 })
3087 .unwrap();
3088 store
3089 .append_mutation(&DurableMutation {
3090 format_version: PERSISTENCE_FORMAT_VERSION,
3091 sync_key: key.into(),
3092 id: 11,
3093 path: "child".into(),
3094 kind: DurableMutationKind::Patch,
3095 payload: serde_json::json!({"count": 1}),
3096 generation: 8,
3097 created_at_ms: now_ms(),
3098 })
3099 .unwrap();
3100 assert_eq!(store.load(key).unwrap().pending.len(), 1);
3101 assert!(store.load("other/path").unwrap().pending.is_empty());
3102 store.acknowledge_mutation(key, 11).unwrap();
3103 store.compact(key).unwrap();
3104 assert!(store.load(key).unwrap().pending.is_empty());
3105 let snapshot_path = store.path(key, "snapshot.json").unwrap();
3106 fs::write(snapshot_path, b"not-json").unwrap();
3107 assert!(matches!(store.load(key), Err(SyncError::Persistence(_))));
3108 store.clear(key).unwrap();
3109 let _ = fs::remove_dir_all(&root);
3110 }
3111
3112 #[tokio::test]
3113 async fn durable_queue_restores_offline_state_and_replays_after_restart() {
3114 let store = Arc::new(MemoryPersistence::default());
3115 let online = Arc::new(std::sync::atomic::AtomicBool::new(false));
3116 let (events, _) = tokio::sync::broadcast::channel(16);
3117 let backend = Arc::new(Recoverable {
3118 online: online.clone(),
3119 value: Arc::new(Mutex::new(serde_json::json!({"count": 0}))),
3120 events,
3121 });
3122 let config = Config {
3123 persistence: Some(store.clone()),
3124 persistence_key: Some("durable-test".into()),
3125 offline_policy: OfflinePolicy::QueueWithLimit { max_pending: 2 },
3126 write_policy: WritePolicy::Optimistic,
3127 retry: RetryPolicy::Exponential {
3128 max_attempts: None,
3129 base: Duration::from_millis(5),
3130 max: Duration::from_millis(20),
3131 },
3132 ..Config::default()
3133 };
3134 let first = start(backend.clone(), "root", config.clone());
3135 tokio::time::timeout(Duration::from_secs(1), async {
3136 loop {
3137 if first.status() == SyncStatus::Offline {
3138 break;
3139 }
3140 tokio::time::sleep(Duration::from_millis(5)).await;
3141 }
3142 })
3143 .await
3144 .unwrap();
3145 first
3146 .put("", serde_json::json!({"count": 1}))
3147 .await
3148 .unwrap();
3149 assert_eq!(first.metrics().pending_mutations, 1);
3150 first.shutdown().await;
3151 assert_eq!(store.load("durable-test").unwrap().pending.len(), 1);
3152
3153 online.store(true, Ordering::Relaxed);
3154 let second = start(backend.clone(), "root", config);
3155 tokio::time::timeout(Duration::from_secs(2), async {
3156 loop {
3157 if backend.get("root").await.unwrap()["count"] == 1
3158 && second.metrics().pending_mutations == 0
3159 && second.status() == SyncStatus::Connected
3160 {
3161 break;
3162 }
3163 tokio::time::sleep(Duration::from_millis(10)).await;
3164 }
3165 })
3166 .await
3167 .unwrap();
3168 assert_eq!(store.load("durable-test").unwrap().pending.len(), 0);
3169 second.shutdown().await;
3170 }
3171
3172 #[tokio::test]
3173 async fn offline_queue_limit_is_visible_and_missing_store_is_rejected() {
3174 let backend = Arc::new(Failing);
3175 let no_store = start(
3176 backend.clone(),
3177 "root",
3178 Config {
3179 offline_policy: OfflinePolicy::QueueWhileOffline,
3180 ..Config::default()
3181 },
3182 );
3183 tokio::time::timeout(Duration::from_secs(1), async {
3184 loop {
3185 if matches!(
3186 no_store.status(),
3187 SyncStatus::Failed(SyncError::Persistence(_))
3188 ) {
3189 break;
3190 }
3191 tokio::time::sleep(Duration::from_millis(5)).await;
3192 }
3193 })
3194 .await
3195 .unwrap();
3196 no_store.shutdown().await;
3197
3198 let store = Arc::new(MemoryPersistence::default());
3199 let online = Arc::new(std::sync::atomic::AtomicBool::new(false));
3200 let (events, _) = tokio::sync::broadcast::channel(4);
3201 let backend = Arc::new(Recoverable {
3202 online,
3203 value: Arc::new(Mutex::new(Value::Null)),
3204 events,
3205 });
3206 let handle = start(
3207 backend,
3208 "root",
3209 Config {
3210 persistence: Some(store),
3211 persistence_key: Some("limit".into()),
3212 offline_policy: OfflinePolicy::QueueWithLimit { max_pending: 1 },
3213 write_policy: WritePolicy::Optimistic,
3214 retry: RetryPolicy::Exponential {
3215 max_attempts: None,
3216 base: Duration::from_millis(5),
3217 max: Duration::from_millis(20),
3218 },
3219 ..Config::default()
3220 },
3221 );
3222 tokio::time::timeout(Duration::from_secs(1), async {
3223 while handle.status() != SyncStatus::Offline {
3224 tokio::time::sleep(Duration::from_millis(5)).await;
3225 }
3226 })
3227 .await
3228 .unwrap();
3229 handle.put("a", 1).await.unwrap();
3230 assert!(matches!(
3231 handle.put("b", 2).await,
3232 Err(SyncError::Persistence(_))
3233 ));
3234 handle.shutdown().await;
3235 }
3236
3237 #[tokio::test]
3238 async fn restored_pending_mutation_uses_explicit_remote_wins_policy() {
3239 let store = Arc::new(MemoryPersistence::default());
3240 store
3241 .store_snapshot(&DurableSnapshot {
3242 format_version: PERSISTENCE_FORMAT_VERSION,
3243 sync_key: "conflict".into(),
3244 generation: 1,
3245 value: serde_json::json!({"count": 1}),
3246 saved_at_ms: now_ms(),
3247 })
3248 .unwrap();
3249 store
3250 .append_mutation(&DurableMutation {
3251 format_version: PERSISTENCE_FORMAT_VERSION,
3252 sync_key: "conflict".into(),
3253 id: 1,
3254 path: "".into(),
3255 kind: DurableMutationKind::Put,
3256 payload: serde_json::json!({"count": 2}),
3257 generation: 2,
3258 created_at_ms: now_ms(),
3259 })
3260 .unwrap();
3261 let (events, _) = tokio::sync::broadcast::channel(4);
3262 let backend = Arc::new(Recoverable {
3263 online: Arc::new(std::sync::atomic::AtomicBool::new(true)),
3264 value: Arc::new(Mutex::new(serde_json::json!({"count": 9}))),
3265 events,
3266 });
3267 let handle = start(
3268 backend.clone(),
3269 "root",
3270 Config {
3271 persistence: Some(store.clone()),
3272 persistence_key: Some("conflict".into()),
3273 conflict_policy: ConflictPolicy::RemoteWins,
3274 ..Config::default()
3275 },
3276 );
3277 wait_for_status(&handle, SyncStatus::Connected).await;
3278 assert_eq!(backend.get("root").await.unwrap()["count"], 9);
3279 assert_eq!(handle.metrics().conflict_count, 1);
3280 assert_eq!(handle.metrics().pending_mutations, 0);
3281 assert!(store.load("conflict").unwrap().pending.is_empty());
3282 handle.shutdown().await;
3283 }
3284
3285 fn durable_emulator_config(store: Arc<FilePersistence>) -> Config {
3286 Config {
3287 persistence: Some(store),
3288 persistence_key: Some("durable-emulator-acceptance".into()),
3289 offline_policy: OfflinePolicy::QueueWhileOffline,
3290 write_policy: WritePolicy::Optimistic,
3291 retry: RetryPolicy::Exponential {
3292 max_attempts: None,
3293 base: Duration::from_millis(10),
3294 max: Duration::from_millis(50),
3295 },
3296 ..Config::default()
3297 }
3298 }
3299
3300 fn durable_emulator_store() -> Arc<FilePersistence> {
3301 Arc::new(
3302 FilePersistence::new(std::env::var("RTDB_DURABLE_STORE").expect("RTDB_DURABLE_STORE"))
3303 .unwrap(),
3304 )
3305 }
3306
3307 #[tokio::test]
3308 #[ignore = "run through scripts/test-emulator-offline.sh"]
3309 async fn durable_emulator_seed_persists_snapshot() {
3310 let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
3311 let store = durable_emulator_store();
3312 let backend = Arc::new(
3313 RtdbBackend::new(format!("http://{host}"), "")
3314 .with_namespace("demo-rtdb-sync-default-rtdb"),
3315 );
3316 emulator_put_retry(
3317 &backend,
3318 "durable-acceptance",
3319 serde_json::json!({"count": 1}),
3320 )
3321 .await
3322 .unwrap();
3323 let handle = start(
3324 backend,
3325 "durable-acceptance",
3326 durable_emulator_config(store.clone()),
3327 );
3328 wait_for_status(&handle, SyncStatus::Connected).await;
3329 handle.shutdown().await;
3330 assert!(store
3331 .load("durable-emulator-acceptance")
3332 .unwrap()
3333 .snapshot
3334 .is_some());
3335 }
3336
3337 #[tokio::test]
3338 #[ignore = "run through scripts/test-emulator-offline.sh"]
3339 async fn durable_emulator_queues_while_database_is_down() {
3340 let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
3341 let store = durable_emulator_store();
3342 let backend = Arc::new(
3343 RtdbBackend::new(format!("http://{host}"), "")
3344 .with_namespace("demo-rtdb-sync-default-rtdb"),
3345 );
3346 let handle = start(
3347 backend,
3348 "durable-acceptance",
3349 durable_emulator_config(store.clone()),
3350 );
3351 tokio::time::timeout(Duration::from_secs(5), async {
3352 loop {
3353 if handle.status() == SyncStatus::Offline {
3354 break;
3355 }
3356 tokio::time::sleep(Duration::from_millis(10)).await;
3357 }
3358 })
3359 .await
3360 .unwrap();
3361 handle
3362 .put("", serde_json::json!({"count": 2}))
3363 .await
3364 .unwrap();
3365 if std::env::var_os("RTDB_DURABLE_CRASH_AFTER_QUEUE").is_some() {
3366 std::process::exit(0);
3367 }
3368 handle.shutdown().await;
3369 assert_eq!(
3370 store
3371 .load("durable-emulator-acceptance")
3372 .unwrap()
3373 .pending
3374 .len(),
3375 1
3376 );
3377 }
3378
3379 #[tokio::test]
3380 #[ignore = "run through scripts/test-emulator-offline.sh"]
3381 async fn durable_emulator_replays_after_database_returns() {
3382 let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
3383 let store = durable_emulator_store();
3384 let backend = Arc::new(
3385 RtdbBackend::new(format!("http://{host}"), "")
3386 .with_namespace("demo-rtdb-sync-default-rtdb"),
3387 );
3388 let handle = start(
3389 backend.clone(),
3390 "durable-acceptance",
3391 durable_emulator_config(store.clone()),
3392 );
3393 tokio::time::timeout(Duration::from_secs(10), async {
3394 loop {
3395 if backend.get("durable-acceptance").await.unwrap()["count"] == 2
3396 && handle.metrics().pending_mutations == 0
3397 && handle.metrics().replay_successes == 1
3398 {
3399 break;
3400 }
3401 tokio::time::sleep(Duration::from_millis(20)).await;
3402 }
3403 })
3404 .await
3405 .unwrap();
3406 wait_for_status(&handle, SyncStatus::Connected).await;
3407 assert!(store
3408 .load("durable-emulator-acceptance")
3409 .unwrap()
3410 .pending
3411 .is_empty());
3412 handle.shutdown().await;
3413 }
3414
3415 #[test]
3416 fn retry_policy_is_bounded_and_paths_are_safe() {
3417 assert!(segments("a/../b").is_err());
3418 assert!(matches!(RetryPolicy::Never, RetryPolicy::Never));
3419 let policy = RetryPolicy::Exponential {
3420 max_attempts: Some(3),
3421 base: Duration::from_millis(10),
3422 max: Duration::from_millis(40),
3423 };
3424 assert_eq!(
3425 policy.delay(1, Duration::ZERO),
3426 Some(Duration::from_millis(10))
3427 );
3428 assert!(policy.delay(3, Duration::from_millis(5)).unwrap() <= Duration::from_millis(45));
3429 assert_eq!(policy.delay(4, Duration::ZERO), None);
3430 let mut value = Value::Null;
3431 assert!(apply_event(
3432 "root",
3433 &mut value,
3434 Event::Put {
3435 path: "a/../b".into(),
3436 data: Value::Null
3437 }
3438 )
3439 .is_err());
3440 }
3441
3442 #[test]
3443 fn conflict_policy_is_explicit_and_echoes_are_suppressed() {
3444 let (tx, rx) = watch::channel(Snapshot {
3445 generation: 0,
3446 value: serde_json::json!({"count": 2}),
3447 });
3448 let mut state = serde_json::json!({"count": 2});
3449 let mut pending = vec![PendingMutation {
3450 path: "".into(),
3451 event: Event::Put {
3452 path: "".into(),
3453 data: serde_json::json!({"count": 2}),
3454 },
3455 }];
3456 let mut generation = 0;
3457 reconcile_event(
3458 "root",
3459 &mut state,
3460 &mut generation,
3461 &tx,
3462 &mut pending,
3463 Event::Put {
3464 path: "".into(),
3465 data: serde_json::json!({"count": 2}),
3466 },
3467 ConflictPolicy::Reject,
3468 )
3469 .unwrap();
3470 assert_eq!(generation, 0);
3471 pending.push(PendingMutation {
3472 path: "".into(),
3473 event: Event::Put {
3474 path: "".into(),
3475 data: serde_json::json!({"count": 2}),
3476 },
3477 });
3478 reconcile_event(
3479 "root",
3480 &mut state,
3481 &mut generation,
3482 &tx,
3483 &mut pending,
3484 Event::Put {
3485 path: "".into(),
3486 data: serde_json::json!({"count": 3}),
3487 },
3488 ConflictPolicy::LocalWins,
3489 )
3490 .unwrap();
3491 assert_eq!(state["count"], 2);
3492 assert!(reconcile_event(
3493 "root",
3494 &mut state,
3495 &mut generation,
3496 &tx,
3497 &mut pending,
3498 Event::Put {
3499 path: "".into(),
3500 data: serde_json::json!({"count": 4})
3501 },
3502 ConflictPolicy::Reject
3503 )
3504 .is_err());
3505 drop(rx);
3506 }
3507}