Skip to main content

rtdb_sync/
lib.rs

1//! Realtime synchronized Rust state for Firebase Realtime Database.
2//!
3//! `rtdb-sync` owns local synchronization semantics.  A [`Backend`] supplies
4//! hydration, realtime events, and writes; the Firebase REST/SSE implementation
5//! is provided as [`RtdbBackend`].
6
7use async_trait::async_trait;
8use futures_core::Stream;
9use futures_util::StreamExt;
10use rtdb_rs::{RtdbClient, RtdbEvent};
11use serde::{de::DeserializeOwned, Serialize};
12use serde_json::{Map, Value};
13use std::{
14    fmt,
15    fs::{self, File, OpenOptions},
16    hash::{Hash, Hasher},
17    io::{BufRead, BufReader, Write as IoWrite},
18    path::{Path, PathBuf},
19    pin::Pin,
20    sync::{
21        atomic::{AtomicU64, Ordering},
22        Arc, Mutex,
23    },
24    time::{Duration, SystemTime, UNIX_EPOCH},
25};
26use thiserror::Error;
27use tokio::sync::{mpsc, watch};
28use tokio_util::sync::CancellationToken;
29
30pub type EventStream = Pin<Box<dyn Stream<Item = Result<Event, SyncError>> + Send>>;
31
32#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
33pub enum Event {
34    Put {
35        path: String,
36        data: Value,
37    },
38    Patch {
39        path: String,
40        data: Map<String, Value>,
41    },
42}
43
44#[derive(Debug, Error, Clone, PartialEq, Eq)]
45pub enum SyncError {
46    #[error("backend: {0}")]
47    Backend(String),
48    #[error("invalid event path: {0}")]
49    InvalidPath(String),
50    #[error("conversion failed: {0}")]
51    Conversion(String),
52    #[error("synchronization cancelled")]
53    Cancelled,
54    #[error("conflict: {0}")]
55    Conflict(String),
56    #[error("persistence: {0}")]
57    Persistence(String),
58}
59
60pub const PERSISTENCE_FORMAT_VERSION: u32 = 1;
61
62#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
63pub struct DurableSnapshot {
64    pub format_version: u32,
65    pub sync_key: String,
66    pub generation: u64,
67    pub value: Value,
68    pub saved_at_ms: u64,
69}
70
71#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
72pub enum DurableMutationKind {
73    Put,
74    Patch,
75}
76
77#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
78pub struct DurableMutation {
79    pub format_version: u32,
80    pub sync_key: String,
81    pub id: u64,
82    pub path: String,
83    pub kind: DurableMutationKind,
84    pub payload: Value,
85    pub generation: u64,
86    pub created_at_ms: u64,
87}
88
89#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)]
90pub struct SyncMetadata {
91    pub format_version: u32,
92    pub sync_key: String,
93    pub last_remote_event_at_ms: Option<u64>,
94    pub last_successful_sync_at_ms: Option<u64>,
95}
96
97#[derive(Debug, Clone, Default, PartialEq)]
98pub struct PersistedState {
99    pub snapshot: Option<DurableSnapshot>,
100    pub pending: Vec<DurableMutation>,
101    pub metadata: Option<SyncMetadata>,
102}
103
104pub trait PersistenceBackend: Send + Sync + fmt::Debug {
105    fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError>;
106    fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError>;
107    fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError>;
108    fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError>;
109    fn compact(&self, sync_key: &str) -> Result<(), SyncError>;
110    fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError>;
111    fn clear(&self, sync_key: &str) -> Result<(), SyncError>;
112}
113
114#[derive(Debug, Default, Clone)]
115pub struct MemoryPersistence {
116    entries: Arc<Mutex<std::collections::HashMap<String, PersistedState>>>,
117}
118
119impl PersistenceBackend for MemoryPersistence {
120    fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError> {
121        Ok(self
122            .entries
123            .lock()
124            .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?
125            .get(sync_key)
126            .cloned()
127            .unwrap_or_default())
128    }
129    fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError> {
130        let mut entries = self
131            .entries
132            .lock()
133            .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
134        let entry = entries.entry(snapshot.sync_key.clone()).or_default();
135        entry.snapshot = Some(snapshot.clone());
136        Ok(())
137    }
138    fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError> {
139        let mut entries = self
140            .entries
141            .lock()
142            .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
143        entries
144            .entry(mutation.sync_key.clone())
145            .or_default()
146            .pending
147            .push(mutation.clone());
148        Ok(())
149    }
150    fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError> {
151        let mut entries = self
152            .entries
153            .lock()
154            .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
155        if let Some(entry) = entries.get_mut(sync_key) {
156            entry.pending.retain(|mutation| mutation.id != mutation_id);
157        }
158        Ok(())
159    }
160    fn compact(&self, _sync_key: &str) -> Result<(), SyncError> {
161        Ok(())
162    }
163    fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError> {
164        let mut entries = self
165            .entries
166            .lock()
167            .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?;
168        entries
169            .entry(metadata.sync_key.clone())
170            .or_default()
171            .metadata = Some(metadata.clone());
172        Ok(())
173    }
174    fn clear(&self, sync_key: &str) -> Result<(), SyncError> {
175        self.entries
176            .lock()
177            .map_err(|_| SyncError::Persistence("memory store lock poisoned".into()))?
178            .remove(sync_key);
179        Ok(())
180    }
181}
182
183#[derive(Debug, Clone)]
184pub struct FilePersistence {
185    root: Arc<PathBuf>,
186}
187
188impl FilePersistence {
189    pub fn new(root: impl Into<PathBuf>) -> Result<Self, SyncError> {
190        let root = root.into();
191        fs::create_dir_all(&root).map_err(|error| persistence_io(&root, error))?;
192        restrict_permissions(&root, 0o700)?;
193        Ok(Self {
194            root: Arc::new(root),
195        })
196    }
197
198    fn directory(&self, sync_key: &str) -> Result<PathBuf, SyncError> {
199        let mut hasher = std::collections::hash_map::DefaultHasher::new();
200        sync_key.hash(&mut hasher);
201        let directory = self.root.join(format!("{:016x}", hasher.finish()));
202        fs::create_dir_all(&directory).map_err(|error| persistence_io(&directory, error))?;
203        restrict_permissions(&directory, 0o700)?;
204        Ok(directory)
205    }
206    fn path(&self, sync_key: &str, name: &str) -> Result<PathBuf, SyncError> {
207        Ok(self.directory(sync_key)?.join(name))
208    }
209}
210
211#[derive(Debug, serde::Serialize, serde::Deserialize)]
212enum JournalEntry {
213    Append(DurableMutation),
214    Acknowledge { sync_key: String, mutation_id: u64 },
215}
216
217impl PersistenceBackend for FilePersistence {
218    fn load(&self, sync_key: &str) -> Result<PersistedState, SyncError> {
219        let snapshot_path = self.path(sync_key, "snapshot.json")?;
220        let metadata_path = self.path(sync_key, "metadata.json")?;
221        let journal_path = self.path(sync_key, "journal.jsonl")?;
222        let snapshot = if snapshot_path.exists() {
223            let bytes =
224                fs::read(&snapshot_path).map_err(|error| persistence_io(&snapshot_path, error))?;
225            let value: DurableSnapshot = serde_json::from_slice(&bytes)
226                .map_err(|error| SyncError::Persistence(format!("invalid snapshot: {error}")))?;
227            validate_key_and_version(&value.format_version, &value.sync_key, sync_key)?;
228            Some(value)
229        } else {
230            None
231        };
232        let metadata = if metadata_path.exists() {
233            let bytes =
234                fs::read(&metadata_path).map_err(|error| persistence_io(&metadata_path, error))?;
235            let value: SyncMetadata = serde_json::from_slice(&bytes)
236                .map_err(|error| SyncError::Persistence(format!("invalid metadata: {error}")))?;
237            validate_key_and_version(&value.format_version, &value.sync_key, sync_key)?;
238            Some(value)
239        } else {
240            None
241        };
242        let mut pending = std::collections::BTreeMap::new();
243        if journal_path.exists() {
244            for line in BufReader::new(
245                File::open(&journal_path).map_err(|error| persistence_io(&journal_path, error))?,
246            )
247            .lines()
248            {
249                let line = line.map_err(|error| persistence_io(&journal_path, error))?;
250                let entry: JournalEntry = serde_json::from_str(&line).map_err(|error| {
251                    SyncError::Persistence(format!("invalid journal entry: {error}"))
252                })?;
253                match entry {
254                    JournalEntry::Append(mutation) => {
255                        validate_key_and_version(
256                            &mutation.format_version,
257                            &mutation.sync_key,
258                            sync_key,
259                        )?;
260                        pending.insert(mutation.id, mutation);
261                    }
262                    JournalEntry::Acknowledge {
263                        sync_key: entry_key,
264                        mutation_id,
265                    } => {
266                        if entry_key != sync_key {
267                            return Err(SyncError::Persistence(
268                                "journal namespace mismatch".into(),
269                            ));
270                        }
271                        pending.remove(&mutation_id);
272                    }
273                }
274            }
275        }
276        Ok(PersistedState {
277            snapshot,
278            pending: pending.into_values().collect(),
279            metadata,
280        })
281    }
282    fn store_snapshot(&self, snapshot: &DurableSnapshot) -> Result<(), SyncError> {
283        validate_key_and_version(
284            &snapshot.format_version,
285            &snapshot.sync_key,
286            &snapshot.sync_key,
287        )?;
288        let path = self.path(&snapshot.sync_key, "snapshot.json")?;
289        let tmp = path.with_extension("json.tmp");
290        let bytes = serde_json::to_vec(snapshot)
291            .map_err(|error| SyncError::Persistence(error.to_string()))?;
292        write_atomic(&path, &tmp, &bytes)
293    }
294    fn append_mutation(&self, mutation: &DurableMutation) -> Result<(), SyncError> {
295        validate_key_and_version(
296            &mutation.format_version,
297            &mutation.sync_key,
298            &mutation.sync_key,
299        )?;
300        let path = self.path(&mutation.sync_key, "journal.jsonl")?;
301        append_line(&path, &JournalEntry::Append(mutation.clone()))
302    }
303    fn acknowledge_mutation(&self, sync_key: &str, mutation_id: u64) -> Result<(), SyncError> {
304        let path = self.path(sync_key, "journal.jsonl")?;
305        append_line(
306            &path,
307            &JournalEntry::Acknowledge {
308                sync_key: sync_key.into(),
309                mutation_id,
310            },
311        )
312    }
313    fn compact(&self, sync_key: &str) -> Result<(), SyncError> {
314        let state = self.load(sync_key)?;
315        let path = self.path(sync_key, "journal.jsonl")?;
316        let tmp = path.with_extension("jsonl.tmp");
317        let mut bytes = Vec::new();
318        for mutation in state.pending {
319            serde_json::to_writer(&mut bytes, &JournalEntry::Append(mutation))
320                .map_err(|error| SyncError::Persistence(error.to_string()))?;
321            bytes.push(b'\n');
322        }
323        write_atomic(&path, &tmp, &bytes)
324    }
325    fn store_metadata(&self, metadata: &SyncMetadata) -> Result<(), SyncError> {
326        validate_key_and_version(
327            &metadata.format_version,
328            &metadata.sync_key,
329            &metadata.sync_key,
330        )?;
331        let path = self.path(&metadata.sync_key, "metadata.json")?;
332        let tmp = path.with_extension("json.tmp");
333        let bytes = serde_json::to_vec(metadata)
334            .map_err(|error| SyncError::Persistence(error.to_string()))?;
335        write_atomic(&path, &tmp, &bytes)
336    }
337    fn clear(&self, sync_key: &str) -> Result<(), SyncError> {
338        let directory = self.directory(sync_key)?;
339        if directory.exists() {
340            fs::remove_dir_all(&directory).map_err(|error| persistence_io(&directory, error))?;
341        }
342        Ok(())
343    }
344}
345
346fn now_ms() -> u64 {
347    SystemTime::now()
348        .duration_since(UNIX_EPOCH)
349        .unwrap_or_default()
350        .as_millis() as u64
351}
352
353fn validate_key_and_version(
354    version: &u32,
355    entry_key: &str,
356    expected_key: &str,
357) -> Result<(), SyncError> {
358    if *version != PERSISTENCE_FORMAT_VERSION {
359        return Err(SyncError::Persistence(format!(
360            "unsupported persistence format version {version}"
361        )));
362    }
363    if entry_key != expected_key {
364        return Err(SyncError::Persistence(
365            "persistence namespace mismatch".into(),
366        ));
367    }
368    Ok(())
369}
370
371fn persistence_io(path: &Path, error: std::io::Error) -> SyncError {
372    SyncError::Persistence(format!("{}: {error}", path.display()))
373}
374
375fn restrict_permissions(path: &Path, mode: u32) -> Result<(), SyncError> {
376    #[cfg(unix)]
377    {
378        use std::os::unix::fs::PermissionsExt;
379        fs::set_permissions(path, fs::Permissions::from_mode(mode))
380            .map_err(|error| persistence_io(path, error))?;
381    }
382    Ok(())
383}
384
385fn write_atomic(path: &Path, tmp: &Path, bytes: &[u8]) -> Result<(), SyncError> {
386    let mut file = OpenOptions::new()
387        .create(true)
388        .truncate(true)
389        .write(true)
390        .open(tmp)
391        .map_err(|error| persistence_io(tmp, error))?;
392    file.write_all(bytes)
393        .map_err(|error| persistence_io(tmp, error))?;
394    file.sync_all()
395        .map_err(|error| persistence_io(tmp, error))?;
396    restrict_permissions(tmp, 0o600)?;
397    fs::rename(tmp, path).map_err(|error| persistence_io(path, error))?;
398    if let Some(parent) = path.parent() {
399        if let Ok(directory) = File::open(parent) {
400            let _ = directory.sync_all();
401        }
402    }
403    Ok(())
404}
405
406fn append_line<T: serde::Serialize>(path: &Path, entry: &T) -> Result<(), SyncError> {
407    let mut file = OpenOptions::new()
408        .create(true)
409        .append(true)
410        .open(path)
411        .map_err(|error| persistence_io(path, error))?;
412    serde_json::to_writer(&mut file, entry)
413        .map_err(|error| SyncError::Persistence(error.to_string()))?;
414    file.write_all(b"\n")
415        .map_err(|error| persistence_io(path, error))?;
416    file.sync_all()
417        .map_err(|error| persistence_io(path, error))?;
418    restrict_permissions(path, 0o600)
419}
420
421async fn persistence_load(
422    store: Arc<dyn PersistenceBackend>,
423    sync_key: String,
424) -> Result<PersistedState, SyncError> {
425    tokio::task::spawn_blocking(move || store.load(&sync_key))
426        .await
427        .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
428}
429
430async fn persistence_snapshot(
431    store: Arc<dyn PersistenceBackend>,
432    snapshot: DurableSnapshot,
433) -> Result<(), SyncError> {
434    tokio::task::spawn_blocking(move || store.store_snapshot(&snapshot))
435        .await
436        .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
437}
438
439async fn persistence_append(
440    store: Arc<dyn PersistenceBackend>,
441    mutation: DurableMutation,
442) -> Result<(), SyncError> {
443    tokio::task::spawn_blocking(move || store.append_mutation(&mutation))
444        .await
445        .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
446}
447
448async fn persistence_ack(
449    store: Arc<dyn PersistenceBackend>,
450    sync_key: String,
451    mutation_id: u64,
452) -> Result<(), SyncError> {
453    tokio::task::spawn_blocking(move || store.acknowledge_mutation(&sync_key, mutation_id))
454        .await
455        .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
456}
457
458async fn persistence_compact(
459    store: Arc<dyn PersistenceBackend>,
460    sync_key: String,
461) -> Result<(), SyncError> {
462    tokio::task::spawn_blocking(move || store.compact(&sync_key))
463        .await
464        .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
465}
466
467async fn persistence_metadata(
468    store: Arc<dyn PersistenceBackend>,
469    metadata: SyncMetadata,
470) -> Result<(), SyncError> {
471    tokio::task::spawn_blocking(move || store.store_metadata(&metadata))
472        .await
473        .map_err(|error| SyncError::Persistence(format!("persistence worker failed: {error}")))?
474}
475
476#[async_trait]
477pub trait Backend: Send + Sync + 'static {
478    async fn get(&self, path: &str) -> Result<Value, SyncError>;
479    async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError>;
480    async fn put(&self, path: &str, value: Value) -> Result<(), SyncError>;
481    async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError>;
482    async fn before_reconnect(&self, _attempt: usize) -> Result<(), SyncError> {
483        Ok(())
484    }
485}
486
487#[derive(Debug, Clone, PartialEq)]
488pub struct Snapshot {
489    pub generation: u64,
490    pub value: Value,
491}
492
493impl Snapshot {
494    pub fn decode<T: DeserializeOwned>(&self) -> Result<T, SyncError> {
495        serde_json::from_value(self.value.clone()).map_err(|e| SyncError::Conversion(e.to_string()))
496    }
497}
498
499#[derive(Debug, Clone, PartialEq)]
500pub struct TypedSnapshot<T> {
501    pub generation: u64,
502    pub value: T,
503}
504
505pub struct TypedSyncHandle<T> {
506    inner: SyncHandle,
507    snapshot: watch::Receiver<Result<TypedSnapshot<T>, SyncError>>,
508    conversion_task: tokio::task::JoinHandle<()>,
509}
510
511impl<T: Clone> TypedSyncHandle<T> {
512    pub fn snapshot(&self) -> Result<TypedSnapshot<T>, SyncError> {
513        self.snapshot.borrow().clone()
514    }
515    pub fn subscribe(&self) -> watch::Receiver<Result<TypedSnapshot<T>, SyncError>> {
516        self.snapshot.clone()
517    }
518    pub fn status(&self) -> SyncStatus {
519        self.inner.status()
520    }
521    pub fn subscribe_status(&self) -> watch::Receiver<SyncStatus> {
522        self.inner.subscribe_status()
523    }
524    pub fn metrics(&self) -> MetricsSnapshot {
525        self.inner.metrics()
526    }
527    pub async fn put<V: Serialize>(
528        &self,
529        path: impl Into<String>,
530        value: V,
531    ) -> Result<(), SyncError> {
532        self.inner.put(path, value).await
533    }
534    pub async fn patch<V: Serialize>(
535        &self,
536        path: impl Into<String>,
537        value: V,
538    ) -> Result<(), SyncError> {
539        self.inner.patch(path, value).await
540    }
541    pub async fn shutdown(self) {
542        self.inner.shutdown().await;
543        let _ = self.conversion_task.await;
544    }
545}
546
547pub fn start_typed<B, T>(
548    backend: Arc<B>,
549    path: impl Into<String>,
550    config: Config,
551) -> TypedSyncHandle<T>
552where
553    B: Backend,
554    T: DeserializeOwned + Clone + Send + Sync + 'static,
555{
556    let inner = start(backend, path, config);
557    let mut raw = inner.subscribe();
558    let initial = raw.borrow().decode::<T>().map(|value| TypedSnapshot {
559        generation: raw.borrow().generation,
560        value,
561    });
562    let (snapshot, receiver) = watch::channel(initial);
563    let conversion_task = tokio::spawn(async move {
564        loop {
565            if raw.changed().await.is_err() {
566                break;
567            }
568            let current = raw.borrow().clone();
569            let converted = current.decode::<T>().map(|value| TypedSnapshot {
570                generation: current.generation,
571                value,
572            });
573            let _ = snapshot.send(converted);
574        }
575    });
576    TypedSyncHandle {
577        inner,
578        snapshot: receiver,
579        conversion_task,
580    }
581}
582
583#[derive(Debug, Clone, Copy, PartialEq, Eq)]
584pub enum RetryPolicy {
585    Never,
586    Exponential {
587        max_attempts: Option<usize>,
588        base: Duration,
589        max: Duration,
590    },
591}
592
593impl RetryPolicy {
594    pub fn delay(&self, attempt: usize, jitter_max: Duration) -> Option<Duration> {
595        match *self {
596            RetryPolicy::Never => None,
597            RetryPolicy::Exponential {
598                max_attempts,
599                base,
600                max,
601            } => {
602                if max_attempts.map(|n| attempt > n).unwrap_or(false) {
603                    return None;
604                }
605                let factor = 2u32.saturating_pow(attempt.saturating_sub(1) as u32);
606                let backoff = base.saturating_mul(factor).min(max);
607                let jitter = if jitter_max.is_zero() {
608                    Duration::ZERO
609                } else {
610                    let seed = (attempt as u64)
611                        .wrapping_mul(6364136223846793005)
612                        .wrapping_add(1442695040888963407);
613                    Duration::from_nanos(seed % (jitter_max.as_nanos() as u64 + 1))
614                };
615                Some(
616                    backoff
617                        .saturating_add(jitter)
618                        .min(max.saturating_add(jitter_max)),
619                )
620            }
621        }
622    }
623}
624
625#[derive(Debug, Clone, Copy, PartialEq, Eq)]
626pub enum WritePolicy {
627    Confirmed,
628    Optimistic,
629}
630
631#[derive(Debug, Clone, Copy, PartialEq, Eq)]
632pub enum ConflictPolicy {
633    RemoteWins,
634    LocalWins,
635    Reject,
636}
637
638#[derive(Debug, Clone, Copy, PartialEq, Eq)]
639pub enum OfflinePolicy {
640    RejectWhileOffline,
641    QueueWhileOffline,
642    QueueWithLimit { max_pending: usize },
643}
644
645#[derive(Debug, Clone)]
646pub struct Config {
647    pub retry: RetryPolicy,
648    pub write_policy: WritePolicy,
649    pub jitter_max: Duration,
650    pub conflict_policy: ConflictPolicy,
651    /// Maximum number of local mutations waiting for the synchronization task.
652    pub write_queue_capacity: usize,
653    pub persistence: Option<Arc<dyn PersistenceBackend>>,
654    pub persistence_key: Option<String>,
655    pub offline_policy: OfflinePolicy,
656}
657impl Default for Config {
658    fn default() -> Self {
659        Self {
660            retry: RetryPolicy::Exponential {
661                max_attempts: None,
662                base: Duration::from_millis(50),
663                max: Duration::from_secs(5),
664            },
665            write_policy: WritePolicy::Confirmed,
666            jitter_max: Duration::ZERO,
667            conflict_policy: ConflictPolicy::RemoteWins,
668            write_queue_capacity: 64,
669            persistence: None,
670            persistence_key: None,
671            offline_policy: OfflinePolicy::RejectWhileOffline,
672        }
673    }
674}
675
676#[derive(Debug, Clone, Default, PartialEq, Eq)]
677pub struct MetricsSnapshot {
678    pub reconnect_attempts: u64,
679    pub stream_failures: u64,
680    pub hydration_failures: u64,
681    pub successful_writes: u64,
682    pub failed_writes: u64,
683    pub pending_mutations: u64,
684    pub persistence_failures: u64,
685    pub replay_attempts: u64,
686    pub replay_successes: u64,
687    pub replay_failures: u64,
688    pub conflict_count: u64,
689}
690
691#[derive(Default)]
692struct Metrics {
693    reconnect_attempts: AtomicU64,
694    stream_failures: AtomicU64,
695    hydration_failures: AtomicU64,
696    successful_writes: AtomicU64,
697    failed_writes: AtomicU64,
698    pending_mutations: AtomicU64,
699    persistence_failures: AtomicU64,
700    replay_attempts: AtomicU64,
701    replay_successes: AtomicU64,
702    replay_failures: AtomicU64,
703    conflict_count: AtomicU64,
704}
705impl Metrics {
706    fn snapshot(&self) -> MetricsSnapshot {
707        MetricsSnapshot {
708            reconnect_attempts: self.reconnect_attempts.load(Ordering::Relaxed),
709            stream_failures: self.stream_failures.load(Ordering::Relaxed),
710            hydration_failures: self.hydration_failures.load(Ordering::Relaxed),
711            successful_writes: self.successful_writes.load(Ordering::Relaxed),
712            failed_writes: self.failed_writes.load(Ordering::Relaxed),
713            pending_mutations: self.pending_mutations.load(Ordering::Relaxed),
714            persistence_failures: self.persistence_failures.load(Ordering::Relaxed),
715            replay_attempts: self.replay_attempts.load(Ordering::Relaxed),
716            replay_successes: self.replay_successes.load(Ordering::Relaxed),
717            replay_failures: self.replay_failures.load(Ordering::Relaxed),
718            conflict_count: self.conflict_count.load(Ordering::Relaxed),
719        }
720    }
721}
722
723pub struct SyncHandle {
724    snapshot: watch::Receiver<Snapshot>,
725    status: watch::Receiver<SyncStatus>,
726    cancel: CancellationToken,
727    writes: mpsc::Sender<Write>,
728    join: tokio::task::JoinHandle<()>,
729    metrics: Arc<Metrics>,
730}
731
732enum Write {
733    Put(
734        String,
735        Value,
736        tokio::sync::oneshot::Sender<Result<(), SyncError>>,
737    ),
738    Patch(
739        String,
740        Map<String, Value>,
741        tokio::sync::oneshot::Sender<Result<(), SyncError>>,
742    ),
743}
744
745struct PendingMutation {
746    path: String,
747    event: Event,
748}
749
750impl SyncHandle {
751    pub fn snapshot(&self) -> Snapshot {
752        self.snapshot.borrow().clone()
753    }
754    pub fn subscribe(&self) -> watch::Receiver<Snapshot> {
755        self.snapshot.clone()
756    }
757    pub fn status(&self) -> SyncStatus {
758        self.status.borrow().clone()
759    }
760    pub fn subscribe_status(&self) -> watch::Receiver<SyncStatus> {
761        self.status.clone()
762    }
763    pub fn metrics(&self) -> MetricsSnapshot {
764        self.metrics.snapshot()
765    }
766    pub async fn put<T: Serialize>(
767        &self,
768        path: impl Into<String>,
769        value: T,
770    ) -> Result<(), SyncError> {
771        let (tx, rx) = tokio::sync::oneshot::channel();
772        let v = serde_json::to_value(value).map_err(|e| SyncError::Conversion(e.to_string()))?;
773        self.writes
774            .send(Write::Put(path.into(), v, tx))
775            .await
776            .map_err(|_| SyncError::Cancelled)?;
777        rx.await.map_err(|_| SyncError::Cancelled)?
778    }
779    pub async fn patch<T: Serialize>(
780        &self,
781        path: impl Into<String>,
782        value: T,
783    ) -> Result<(), SyncError> {
784        let (tx, rx) = tokio::sync::oneshot::channel();
785        let v = serde_json::to_value(value).map_err(|e| SyncError::Conversion(e.to_string()))?;
786        let data = v
787            .as_object()
788            .cloned()
789            .ok_or_else(|| SyncError::Conversion("patch must be an object".into()))?;
790        self.writes
791            .send(Write::Patch(path.into(), data, tx))
792            .await
793            .map_err(|_| SyncError::Cancelled)?;
794        rx.await.map_err(|_| SyncError::Cancelled)?
795    }
796    pub async fn shutdown(self) {
797        self.cancel.cancel();
798        let _ = self.join.await;
799    }
800}
801
802pub fn start<B: Backend>(backend: Arc<B>, path: impl Into<String>, config: Config) -> SyncHandle {
803    let path = path.into();
804    let persistence = config.persistence.clone();
805    let sync_key = config
806        .persistence_key
807        .clone()
808        .unwrap_or_else(|| path.clone());
809    let (snap_tx, snap_rx) = watch::channel(Snapshot {
810        generation: 0,
811        value: Value::Null,
812    });
813    let (status_tx, status_rx) = watch::channel(SyncStatus::Idle);
814    let cancel = CancellationToken::new();
815    let (writes, mut write_rx) = mpsc::channel(config.write_queue_capacity.max(1));
816    let child = cancel.clone();
817    let metrics = Arc::new(Metrics::default());
818    let task_metrics = metrics.clone();
819    let join = tokio::spawn(async move {
820        if matches!(
821            config.offline_policy,
822            OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
823        ) && persistence.is_none()
824        {
825            let _ = status_tx.send(SyncStatus::Failed(SyncError::Persistence(
826                "offline queue requires a persistence backend".into(),
827            )));
828            return;
829        }
830        #[allow(unused_assignments)]
831        let mut state: Option<Value> = None;
832        let mut generation = 0;
833        let mut attempt = 0usize;
834        let mut pending = Vec::new();
835        let mut durable_pending = Vec::new();
836        let mut restored_base = None;
837        if let Some(store) = persistence.as_ref() {
838            match persistence_load(store.clone(), sync_key.clone()).await {
839                Ok(restored) => {
840                    if let Some(snapshot) = restored.snapshot {
841                        state = Some(snapshot.value);
842                        restored_base = state.clone();
843                        generation = snapshot.generation;
844                        let _ = snap_tx.send(Snapshot {
845                            generation,
846                            value: state.as_ref().unwrap().clone(),
847                        });
848                        let _ = status_tx.send(SyncStatus::RestoredStale);
849                    }
850                    for mutation in restored.pending {
851                        match durable_event(&mutation) {
852                            Ok(event) => {
853                                pending.push(PendingMutation {
854                                    path: mutation.path.clone(),
855                                    event,
856                                });
857                                durable_pending.push(mutation);
858                            }
859                            Err(error) => {
860                                task_metrics
861                                    .persistence_failures
862                                    .fetch_add(1, Ordering::Relaxed);
863                                let _ = status_tx.send(SyncStatus::Failed(error));
864                                return;
865                            }
866                        }
867                    }
868                    task_metrics
869                        .pending_mutations
870                        .store(durable_pending.len() as u64, Ordering::Relaxed);
871                }
872                Err(error) => {
873                    task_metrics
874                        .persistence_failures
875                        .fetch_add(1, Ordering::Relaxed);
876                    let _ = status_tx.send(SyncStatus::Failed(error));
877                    return;
878                }
879            }
880        }
881        let _ = status_tx.send(SyncStatus::Hydrating);
882        loop {
883            let hydrated = tokio::select! {
884                _ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
885                result = backend.get(&path) => result
886            };
887            match hydrated {
888                Ok(value) => {
889                    state = Some(value);
890                    if let Some(base) = restored_base.take() {
891                        let remote = state.as_ref().expect("hydrated").clone();
892                        if let Err(error) = resolve_restored_conflicts(
893                            &base,
894                            &remote,
895                            config.conflict_policy,
896                            &mut pending,
897                            &mut durable_pending,
898                            persistence.as_ref(),
899                            &sync_key,
900                            &task_metrics,
901                        )
902                        .await
903                        {
904                            if matches!(error, SyncError::Conflict(_)) {
905                                task_metrics.conflict_count.fetch_add(1, Ordering::Relaxed);
906                                let _ = status_tx.send(SyncStatus::Conflict(error));
907                            } else {
908                                task_metrics
909                                    .persistence_failures
910                                    .fetch_add(1, Ordering::Relaxed);
911                                let _ = status_tx.send(SyncStatus::Failed(error));
912                            }
913                            return;
914                        }
915                        task_metrics
916                            .pending_mutations
917                            .store(durable_pending.len() as u64, Ordering::Relaxed);
918                    }
919                    let state_ref = state.as_ref().expect("hydration set state");
920                    generation += 1;
921                    let _ = snap_tx.send(Snapshot {
922                        generation,
923                        value: state_ref.clone(),
924                    });
925                    if let Some(store) = persistence.as_ref() {
926                        if let Err(error) = persistence_snapshot(
927                            store.clone(),
928                            DurableSnapshot {
929                                format_version: PERSISTENCE_FORMAT_VERSION,
930                                sync_key: sync_key.clone(),
931                                generation,
932                                value: state_ref.clone(),
933                                saved_at_ms: now_ms(),
934                            },
935                        )
936                        .await
937                        {
938                            task_metrics
939                                .persistence_failures
940                                .fetch_add(1, Ordering::Relaxed);
941                            let _ = status_tx.send(SyncStatus::Failed(error));
942                            return;
943                        }
944                        if let Err(error) = persistence_metadata(
945                            store.clone(),
946                            SyncMetadata {
947                                format_version: PERSISTENCE_FORMAT_VERSION,
948                                sync_key: sync_key.clone(),
949                                last_remote_event_at_ms: None,
950                                last_successful_sync_at_ms: Some(now_ms()),
951                            },
952                        )
953                        .await
954                        {
955                            task_metrics
956                                .persistence_failures
957                                .fetch_add(1, Ordering::Relaxed);
958                            let _ = status_tx.send(SyncStatus::Failed(error));
959                            return;
960                        }
961                    }
962                    if !durable_pending.is_empty() {
963                        let _ = status_tx.send(SyncStatus::Replaying);
964                        if replay_pending(
965                            &*backend,
966                            &path,
967                            state.as_mut().unwrap(),
968                            &mut generation,
969                            &snap_tx,
970                            &mut pending,
971                            &mut durable_pending,
972                            persistence.as_ref(),
973                            &sync_key,
974                            &task_metrics,
975                        )
976                        .await
977                        .is_err()
978                        {
979                            let _ = status_tx.send(SyncStatus::Offline);
980                            continue;
981                        }
982                    }
983                    attempt = 0;
984                }
985                Err(e) => {
986                    task_metrics
987                        .hydration_failures
988                        .fetch_add(1, Ordering::Relaxed);
989                    let offline_store = if matches!(
990                        config.offline_policy,
991                        OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
992                    ) {
993                        persistence.as_ref()
994                    } else {
995                        None
996                    };
997                    if let Some(store) = offline_store {
998                        let _ = status_tx.send(SyncStatus::Offline);
999                        let delay = config
1000                            .retry
1001                            .delay(attempt.saturating_add(1), config.jitter_max)
1002                            .unwrap_or(Duration::from_secs(1));
1003                        tokio::select! {
1004                            _ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
1005                            Some(write) = write_rx.recv() => {
1006                                let result = handle_offline_write(
1007                                    write,
1008                                    state.get_or_insert(Value::Null),
1009                                    &mut generation,
1010                                    &snap_tx,
1011                                    &mut pending,
1012                                    &mut durable_pending,
1013                                    store,
1014                                    &sync_key,
1015                                    config.offline_policy,
1016                                ).await;
1017                                if result.is_ok() { task_metrics.successful_writes.fetch_add(1, Ordering::Relaxed); }
1018                                else { task_metrics.failed_writes.fetch_add(1, Ordering::Relaxed); }
1019                                task_metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed);
1020                            }
1021                            _ = tokio::time::sleep(delay) => {
1022                                attempt = attempt.saturating_add(1);
1023                                task_metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
1024                                let _ = backend.before_reconnect(attempt).await;
1025                            }
1026                        }
1027                        continue;
1028                    }
1029                    if !retry(
1030                        &*backend,
1031                        &config,
1032                        &mut attempt,
1033                        &child,
1034                        &status_tx,
1035                        &task_metrics,
1036                    )
1037                    .await
1038                    {
1039                        let _ = status_tx.send(SyncStatus::Failed(e));
1040                        return;
1041                    }
1042                    continue;
1043                }
1044            }
1045            let mut stream = match backend.subscribe(&path).await {
1046                Ok(s) => s,
1047                Err(e) => {
1048                    let queue_offline = matches!(
1049                        config.offline_policy,
1050                        OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
1051                    );
1052                    if queue_offline {
1053                        if let Some(store) = persistence.as_ref() {
1054                            if !reconnect_or_queue(
1055                                &*backend,
1056                                &config,
1057                                &mut attempt,
1058                                &child,
1059                                &status_tx,
1060                                &task_metrics,
1061                                &mut write_rx,
1062                                &mut state,
1063                                &mut generation,
1064                                &snap_tx,
1065                                &mut pending,
1066                                &mut durable_pending,
1067                                store,
1068                                &sync_key,
1069                            )
1070                            .await
1071                            {
1072                                let _ = status_tx.send(SyncStatus::Failed(e));
1073                                return;
1074                            }
1075                            continue;
1076                        }
1077                    }
1078                    if !retry(
1079                        &*backend,
1080                        &config,
1081                        &mut attempt,
1082                        &child,
1083                        &status_tx,
1084                        &task_metrics,
1085                    )
1086                    .await
1087                    {
1088                        let _ = status_tx.send(SyncStatus::Failed(e));
1089                        return;
1090                    }
1091                    continue;
1092                }
1093            };
1094            let _ = status_tx.send(SyncStatus::Connected);
1095            loop {
1096                tokio::select! {
1097                    _ = child.cancelled() => { let _ = status_tx.send(SyncStatus::Stopped); return; },
1098                Some(write) = write_rx.recv() => { let result = handle_write(&*backend, &path, state.as_mut().expect("hydrated"), &mut generation, &snap_tx, &mut pending, write, config.write_policy, persistence.as_ref(), &sync_key, &mut durable_pending).await; task_metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed); if result.is_ok() { task_metrics.successful_writes.fetch_add(1, Ordering::Relaxed); } else { task_metrics.failed_writes.fetch_add(1, Ordering::Relaxed); let _ = status_tx.send(SyncStatus::Failed(result.clone().unwrap_err())); } },
1099                event = stream.next() => match event {
1100                    Some(Ok(event)) => match reconcile_event(&path, state.as_mut().expect("hydrated"), &mut generation, &snap_tx, &mut pending, event, config.conflict_policy) {
1101                        Ok(()) => {
1102                            if let Some(store) = persistence.as_ref() {
1103                                if let Err(error) = persistence_snapshot(store.clone(), DurableSnapshot {
1104                                    format_version: PERSISTENCE_FORMAT_VERSION,
1105                                    sync_key: sync_key.clone(),
1106                                    generation,
1107                                    value: state.as_ref().expect("hydrated").clone(),
1108                                    saved_at_ms: now_ms(),
1109                                }).await {
1110                                    task_metrics.persistence_failures.fetch_add(1, Ordering::Relaxed);
1111                                    let _ = status_tx.send(SyncStatus::Failed(error));
1112                                    return;
1113                                }
1114                            }
1115                        },
1116                        Err(e) => { let _ = status_tx.send(SyncStatus::Failed(e)); return; }
1117                    },
1118                    Some(Err(_)) | None => break
1119                }
1120                }
1121            }
1122            task_metrics.stream_failures.fetch_add(1, Ordering::Relaxed);
1123            let queue_offline = matches!(
1124                config.offline_policy,
1125                OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
1126            );
1127            if queue_offline {
1128                if let Some(store) = persistence.as_ref() {
1129                    if !reconnect_or_queue(
1130                        &*backend,
1131                        &config,
1132                        &mut attempt,
1133                        &child,
1134                        &status_tx,
1135                        &task_metrics,
1136                        &mut write_rx,
1137                        &mut state,
1138                        &mut generation,
1139                        &snap_tx,
1140                        &mut pending,
1141                        &mut durable_pending,
1142                        store,
1143                        &sync_key,
1144                    )
1145                    .await
1146                    {
1147                        let _ = status_tx.send(SyncStatus::Stopped);
1148                        return;
1149                    }
1150                    continue;
1151                }
1152            }
1153            if !retry(
1154                &*backend,
1155                &config,
1156                &mut attempt,
1157                &child,
1158                &status_tx,
1159                &task_metrics,
1160            )
1161            .await
1162            {
1163                let _ = status_tx.send(SyncStatus::Stopped);
1164                return;
1165            }
1166        }
1167    });
1168    SyncHandle {
1169        snapshot: snap_rx,
1170        status: status_rx,
1171        cancel,
1172        writes,
1173        join,
1174        metrics,
1175    }
1176}
1177
1178async fn retry<B: Backend + ?Sized>(
1179    backend: &B,
1180    config: &Config,
1181    attempt: &mut usize,
1182    cancel: &CancellationToken,
1183    status: &watch::Sender<SyncStatus>,
1184    metrics: &Metrics,
1185) -> bool {
1186    *attempt += 1;
1187    let Some(delay) = config.retry.delay(*attempt, config.jitter_max) else {
1188        return false;
1189    };
1190    metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
1191    let _ = status.send(SyncStatus::Reconnecting {
1192        attempt: *attempt,
1193        delay,
1194    });
1195    if backend.before_reconnect(*attempt).await.is_err() {
1196        return false;
1197    }
1198    tokio::select! { _ = cancel.cancelled() => false, _ = tokio::time::sleep(delay) => true }
1199}
1200
1201fn durable_event(mutation: &DurableMutation) -> Result<Event, SyncError> {
1202    validate_key_and_version(
1203        &mutation.format_version,
1204        &mutation.sync_key,
1205        &mutation.sync_key,
1206    )?;
1207    match mutation.kind {
1208        DurableMutationKind::Put => Ok(Event::Put {
1209            path: mutation.path.clone(),
1210            data: mutation.payload.clone(),
1211        }),
1212        DurableMutationKind::Patch => Ok(Event::Patch {
1213            path: mutation.path.clone(),
1214            data: mutation.payload.as_object().cloned().ok_or_else(|| {
1215                SyncError::Persistence("journal patch payload is not an object".into())
1216            })?,
1217        }),
1218    }
1219}
1220
1221fn durable_mutation(
1222    sync_key: &str,
1223    pending: &[DurableMutation],
1224    event: &Event,
1225    generation: u64,
1226) -> Option<DurableMutation> {
1227    let (path, kind, payload) = match event {
1228        Event::Put { path, data } => (path.clone(), DurableMutationKind::Put, data.clone()),
1229        Event::Patch { path, data } => (
1230            path.clone(),
1231            DurableMutationKind::Patch,
1232            Value::Object(data.clone()),
1233        ),
1234    };
1235    Some(DurableMutation {
1236        format_version: PERSISTENCE_FORMAT_VERSION,
1237        sync_key: sync_key.into(),
1238        id: pending
1239            .iter()
1240            .map(|entry| entry.id)
1241            .max()
1242            .unwrap_or(0)
1243            .saturating_add(1),
1244        path,
1245        kind,
1246        payload,
1247        generation,
1248        created_at_ms: now_ms(),
1249    })
1250}
1251
1252fn event_would_change(root: &str, state: &Value, event: &Event) -> Result<bool, SyncError> {
1253    let mut candidate = state.clone();
1254    apply_event(root, &mut candidate, event.clone())?;
1255    Ok(candidate != *state)
1256}
1257
1258#[allow(clippy::too_many_arguments)]
1259async fn handle_offline_write(
1260    write: Write,
1261    state: &mut Value,
1262    generation: &mut u64,
1263    tx: &watch::Sender<Snapshot>,
1264    pending: &mut Vec<PendingMutation>,
1265    durable_pending: &mut Vec<DurableMutation>,
1266    persistence: &Arc<dyn PersistenceBackend>,
1267    sync_key: &str,
1268    policy: OfflinePolicy,
1269) -> Result<(), SyncError> {
1270    if let OfflinePolicy::QueueWithLimit { max_pending } = policy {
1271        if durable_pending.len() >= max_pending {
1272            let error = SyncError::Persistence("offline mutation queue is full".into());
1273            match write {
1274                Write::Put(_, _, done) | Write::Patch(_, _, done) => {
1275                    let _ = done.send(Err(error.clone()));
1276                }
1277            }
1278            return Err(error);
1279        }
1280    }
1281    let (path, kind, payload, done) = match write {
1282        Write::Put(path, payload, done) => (path, DurableMutationKind::Put, payload, done),
1283        Write::Patch(path, payload, done) => (
1284            path,
1285            DurableMutationKind::Patch,
1286            Value::Object(payload),
1287            done,
1288        ),
1289    };
1290    let event = match kind {
1291        DurableMutationKind::Put => Event::Put {
1292            path: path.clone(),
1293            data: payload.clone(),
1294        },
1295        DurableMutationKind::Patch => Event::Patch {
1296            path: path.clone(),
1297            data: payload.as_object().cloned().unwrap_or_default(),
1298        },
1299    };
1300    let mut next = state.clone();
1301    if matches!(
1302        policy,
1303        OfflinePolicy::QueueWhileOffline | OfflinePolicy::QueueWithLimit { .. }
1304    ) {
1305        apply_event("", &mut next, event.clone())?;
1306    }
1307    let mutation = DurableMutation {
1308        format_version: PERSISTENCE_FORMAT_VERSION,
1309        sync_key: sync_key.into(),
1310        id: durable_pending
1311            .iter()
1312            .map(|entry| entry.id)
1313            .max()
1314            .unwrap_or(0)
1315            .saturating_add(1),
1316        path: path.clone(),
1317        kind,
1318        payload,
1319        generation: generation.saturating_add(1),
1320        created_at_ms: now_ms(),
1321    };
1322    if let Err(error) = persistence_append(persistence.clone(), mutation.clone()).await {
1323        let _ = done.send(Err(error.clone()));
1324        return Err(error);
1325    }
1326    *state = next;
1327    *generation = mutation.generation;
1328    let _ = tx.send(Snapshot {
1329        generation: *generation,
1330        value: state.clone(),
1331    });
1332    pending.push(PendingMutation { path, event });
1333    durable_pending.push(mutation);
1334    if let Err(error) = persistence_snapshot(
1335        persistence.clone(),
1336        DurableSnapshot {
1337            format_version: PERSISTENCE_FORMAT_VERSION,
1338            sync_key: sync_key.into(),
1339            generation: *generation,
1340            value: state.clone(),
1341            saved_at_ms: now_ms(),
1342        },
1343    )
1344    .await
1345    {
1346        let _ = done.send(Err(error.clone()));
1347        return Err(error);
1348    }
1349    let _ = done.send(Ok(()));
1350    Ok(())
1351}
1352
1353#[allow(clippy::too_many_arguments)]
1354async fn replay_pending<B: Backend + ?Sized>(
1355    backend: &B,
1356    root: &str,
1357    state: &mut Value,
1358    generation: &mut u64,
1359    tx: &watch::Sender<Snapshot>,
1360    pending: &mut Vec<PendingMutation>,
1361    durable_pending: &mut Vec<DurableMutation>,
1362    persistence: Option<&Arc<dyn PersistenceBackend>>,
1363    sync_key: &str,
1364    metrics: &Metrics,
1365) -> Result<(), SyncError> {
1366    for mutation in durable_pending.clone() {
1367        metrics.replay_attempts.fetch_add(1, Ordering::Relaxed);
1368        let remote_path = join(root, &mutation.path)?;
1369        let event = durable_event(&mutation)?;
1370        let result = match mutation.kind {
1371            DurableMutationKind::Put => backend.put(&remote_path, mutation.payload.clone()).await,
1372            DurableMutationKind::Patch => {
1373                backend
1374                    .patch(
1375                        &remote_path,
1376                        mutation.payload.as_object().cloned().ok_or_else(|| {
1377                            SyncError::Persistence("journal patch payload is not an object".into())
1378                        })?,
1379                    )
1380                    .await
1381            }
1382        };
1383        if let Err(error) = result {
1384            metrics.replay_failures.fetch_add(1, Ordering::Relaxed);
1385            return Err(error);
1386        }
1387        apply_event(root, state, event.clone())?;
1388        *generation += 1;
1389        let _ = tx.send(Snapshot {
1390            generation: *generation,
1391            value: state.clone(),
1392        });
1393        if let Some(store) = persistence {
1394            persistence_snapshot(
1395                store.clone(),
1396                DurableSnapshot {
1397                    format_version: PERSISTENCE_FORMAT_VERSION,
1398                    sync_key: sync_key.into(),
1399                    generation: *generation,
1400                    value: state.clone(),
1401                    saved_at_ms: now_ms(),
1402                },
1403            )
1404            .await?;
1405            persistence_ack(store.clone(), sync_key.into(), mutation.id).await?;
1406        }
1407        if let Some(index) = durable_pending
1408            .iter()
1409            .position(|entry| entry.id == mutation.id)
1410        {
1411            durable_pending.remove(index);
1412        }
1413        if let Some(index) = pending
1414            .iter()
1415            .position(|entry| equivalent_event(&entry.event, &event))
1416        {
1417            pending.remove(index);
1418        }
1419        metrics.replay_successes.fetch_add(1, Ordering::Relaxed);
1420    }
1421    if let Some(store) = persistence {
1422        persistence_compact(store.clone(), sync_key.into()).await?;
1423    }
1424    metrics
1425        .pending_mutations
1426        .store(durable_pending.len() as u64, Ordering::Relaxed);
1427    Ok(())
1428}
1429
1430#[allow(clippy::too_many_arguments)]
1431async fn resolve_restored_conflicts(
1432    base: &Value,
1433    remote: &Value,
1434    policy: ConflictPolicy,
1435    pending: &mut Vec<PendingMutation>,
1436    durable_pending: &mut Vec<DurableMutation>,
1437    persistence: Option<&Arc<dyn PersistenceBackend>>,
1438    sync_key: &str,
1439    metrics: &Metrics,
1440) -> Result<(), SyncError> {
1441    let mut conflicted = false;
1442    for mutation in durable_pending.clone() {
1443        let event = durable_event(&mutation)?;
1444        if base != remote && event_would_change("", base, &event)? {
1445            conflicted = true;
1446            match policy {
1447                ConflictPolicy::RemoteWins => {
1448                    if let Some(store) = persistence {
1449                        persistence_ack(store.clone(), sync_key.into(), mutation.id).await?;
1450                    }
1451                    durable_pending.retain(|entry| entry.id != mutation.id);
1452                    if let Some(index) = pending
1453                        .iter()
1454                        .position(|entry| equivalent_event(&entry.event, &event))
1455                    {
1456                        pending.remove(index);
1457                    }
1458                }
1459                ConflictPolicy::LocalWins => {}
1460                ConflictPolicy::Reject => {
1461                    return Err(SyncError::Conflict(format!(
1462                        "restored mutation conflicts at {}",
1463                        mutation.path
1464                    )))
1465                }
1466            }
1467        }
1468    }
1469    if conflicted {
1470        metrics.conflict_count.fetch_add(1, Ordering::Relaxed);
1471        if let Some(store) = persistence {
1472            persistence_compact(store.clone(), sync_key.into()).await?;
1473        }
1474    }
1475    Ok(())
1476}
1477
1478#[allow(clippy::too_many_arguments)]
1479async fn reconnect_or_queue<B: Backend + ?Sized>(
1480    backend: &B,
1481    config: &Config,
1482    attempt: &mut usize,
1483    cancel: &CancellationToken,
1484    status: &watch::Sender<SyncStatus>,
1485    metrics: &Metrics,
1486    write_rx: &mut mpsc::Receiver<Write>,
1487    state: &mut Option<Value>,
1488    generation: &mut u64,
1489    snapshot_tx: &watch::Sender<Snapshot>,
1490    pending: &mut Vec<PendingMutation>,
1491    durable_pending: &mut Vec<DurableMutation>,
1492    persistence: &Arc<dyn PersistenceBackend>,
1493    sync_key: &str,
1494) -> bool {
1495    loop {
1496        *attempt = attempt.saturating_add(1);
1497        let delay = config
1498            .retry
1499            .delay(*attempt, config.jitter_max)
1500            .unwrap_or(Duration::from_secs(1));
1501        metrics.reconnect_attempts.fetch_add(1, Ordering::Relaxed);
1502        let _ = status.send(SyncStatus::Offline);
1503        tokio::select! {
1504            _ = cancel.cancelled() => return false,
1505            Some(write) = write_rx.recv() => {
1506                let result = handle_offline_write(
1507                    write,
1508                    state.get_or_insert(Value::Null),
1509                    generation,
1510                    snapshot_tx,
1511                    pending,
1512                    durable_pending,
1513                    persistence,
1514                    sync_key,
1515                    config.offline_policy,
1516                ).await;
1517                if result.is_ok() {
1518                    metrics.successful_writes.fetch_add(1, Ordering::Relaxed);
1519                } else {
1520                    metrics.failed_writes.fetch_add(1, Ordering::Relaxed);
1521                }
1522                metrics.pending_mutations.store(durable_pending.len() as u64, Ordering::Relaxed);
1523            }
1524            _ = tokio::time::sleep(delay) => {
1525                if backend.before_reconnect(*attempt).await.is_ok() {
1526                    return true;
1527                }
1528            }
1529        }
1530    }
1531}
1532
1533#[allow(clippy::too_many_arguments)]
1534async fn handle_write<B: Backend + ?Sized>(
1535    backend: &B,
1536    root: &str,
1537    state: &mut Value,
1538    generation: &mut u64,
1539    tx: &watch::Sender<Snapshot>,
1540    pending: &mut Vec<PendingMutation>,
1541    write: Write,
1542    policy: WritePolicy,
1543    persistence: Option<&Arc<dyn PersistenceBackend>>,
1544    sync_key: &str,
1545    durable_pending: &mut Vec<DurableMutation>,
1546) -> Result<(), SyncError> {
1547    match write {
1548        Write::Put(path, value, done) => {
1549            let old = state.clone();
1550            let backend_path = join(root, &path)?;
1551            let event = Event::Put {
1552                path: path.clone(),
1553                data: value.clone(),
1554            };
1555            let durable = durable_mutation(sync_key, durable_pending, &event, *generation + 1);
1556            if let Some(store) = persistence {
1557                let mutation = durable.clone().unwrap();
1558                if let Err(error) = persistence_append(Arc::clone(store), mutation.clone()).await {
1559                    let _ = done.send(Err(error.clone()));
1560                    return Err(error);
1561                }
1562                durable_pending.push(mutation);
1563            }
1564            if policy == WritePolicy::Optimistic {
1565                let mut next = state.clone();
1566                apply_event(root, &mut next, event.clone())?;
1567                *state = next;
1568                *generation += 1;
1569                let _ = tx.send(Snapshot {
1570                    generation: *generation,
1571                    value: state.clone(),
1572                });
1573                if let Some(store) = persistence {
1574                    if let Err(error) = persistence_snapshot(
1575                        Arc::clone(store),
1576                        DurableSnapshot {
1577                            format_version: PERSISTENCE_FORMAT_VERSION,
1578                            sync_key: sync_key.into(),
1579                            generation: *generation,
1580                            value: state.clone(),
1581                            saved_at_ms: now_ms(),
1582                        },
1583                    )
1584                    .await
1585                    {
1586                        let _ = done.send(Err(error.clone()));
1587                        return Err(error);
1588                    }
1589                }
1590            }
1591            pending.push(PendingMutation {
1592                path: path.clone(),
1593                event: event.clone(),
1594            });
1595            let result = backend.put(&backend_path, value).await;
1596            if result.is_ok() {
1597                if let (Some(store), Some(mutation)) = (persistence, durable) {
1598                    if let Err(error) =
1599                        persistence_ack(Arc::clone(store), sync_key.into(), mutation.id).await
1600                    {
1601                        let _ = done.send(Err(error.clone()));
1602                        return Err(error);
1603                    }
1604                    if let Err(error) =
1605                        persistence_compact(Arc::clone(store), sync_key.into()).await
1606                    {
1607                        let _ = done.send(Err(error.clone()));
1608                        return Err(error);
1609                    }
1610                    durable_pending.retain(|entry| entry.id != mutation.id);
1611                }
1612            }
1613            if result.is_err() && policy == WritePolicy::Optimistic {
1614                *state = old;
1615                *generation += 1;
1616                let _ = tx.send(Snapshot {
1617                    generation: *generation,
1618                    value: state.clone(),
1619                });
1620            }
1621            if result.is_err() {
1622                let mut removed = false;
1623                pending.retain(|mutation| {
1624                    if !removed && equivalent_event(&mutation.event, &event) {
1625                        removed = true;
1626                        false
1627                    } else {
1628                        true
1629                    }
1630                });
1631            }
1632            let _ = done.send(result.clone());
1633            result
1634        }
1635        Write::Patch(path, data, done) => {
1636            let old = state.clone();
1637            let backend_path = join(root, &path)?;
1638            let event = Event::Patch {
1639                path: path.clone(),
1640                data: data.clone(),
1641            };
1642            let durable = durable_mutation(sync_key, durable_pending, &event, *generation + 1);
1643            if let Some(store) = persistence {
1644                let mutation = durable.clone().unwrap();
1645                if let Err(error) = persistence_append(Arc::clone(store), mutation.clone()).await {
1646                    let _ = done.send(Err(error.clone()));
1647                    return Err(error);
1648                }
1649                durable_pending.push(mutation);
1650            }
1651            if policy == WritePolicy::Optimistic {
1652                let mut next = state.clone();
1653                apply_event(root, &mut next, event.clone())?;
1654                *state = next;
1655                *generation += 1;
1656                let _ = tx.send(Snapshot {
1657                    generation: *generation,
1658                    value: state.clone(),
1659                });
1660                if let Some(store) = persistence {
1661                    if let Err(error) = persistence_snapshot(
1662                        Arc::clone(store),
1663                        DurableSnapshot {
1664                            format_version: PERSISTENCE_FORMAT_VERSION,
1665                            sync_key: sync_key.into(),
1666                            generation: *generation,
1667                            value: state.clone(),
1668                            saved_at_ms: now_ms(),
1669                        },
1670                    )
1671                    .await
1672                    {
1673                        let _ = done.send(Err(error.clone()));
1674                        return Err(error);
1675                    }
1676                }
1677            }
1678            pending.push(PendingMutation {
1679                path: path.clone(),
1680                event: event.clone(),
1681            });
1682            let result = backend.patch(&backend_path, data).await;
1683            if result.is_ok() {
1684                if let (Some(store), Some(mutation)) = (persistence, durable) {
1685                    if let Err(error) =
1686                        persistence_ack(Arc::clone(store), sync_key.into(), mutation.id).await
1687                    {
1688                        let _ = done.send(Err(error.clone()));
1689                        return Err(error);
1690                    }
1691                    if let Err(error) =
1692                        persistence_compact(Arc::clone(store), sync_key.into()).await
1693                    {
1694                        let _ = done.send(Err(error.clone()));
1695                        return Err(error);
1696                    }
1697                    durable_pending.retain(|entry| entry.id != mutation.id);
1698                }
1699            }
1700            if result.is_err() && policy == WritePolicy::Optimistic {
1701                *state = old;
1702                *generation += 1;
1703                let _ = tx.send(Snapshot {
1704                    generation: *generation,
1705                    value: state.clone(),
1706                });
1707            }
1708            if result.is_err() {
1709                let mut removed = false;
1710                pending.retain(|mutation| {
1711                    if !removed && equivalent_event(&mutation.event, &event) {
1712                        removed = true;
1713                        false
1714                    } else {
1715                        true
1716                    }
1717                });
1718            }
1719            let _ = done.send(result.clone());
1720            result
1721        }
1722    }
1723}
1724
1725fn reconcile_event(
1726    root: &str,
1727    state: &mut Value,
1728    generation: &mut u64,
1729    tx: &watch::Sender<Snapshot>,
1730    pending: &mut Vec<PendingMutation>,
1731    event: Event,
1732    policy: ConflictPolicy,
1733) -> Result<(), SyncError> {
1734    if let Some(index) = pending
1735        .iter()
1736        .position(|mutation| mutation.path == event_path(&event))
1737    {
1738        let mutation = &pending[index];
1739        if equivalent_event(&mutation.event, &event) {
1740            pending.remove(index);
1741            return Ok(());
1742        }
1743        match policy {
1744            ConflictPolicy::LocalWins => return Ok(()),
1745            ConflictPolicy::Reject => {
1746                return Err(SyncError::Conflict(format!(
1747                    "remote event conflicts at {}",
1748                    event_path(&event)
1749                )))
1750            }
1751            ConflictPolicy::RemoteWins => {
1752                pending.remove(index);
1753            }
1754        }
1755    }
1756    apply_event(root, state, event)?;
1757    *generation += 1;
1758    let _ = tx.send(Snapshot {
1759        generation: *generation,
1760        value: state.clone(),
1761    });
1762    Ok(())
1763}
1764
1765fn event_path(event: &Event) -> &str {
1766    match event {
1767        Event::Put { path, .. } | Event::Patch { path, .. } => path,
1768    }
1769}
1770fn equivalent_event(left: &Event, right: &Event) -> bool {
1771    match (left, right) {
1772        (Event::Put { path: a, data: b }, Event::Put { path: c, data: d }) => a == c && b == d,
1773        (Event::Patch { path: a, data: b }, Event::Patch { path: c, data: d }) => a == c && b == d,
1774        _ => false,
1775    }
1776}
1777
1778fn join(root: &str, path: &str) -> Result<String, SyncError> {
1779    let a = root.trim_matches('/');
1780    let b = path.trim_matches('/');
1781    if !b.is_empty() && b.split('/').any(|x| x.is_empty() || x == "." || x == "..") {
1782        return Err(SyncError::InvalidPath(path.into()));
1783    }
1784    Ok(if b.is_empty() {
1785        a.into()
1786    } else if a.is_empty() {
1787        b.into()
1788    } else {
1789        format!("{a}/{b}")
1790    })
1791}
1792fn segments(path: &str) -> Result<Vec<&str>, SyncError> {
1793    let p = path.trim_matches('/');
1794    if p.is_empty() {
1795        Ok(vec![])
1796    } else {
1797        let out: Vec<_> = p.split('/').collect();
1798        if out.iter().any(|x| x.is_empty() || *x == "." || *x == "..") {
1799            Err(SyncError::InvalidPath(path.into()))
1800        } else {
1801            Ok(out)
1802        }
1803    }
1804}
1805pub fn apply_event(root: &str, state: &mut Value, event: Event) -> Result<(), SyncError> {
1806    let relative = match &event {
1807        Event::Put { path, .. } | Event::Patch { path, .. } => path.clone(),
1808    };
1809    let _ = join(root, &relative)?;
1810    let parts = segments(&relative)?;
1811    let target = get_or_create(state, &parts);
1812    match event {
1813        Event::Put { data, .. } => {
1814            if data.is_null() && !parts.is_empty() {
1815                remove_at(state, &parts);
1816            } else {
1817                *target = data;
1818            }
1819        }
1820        Event::Patch { data, .. } => {
1821            if data.is_empty() {
1822                return Ok(());
1823            }
1824            if !target.is_object() {
1825                *target = Value::Object(Map::new());
1826            }
1827            let obj = target.as_object_mut().unwrap();
1828            for (key, value) in data {
1829                let keys = segments(&key)?;
1830                set_at(obj, &keys, value);
1831            }
1832        }
1833    }
1834    Ok(())
1835}
1836fn get_or_create<'a>(value: &'a mut Value, parts: &[&str]) -> &'a mut Value {
1837    let mut current = value;
1838    for part in parts {
1839        if !current.is_object() {
1840            *current = Value::Object(Map::new());
1841        }
1842        current = current
1843            .as_object_mut()
1844            .unwrap()
1845            .entry(*part)
1846            .or_insert(Value::Null);
1847    }
1848    current
1849}
1850fn set_at(obj: &mut Map<String, Value>, parts: &[&str], value: Value) {
1851    if parts.is_empty() {
1852        return;
1853    }
1854    if parts.len() == 1 {
1855        if value.is_null() {
1856            obj.remove(parts[0]);
1857        } else {
1858            obj.insert(parts[0].into(), value);
1859        }
1860    } else {
1861        let child = obj
1862            .entry(parts[0])
1863            .or_insert_with(|| Value::Object(Map::new()));
1864        if !child.is_object() {
1865            *child = Value::Object(Map::new());
1866        }
1867        set_at(child.as_object_mut().unwrap(), &parts[1..], value);
1868    }
1869}
1870fn remove_at(value: &mut Value, parts: &[&str]) {
1871    if parts.is_empty() {
1872        *value = Value::Null;
1873        return;
1874    }
1875    if let Some(obj) = value.as_object_mut() {
1876        if parts.len() == 1 {
1877            obj.remove(parts[0]);
1878        } else if let Some(child) = obj.get_mut(parts[0]) {
1879            remove_at(child, &parts[1..]);
1880        }
1881    }
1882}
1883
1884/// Marker used while the 0.1.0 synchronization API is being designed.
1885///
1886/// No stability guarantee is attached to this pre-release scaffold.
1887#[derive(Debug, Clone, PartialEq, Eq)]
1888pub enum SyncStatus {
1889    /// No synchronization task has started.
1890    Idle,
1891    /// Initial state is being hydrated.
1892    Hydrating,
1893    /// A durable snapshot is available, but remote freshness is unconfirmed.
1894    RestoredStale,
1895    /// The remote backend is unavailable while local state remains usable.
1896    Offline,
1897    /// Durable local mutations are being replayed.
1898    Replaying,
1899    /// A restored local mutation conflicts with newly hydrated remote state.
1900    Conflict(SyncError),
1901    /// A realtime stream is active.
1902    Connected,
1903    /// The stream or hydration request is being retried.
1904    Reconnecting { attempt: usize, delay: Duration },
1905    /// Synchronization has stopped.
1906    Stopped,
1907    /// A non-recoverable backend, conversion, or event error occurred.
1908    Failed(SyncError),
1909}
1910
1911impl fmt::Display for SyncStatus {
1912    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1913        write!(f, "{self:?}")
1914    }
1915}
1916
1917/// Backend adapter that delegates REST, SSE, authentication parameters, and
1918/// emulator namespaces to the upstream `rtdb-rs` transport.
1919#[derive(Clone)]
1920pub struct RtdbBackend {
1921    client: Arc<RtdbClient>,
1922    base_url: String,
1923    token: String,
1924    namespace: Option<String>,
1925}
1926
1927/// Typed transport adapter. `rtdb-typed` converts complete PUT events and
1928/// models; partial PATCH payloads intentionally remain JSON until the local
1929/// state has applied them.
1930pub struct TypedBackend<T> {
1931    client: Arc<rtdb_typed::TypedClient>,
1932    marker: std::marker::PhantomData<fn() -> T>,
1933}
1934
1935impl<T> Clone for TypedBackend<T> {
1936    fn clone(&self) -> Self {
1937        Self {
1938            client: self.client.clone(),
1939            marker: std::marker::PhantomData,
1940        }
1941    }
1942}
1943
1944impl<T> TypedBackend<T>
1945where
1946    T: Serialize + DeserializeOwned + Send + Sync + 'static,
1947{
1948    pub fn new(client: rtdb_typed::TypedClient) -> Self {
1949        Self {
1950            client: Arc::new(client),
1951            marker: std::marker::PhantomData,
1952        }
1953    }
1954    pub fn from_parts(base_url: impl Into<String>, token: impl Into<String>) -> Self {
1955        Self::new(rtdb_typed::TypedClient::from_parts(base_url, token))
1956    }
1957}
1958
1959#[async_trait]
1960impl<T> Backend for TypedBackend<T>
1961where
1962    T: Serialize + DeserializeOwned + Send + Sync + 'static,
1963{
1964    async fn get(&self, path: &str) -> Result<Value, SyncError> {
1965        self.client
1966            .get::<T>(path)
1967            .await
1968            .map_err(RtdbBackend::error)
1969            .and_then(|value| {
1970                serde_json::to_value(value)
1971                    .map_err(|error| SyncError::Conversion(error.to_string()))
1972            })
1973    }
1974    async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError> {
1975        let stream = self
1976            .client
1977            .query::<T>(path)
1978            .stream()
1979            .await
1980            .map_err(RtdbBackend::error)?;
1981        let stream = stream.filter_map(|event| async move {
1982            match event {
1983                Ok(rtdb_typed::TypedEvent::Put { path, data }) => Some(
1984                    serde_json::to_value(data)
1985                        .map(|data| Event::Put { path, data })
1986                        .map_err(|error| SyncError::Conversion(error.to_string())),
1987                ),
1988                Ok(rtdb_typed::TypedEvent::Patch { path, data }) => Some(Ok(Event::Patch {
1989                    path,
1990                    data: data.as_object().cloned().unwrap_or_default(),
1991                })),
1992                Ok(rtdb_typed::TypedEvent::KeepAlive) => None,
1993                Ok(rtdb_typed::TypedEvent::Cancel) => {
1994                    Some(Err(SyncError::Backend("Firebase stream cancelled".into())))
1995                }
1996                Err(error) => Some(Err(RtdbBackend::error(error))),
1997            }
1998        });
1999        Ok(Box::pin(stream))
2000    }
2001    async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
2002        self.client
2003            .inner()
2004            .put(path, &value)
2005            .await
2006            .map(|_| ())
2007            .map_err(RtdbBackend::error)
2008    }
2009    async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2010        self.client
2011            .inner()
2012            .patch(path, &Value::Object(value))
2013            .await
2014            .map(|_| ())
2015            .map_err(RtdbBackend::error)
2016    }
2017}
2018
2019impl RtdbBackend {
2020    pub fn new(base_url: impl Into<String>, token: impl Into<String>) -> Self {
2021        let base_url = base_url.into();
2022        let token = token.into();
2023        Self {
2024            client: Arc::new(RtdbClient::new(&base_url, &token)),
2025            base_url,
2026            token,
2027            namespace: None,
2028        }
2029    }
2030    pub fn with_namespace(mut self, namespace: impl Into<String>) -> Self {
2031        let namespace = namespace.into();
2032        self.client = Arc::new(
2033            RtdbClient::new(&self.base_url, &self.token).with_namespace(namespace.clone()),
2034        );
2035        self.namespace = Some(namespace);
2036        self
2037    }
2038    pub fn typed_client(&self) -> rtdb_typed::TypedClient {
2039        let client = RtdbClient::new(&self.base_url, &self.token);
2040        let client = match &self.namespace {
2041            Some(namespace) => client.with_namespace(namespace.clone()),
2042            None => client,
2043        };
2044        rtdb_typed::TypedClient::new(client)
2045    }
2046    fn error(error: impl fmt::Debug) -> SyncError {
2047        SyncError::Backend(format!("{error:?}"))
2048    }
2049}
2050
2051#[async_trait]
2052impl Backend for RtdbBackend {
2053    async fn get(&self, path: &str) -> Result<Value, SyncError> {
2054        self.client.get(path).await.map_err(Self::error)
2055    }
2056    async fn subscribe(&self, path: &str) -> Result<EventStream, SyncError> {
2057        let stream = self.client.stream(path).await.map_err(Self::error)?;
2058        let stream = stream.filter_map(|event| async move {
2059            match event {
2060                Ok(RtdbEvent::Put { path, data }) => Some(Ok(Event::Put { path, data })),
2061                Ok(RtdbEvent::Patch { path, data }) => Some(Ok(Event::Patch {
2062                    path,
2063                    data: data.as_object().cloned().unwrap_or_default(),
2064                })),
2065                Ok(RtdbEvent::KeepAlive) => None,
2066                Ok(RtdbEvent::Cancel) => {
2067                    Some(Err(SyncError::Backend("Firebase stream cancelled".into())))
2068                }
2069                Err(error) => Some(Err(Self::error(error))),
2070            }
2071        });
2072        Ok(Box::pin(stream))
2073    }
2074    async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
2075        self.client
2076            .put(path, &value)
2077            .await
2078            .map(|_| ())
2079            .map_err(Self::error)
2080    }
2081    async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2082        self.client
2083            .patch(path, &Value::Object(value))
2084            .await
2085            .map(|_| ())
2086            .map_err(Self::error)
2087    }
2088}
2089
2090#[cfg(test)]
2091mod tests {
2092    use super::*;
2093    use futures_util::stream;
2094    use serde::Deserialize;
2095    use std::sync::Mutex;
2096
2097    #[derive(Clone)]
2098    struct Mock {
2099        value: Arc<Mutex<Value>>,
2100        events: Arc<Mutex<Vec<Event>>>,
2101        writes: Arc<Mutex<Vec<String>>>,
2102    }
2103
2104    struct Flaky {
2105        subscriptions: AtomicU64,
2106        replacements: AtomicU64,
2107    }
2108
2109    struct Cycling {
2110        subscriptions: AtomicU64,
2111    }
2112    #[async_trait]
2113    impl Backend for Cycling {
2114        async fn get(&self, _: &str) -> Result<Value, SyncError> {
2115            Ok(Value::Null)
2116        }
2117        async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2118            if self.subscriptions.fetch_add(1, Ordering::Relaxed) < 25 {
2119                Ok(Box::pin(stream::empty()))
2120            } else {
2121                Ok(Box::pin(stream::pending()))
2122            }
2123        }
2124        async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
2125            Ok(())
2126        }
2127        async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
2128            Ok(())
2129        }
2130    }
2131
2132    struct Failing;
2133    #[async_trait]
2134    impl Backend for Failing {
2135        async fn get(&self, _: &str) -> Result<Value, SyncError> {
2136            Ok(serde_json::json!({"count": 1}))
2137        }
2138        async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2139            Ok(Box::pin(stream::pending()))
2140        }
2141        async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
2142            Err(SyncError::Backend("injected write failure".into()))
2143        }
2144        async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
2145            Err(SyncError::Backend("injected write failure".into()))
2146        }
2147    }
2148
2149    #[derive(Clone)]
2150    struct Recoverable {
2151        online: Arc<std::sync::atomic::AtomicBool>,
2152        value: Arc<Mutex<Value>>,
2153        events: tokio::sync::broadcast::Sender<Event>,
2154    }
2155
2156    #[async_trait]
2157    impl Backend for Recoverable {
2158        async fn get(&self, _: &str) -> Result<Value, SyncError> {
2159            if !self.online.load(Ordering::Relaxed) {
2160                return Err(SyncError::Backend("offline".into()));
2161            }
2162            Ok(self.value.lock().unwrap().clone())
2163        }
2164        async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2165            if !self.online.load(Ordering::Relaxed) {
2166                return Err(SyncError::Backend("offline".into()));
2167            }
2168            let mut receiver = self.events.subscribe();
2169            Ok(Box::pin(async_stream::stream! {
2170                loop { match receiver.recv().await { Ok(event) => yield Ok(event), Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, Err(tokio::sync::broadcast::error::RecvError::Closed) => break } }
2171            }))
2172        }
2173        async fn put(&self, _: &str, value: Value) -> Result<(), SyncError> {
2174            if !self.online.load(Ordering::Relaxed) {
2175                return Err(SyncError::Backend("offline".into()));
2176            }
2177            *self.value.lock().unwrap() = value.clone();
2178            let _ = self.events.send(Event::Put {
2179                path: "".into(),
2180                data: value,
2181            });
2182            Ok(())
2183        }
2184        async fn patch(&self, _: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2185            if !self.online.load(Ordering::Relaxed) {
2186                return Err(SyncError::Backend("offline".into()));
2187            }
2188            apply_event(
2189                "",
2190                &mut self.value.lock().unwrap(),
2191                Event::Patch {
2192                    path: "".into(),
2193                    data: value.clone(),
2194                },
2195            )?;
2196            let _ = self.events.send(Event::Patch {
2197                path: "".into(),
2198                data: value,
2199            });
2200            Ok(())
2201        }
2202    }
2203
2204    #[derive(Clone)]
2205    struct Live {
2206        value: Arc<Mutex<Value>>,
2207        events: tokio::sync::broadcast::Sender<Event>,
2208    }
2209    #[async_trait]
2210    impl Backend for Live {
2211        async fn get(&self, _: &str) -> Result<Value, SyncError> {
2212            Ok(self.value.lock().unwrap().clone())
2213        }
2214        async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2215            let mut receiver = self.events.subscribe();
2216            Ok(Box::pin(async_stream::stream! {
2217                loop { match receiver.recv().await { Ok(event) => yield Ok(event), Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, Err(tokio::sync::broadcast::error::RecvError::Closed) => break } }
2218            }))
2219        }
2220        async fn put(&self, _: &str, value: Value) -> Result<(), SyncError> {
2221            *self.value.lock().unwrap() = value.clone();
2222            let _ = self.events.send(Event::Put {
2223                path: "".into(),
2224                data: value,
2225            });
2226            Ok(())
2227        }
2228        async fn patch(&self, _: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2229            let mut current = self.value.lock().unwrap();
2230            apply_event(
2231                "",
2232                &mut current,
2233                Event::Patch {
2234                    path: "".into(),
2235                    data: value.clone(),
2236                },
2237            )?;
2238            let _ = self.events.send(Event::Patch {
2239                path: "".into(),
2240                data: value,
2241            });
2242            Ok(())
2243        }
2244    }
2245
2246    #[async_trait]
2247    impl Backend for Flaky {
2248        async fn get(&self, _: &str) -> Result<Value, SyncError> {
2249            Ok(Value::Null)
2250        }
2251        async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2252            if self.subscriptions.fetch_add(1, Ordering::Relaxed) == 0 {
2253                Ok(Box::pin(stream::empty()))
2254            } else {
2255                Ok(Box::pin(stream::pending()))
2256            }
2257        }
2258        async fn put(&self, _: &str, _: Value) -> Result<(), SyncError> {
2259            Ok(())
2260        }
2261        async fn patch(&self, _: &str, _: Map<String, Value>) -> Result<(), SyncError> {
2262            Ok(())
2263        }
2264        async fn before_reconnect(&self, _: usize) -> Result<(), SyncError> {
2265            self.replacements.fetch_add(1, Ordering::Relaxed);
2266            Ok(())
2267        }
2268    }
2269
2270    #[async_trait]
2271    impl Backend for Mock {
2272        async fn get(&self, _: &str) -> Result<Value, SyncError> {
2273            Ok(self.value.lock().unwrap().clone())
2274        }
2275        async fn subscribe(&self, _: &str) -> Result<EventStream, SyncError> {
2276            let events = std::mem::take(&mut *self.events.lock().unwrap());
2277            Ok(Box::pin(
2278                stream::iter(events.into_iter().map(Ok)).chain(stream::pending()),
2279            ))
2280        }
2281        async fn put(&self, path: &str, value: Value) -> Result<(), SyncError> {
2282            *self.value.lock().unwrap() = value;
2283            self.writes.lock().unwrap().push(format!("put:{path}"));
2284            Ok(())
2285        }
2286        async fn patch(&self, path: &str, value: Map<String, Value>) -> Result<(), SyncError> {
2287            apply_event(
2288                "",
2289                &mut self.value.lock().unwrap(),
2290                Event::Patch {
2291                    path: "".into(),
2292                    data: value,
2293                },
2294            )?;
2295            self.writes.lock().unwrap().push(format!("patch:{path}"));
2296            Ok(())
2297        }
2298    }
2299
2300    fn mock(value: Value, events: Vec<Event>) -> Mock {
2301        Mock {
2302            value: Arc::new(Mutex::new(value)),
2303            events: Arc::new(Mutex::new(events)),
2304            writes: Arc::new(Mutex::new(Vec::new())),
2305        }
2306    }
2307
2308    #[test]
2309    fn put_patch_and_null_delete_are_deterministic() {
2310        let mut state = serde_json::json!({"a": {"b": 1, "c": 2}});
2311        apply_event(
2312            "root",
2313            &mut state,
2314            Event::Patch {
2315                path: "".into(),
2316                data: serde_json::json!({"a/b": 3, "a/c": null, "new/x": true})
2317                    .as_object()
2318                    .unwrap()
2319                    .clone(),
2320            },
2321        )
2322        .unwrap();
2323        assert_eq!(
2324            state,
2325            serde_json::json!({"a": {"b": 3}, "new": {"x": true}})
2326        );
2327        apply_event(
2328            "root",
2329            &mut state,
2330            Event::Put {
2331                path: "a".into(),
2332                data: Value::Null,
2333            },
2334        )
2335        .unwrap();
2336        assert_eq!(state, serde_json::json!({"new": {"x": true}}));
2337    }
2338
2339    #[tokio::test]
2340    async fn hydrates_notifies_and_stops() {
2341        let backend = mock(
2342            serde_json::json!({"count": 1}),
2343            vec![Event::Patch {
2344                path: "".into(),
2345                data: serde_json::json!({"count": 2}).as_object().unwrap().clone(),
2346            }],
2347        );
2348        let handle = start(
2349            Arc::new(backend),
2350            "items",
2351            Config {
2352                retry: RetryPolicy::Never,
2353                ..Config::default()
2354            },
2355        );
2356        let mut snapshots = handle.subscribe();
2357        tokio::time::timeout(Duration::from_secs(1), async {
2358            while snapshots.borrow().value != serde_json::json!({"count": 2}) {
2359                snapshots.changed().await.unwrap();
2360            }
2361        })
2362        .await
2363        .unwrap();
2364        assert_eq!(snapshots.borrow().value, serde_json::json!({"count": 2}));
2365        handle.shutdown().await;
2366    }
2367
2368    #[tokio::test]
2369    async fn typed_snapshot_and_optimistic_rollback_contract() {
2370        #[derive(Debug, Deserialize, PartialEq)]
2371        struct Model {
2372            count: u32,
2373        }
2374        let backend = mock(serde_json::json!({"count": 4}), vec![]);
2375        let handle = start(
2376            Arc::new(backend),
2377            "items",
2378            Config {
2379                retry: RetryPolicy::Never,
2380                write_policy: WritePolicy::Optimistic,
2381                ..Config::default()
2382            },
2383        );
2384        let mut status = handle.subscribe_status();
2385        tokio::time::timeout(Duration::from_secs(1), async {
2386            while *status.borrow() != SyncStatus::Connected {
2387                status.changed().await.unwrap();
2388            }
2389        })
2390        .await
2391        .unwrap();
2392        assert_eq!(
2393            handle.snapshot().decode::<Model>().unwrap(),
2394            Model { count: 4 }
2395        );
2396        handle.shutdown().await;
2397    }
2398
2399    #[tokio::test]
2400    async fn typed_handle_surfaces_conversion_errors_after_partial_updates() {
2401        #[derive(Debug, Clone, Deserialize, PartialEq)]
2402        struct Model {
2403            count: u32,
2404        }
2405        let backend = mock(
2406            serde_json::json!({"count": 1}),
2407            vec![
2408                Event::Patch {
2409                    path: "".into(),
2410                    data: serde_json::json!({"count": 2}).as_object().unwrap().clone(),
2411                },
2412                Event::Put {
2413                    path: "".into(),
2414                    data: serde_json::json!({"count": "not-a-number"}),
2415                },
2416            ],
2417        );
2418        let handle = start_typed::<_, Model>(
2419            Arc::new(backend),
2420            "typed",
2421            Config {
2422                retry: RetryPolicy::Never,
2423                ..Config::default()
2424            },
2425        );
2426        let mut snapshots = handle.subscribe();
2427        tokio::time::timeout(Duration::from_secs(1), async {
2428            while handle.status() != SyncStatus::Connected {
2429                snapshots.changed().await.unwrap();
2430            }
2431        })
2432        .await
2433        .unwrap();
2434        tokio::time::sleep(Duration::from_millis(10)).await;
2435        assert!(matches!(
2436            &*snapshots.borrow(),
2437            Err(SyncError::Conversion(_))
2438        ));
2439        handle.shutdown().await;
2440    }
2441
2442    #[tokio::test]
2443    async fn firebase_emulator_crud_and_sse_when_configured() {
2444        let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
2445            return;
2446        };
2447        let root = test_root("crud");
2448        let backend = RtdbBackend::new(format!("http://{host}"), "")
2449            .with_namespace("demo-rtdb-sync-default-rtdb");
2450        backend
2451            .put(&root, serde_json::json!({"generation": 1}))
2452            .await
2453            .unwrap();
2454        assert_eq!(backend.get(&root).await.unwrap()["generation"], 1);
2455        let mut events = backend.subscribe(&root).await.unwrap();
2456        backend
2457            .patch(
2458                &root,
2459                serde_json::json!({"generation": 2})
2460                    .as_object()
2461                    .unwrap()
2462                    .clone(),
2463            )
2464            .await
2465            .unwrap();
2466        let event = tokio::time::timeout(Duration::from_secs(2), async {
2467            loop {
2468                if let Some(Ok(event)) = events.next().await {
2469                    let is_target = match &event {
2470                        Event::Put { data, .. } => data["generation"] == 2,
2471                        Event::Patch { data, .. } => {
2472                            data.get("generation") == Some(&serde_json::json!(2))
2473                        }
2474                    };
2475                    if is_target {
2476                        break event;
2477                    }
2478                }
2479            }
2480        })
2481        .await
2482        .unwrap();
2483        assert!(
2484            matches!(event, Event::Patch { ref data, .. } if data.get("generation") == Some(&serde_json::json!(2)))
2485                || matches!(event, Event::Put { ref data, .. } if data["generation"] == 2)
2486        );
2487        backend.put(&root, Value::Null).await.unwrap();
2488    }
2489
2490    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2491    async fn standard_32_path_profile_converges() {
2492        let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
2493            return;
2494        };
2495        let backend = Arc::new(
2496            RtdbBackend::new(format!("http://{host}"), "")
2497                .with_namespace("demo-rtdb-sync-default-rtdb"),
2498        );
2499        let mut handles = Vec::new();
2500        for path in 0..32 {
2501            let handle = start(
2502                backend.clone(),
2503                test_root(&format!("paths/{path}")),
2504                Config {
2505                    retry: RetryPolicy::Never,
2506                    ..Config::default()
2507                },
2508            );
2509            handles.push((path, handle));
2510        }
2511        for (path, handle) in &handles {
2512            wait_for_status(handle, SyncStatus::Connected).await;
2513            backend
2514                .put(
2515                    &test_root(&format!("paths/{path}")),
2516                    serde_json::json!({"path": path, "generation": 0}),
2517                )
2518                .await
2519                .unwrap();
2520        }
2521        for generation in 1..=50 {
2522            for path in 0..32 {
2523                let root = test_root(&format!("paths/{path}"));
2524                match generation % 3 {
2525                    0 => backend
2526                        .put(
2527                            &root,
2528                            serde_json::json!({"path": path, "generation": generation}),
2529                        )
2530                        .await
2531                        .unwrap(),
2532                    1 => backend
2533                        .patch(
2534                            &root,
2535                            serde_json::json!({"generation": generation})
2536                                .as_object()
2537                                .unwrap()
2538                                .clone(),
2539                        )
2540                        .await
2541                        .unwrap(),
2542                    _ => {
2543                        backend.put(&root, Value::Null).await.unwrap();
2544                        backend
2545                            .put(
2546                                &root,
2547                                serde_json::json!({"path": path, "generation": generation}),
2548                            )
2549                            .await
2550                            .unwrap();
2551                    }
2552                }
2553            }
2554        }
2555        for (path, handle) in &handles {
2556            let mut updates = handle.subscribe();
2557            tokio::time::timeout(Duration::from_secs(10), async {
2558                while updates.borrow().value["generation"] != 50 {
2559                    updates.changed().await.unwrap();
2560                }
2561            })
2562            .await
2563            .unwrap();
2564            assert_eq!(updates.borrow().value["path"], *path);
2565        }
2566        for (_, handle) in handles {
2567            handle.shutdown().await;
2568        }
2569    }
2570
2571    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
2572    async fn manual_heavy_64_path_profile_converges() {
2573        if std::env::var("RTDB_SYNC_HEAVY").ok().as_deref() != Some("1") {
2574            return;
2575        }
2576        let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
2577        let backend = Arc::new(
2578            RtdbBackend::new(format!("http://{host}"), "")
2579                .with_namespace("demo-rtdb-sync-default-rtdb"),
2580        );
2581        let mut handles = Vec::new();
2582        for path in 0..64 {
2583            let root = test_root(&format!("heavy/{path}"));
2584            backend
2585                .put(&root, serde_json::json!({"path": path, "generation": 0}))
2586                .await
2587                .unwrap();
2588            let stream_backend = Arc::new(
2589                RtdbBackend::new(format!("http://{host}"), "")
2590                    .with_namespace("demo-rtdb-sync-default-rtdb"),
2591            );
2592            let handle = start(
2593                stream_backend,
2594                root,
2595                Config {
2596                    retry: RetryPolicy::Never,
2597                    ..Config::default()
2598                },
2599            );
2600            handles.push((path, handle));
2601        }
2602        for (_path, handle) in &handles {
2603            wait_for_status(handle, SyncStatus::Connected).await;
2604        }
2605        for generation in 1..=100 {
2606            let writes = (0..64).map(|path| {
2607                let backend = backend.clone();
2608                async move {
2609                    let root = test_root(&format!("heavy/{path}"));
2610                    if generation % 3 == 1 {
2611                        backend
2612                            .patch(
2613                                &root,
2614                                serde_json::json!({"generation": generation})
2615                                    .as_object()
2616                                    .unwrap()
2617                                    .clone(),
2618                            )
2619                            .await
2620                    } else if generation % 3 == 2 {
2621                        backend.put(&root, Value::Null).await?;
2622                        backend
2623                            .put(
2624                                &root,
2625                                serde_json::json!({"path": path, "generation": generation}),
2626                            )
2627                            .await
2628                    } else {
2629                        backend
2630                            .put(
2631                                &root,
2632                                serde_json::json!({"path": path, "generation": generation}),
2633                            )
2634                            .await
2635                    }
2636                }
2637            });
2638            for result in futures_util::future::join_all(writes).await {
2639                result.unwrap();
2640            }
2641        }
2642        for (path, handle) in &handles {
2643            let mut updates = handle.subscribe();
2644            tokio::time::timeout(Duration::from_secs(30), async {
2645                while updates.borrow().value["generation"] != 100 {
2646                    updates.changed().await.unwrap();
2647                }
2648            })
2649            .await
2650            .unwrap();
2651            assert_eq!(updates.borrow().value["path"], *path);
2652        }
2653        for (_, handle) in handles {
2654            handle.shutdown().await;
2655        }
2656    }
2657
2658    #[tokio::test]
2659    async fn emulator_namespaces_are_isolated_when_configured() {
2660        let Ok(host) = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST") else {
2661            return;
2662        };
2663        let left =
2664            RtdbBackend::new(format!("http://{host}"), "").with_namespace("rtdb-sync-ns-left");
2665        let right =
2666            RtdbBackend::new(format!("http://{host}"), "").with_namespace("rtdb-sync-ns-right");
2667        left.put("isolation", serde_json::json!({"owner": "left"}))
2668            .await
2669            .unwrap();
2670        right
2671            .put("isolation", serde_json::json!({"owner": "right"}))
2672            .await
2673            .unwrap();
2674        assert_eq!(left.get("isolation").await.unwrap()["owner"], "left");
2675        assert_eq!(right.get("isolation").await.unwrap()["owner"], "right");
2676    }
2677
2678    #[tokio::test]
2679    #[ignore = "run through scripts/test-emulator-recovery.sh"]
2680    async fn emulator_restart_rehydrates_and_recovers() {
2681        let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
2682        let ready = std::env::var("RTDB_RECOVERY_READY").expect("ready marker");
2683        let restored = std::env::var("RTDB_RECOVERY_RESTORED").expect("restored marker");
2684        let backend = Arc::new(
2685            RtdbBackend::new(format!("http://{host}"), "")
2686                .with_namespace("demo-rtdb-sync-default-rtdb"),
2687        );
2688        let mut handles = Vec::new();
2689        let path_count = std::env::var("RTDB_RECOVERY_PATHS")
2690            .ok()
2691            .and_then(|value| value.parse::<usize>().ok())
2692            .filter(|value| *value > 0)
2693            .unwrap_or(32);
2694        for path in 0..path_count {
2695            let root = test_root(&format!("restart/{path}"));
2696            emulator_put_retry(
2697                &backend,
2698                &root,
2699                serde_json::json!({"path": path, "generation": 1}),
2700            )
2701            .await
2702            .unwrap();
2703            let stream_backend = Arc::new(
2704                RtdbBackend::new(format!("http://{host}"), "")
2705                    .with_namespace("demo-rtdb-sync-default-rtdb"),
2706            );
2707            handles.push((
2708                path,
2709                start(
2710                    stream_backend,
2711                    root,
2712                    Config {
2713                        retry: RetryPolicy::Exponential {
2714                            max_attempts: Some(20),
2715                            base: Duration::from_millis(100),
2716                            max: Duration::from_secs(1),
2717                        },
2718                        ..Config::default()
2719                    },
2720                ),
2721            ));
2722        }
2723        for (_, handle) in &handles {
2724            wait_for_status(handle, SyncStatus::Connected).await;
2725        }
2726        std::fs::write(&ready, "connected").unwrap();
2727        while !std::path::Path::new(&restored).exists() {
2728            tokio::time::sleep(Duration::from_millis(50)).await;
2729        }
2730        tokio::time::timeout(Duration::from_secs(30), async {
2731            loop {
2732                if handles
2733                    .iter()
2734                    .all(|(_, handle)| handle.metrics().reconnect_attempts >= 1)
2735                {
2736                    break;
2737                }
2738                tokio::time::sleep(Duration::from_millis(50)).await;
2739            }
2740        })
2741        .await
2742        .unwrap();
2743        for path in 0..path_count {
2744            emulator_put_retry(
2745                &backend,
2746                &test_root(&format!("restart/{path}")),
2747                serde_json::json!({"path": path, "generation": 2}),
2748            )
2749            .await
2750            .unwrap();
2751        }
2752        for (path, handle) in &handles {
2753            let mut updates = handle.subscribe();
2754            tokio::time::timeout(Duration::from_secs(30), async {
2755                while updates.borrow().value["generation"] != 2 {
2756                    updates.changed().await.unwrap();
2757                }
2758            })
2759            .await
2760            .unwrap();
2761            assert_eq!(updates.borrow().value["path"], *path);
2762            assert!(handle.metrics().reconnect_attempts >= 1);
2763        }
2764        for (_, handle) in handles {
2765            handle.shutdown().await;
2766        }
2767    }
2768
2769    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2770    async fn fanout_64_subscribers_converge() {
2771        let events = (1..=250)
2772            .map(|generation| Event::Put {
2773                path: "".into(),
2774                data: serde_json::json!({"generation": generation}),
2775            })
2776            .collect();
2777        let handle = start(
2778            Arc::new(mock(serde_json::json!({"generation": 0}), events)),
2779            "fanout",
2780            Config {
2781                retry: RetryPolicy::Never,
2782                ..Config::default()
2783            },
2784        );
2785        let mut subscribers = (0..64).map(|_| handle.subscribe()).collect::<Vec<_>>();
2786        for receiver in &mut subscribers {
2787            tokio::time::timeout(Duration::from_secs(2), async {
2788                while receiver.borrow().value["generation"] != 250 {
2789                    receiver.changed().await.unwrap();
2790                }
2791            })
2792            .await
2793            .unwrap();
2794            assert_eq!(receiver.borrow().value["generation"], 250);
2795        }
2796        handle.shutdown().await;
2797    }
2798
2799    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2800    async fn lifecycle_churn_stops_one_hundred_tasks() {
2801        for _ in 0..100 {
2802            let handle = start(
2803                Arc::new(mock(Value::Null, vec![])),
2804                "churn",
2805                Config {
2806                    retry: RetryPolicy::Never,
2807                    ..Config::default()
2808                },
2809            );
2810            handle.shutdown().await;
2811        }
2812    }
2813
2814    #[tokio::test]
2815    async fn reconnect_status_metrics_and_backend_hook_are_observable() {
2816        let backend = Arc::new(Flaky {
2817            subscriptions: AtomicU64::new(0),
2818            replacements: AtomicU64::new(0),
2819        });
2820        let handle = start(
2821            backend.clone(),
2822            "flaky",
2823            Config {
2824                retry: RetryPolicy::Exponential {
2825                    max_attempts: Some(3),
2826                    base: Duration::from_millis(1),
2827                    max: Duration::from_millis(5),
2828                },
2829                jitter_max: Duration::from_millis(1),
2830                ..Config::default()
2831            },
2832        );
2833        wait_for_status(&handle, SyncStatus::Connected).await;
2834        assert_eq!(backend.replacements.load(Ordering::Relaxed), 1);
2835        assert_eq!(handle.metrics().stream_failures, 1);
2836        assert_eq!(handle.metrics().reconnect_attempts, 1);
2837        handle.shutdown().await;
2838    }
2839
2840    #[tokio::test]
2841    async fn twenty_five_reconnect_cycles_preserve_state_and_subscribers() {
2842        let backend = Arc::new(Cycling {
2843            subscriptions: AtomicU64::new(0),
2844        });
2845        let handle = start(
2846            backend,
2847            "cycles",
2848            Config {
2849                retry: RetryPolicy::Exponential {
2850                    max_attempts: Some(30),
2851                    base: Duration::from_millis(1),
2852                    max: Duration::from_millis(2),
2853                },
2854                ..Config::default()
2855            },
2856        );
2857        wait_for_status(&handle, SyncStatus::Connected).await;
2858        assert_eq!(handle.metrics().stream_failures, 25);
2859        assert_eq!(handle.metrics().reconnect_attempts, 25);
2860        handle.shutdown().await;
2861    }
2862
2863    #[tokio::test]
2864    async fn shutdown_cancels_hydration_backoff_promptly() {
2865        let handle = start(
2866            Arc::new(Failing),
2867            "backoff",
2868            Config {
2869                retry: RetryPolicy::Exponential {
2870                    max_attempts: None,
2871                    base: Duration::from_secs(30),
2872                    max: Duration::from_secs(30),
2873                },
2874                ..Config::default()
2875            },
2876        );
2877        tokio::time::timeout(Duration::from_millis(100), handle.shutdown())
2878            .await
2879            .unwrap();
2880    }
2881
2882    #[tokio::test]
2883    async fn optimistic_write_rolls_back_on_backend_failure() {
2884        let handle = start(
2885            Arc::new(Failing),
2886            "failing",
2887            Config {
2888                write_policy: WritePolicy::Optimistic,
2889                retry: RetryPolicy::Never,
2890                ..Config::default()
2891            },
2892        );
2893        wait_for_status(&handle, SyncStatus::Connected).await;
2894        assert_eq!(
2895            handle.put("", serde_json::json!({"count": 2})).await,
2896            Err(SyncError::Backend("injected write failure".into()))
2897        );
2898        assert_eq!(handle.snapshot().value, serde_json::json!({"count": 1}));
2899        assert_eq!(handle.metrics().failed_writes, 1);
2900        handle.shutdown().await;
2901    }
2902
2903    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2904    async fn concurrent_local_and_remote_writers_converge_without_echo_loop() {
2905        let (events, _) = tokio::sync::broadcast::channel(256);
2906        let backend = Arc::new(Live {
2907            value: Arc::new(Mutex::new(serde_json::json!({"count": 0}))),
2908            events,
2909        });
2910        let handle = start(
2911            backend.clone(),
2912            "writers",
2913            Config {
2914                write_policy: WritePolicy::Optimistic,
2915                conflict_policy: ConflictPolicy::RemoteWins,
2916                retry: RetryPolicy::Never,
2917                ..Config::default()
2918            },
2919        );
2920        wait_for_status(&handle, SyncStatus::Connected).await;
2921        for count in 1..=100u64 {
2922            let local = handle.put("", serde_json::json!({"count": count}));
2923            let remote = backend.put("", serde_json::json!({"count": count + 10_000}));
2924            let (local_result, remote_result) = tokio::join!(local, remote);
2925            local_result.unwrap();
2926            remote_result.unwrap();
2927        }
2928        tokio::time::sleep(Duration::from_millis(25)).await;
2929        assert_eq!(
2930            handle.snapshot().value,
2931            backend.get("writers").await.unwrap()
2932        );
2933        assert_eq!(handle.metrics().successful_writes, 100);
2934        handle.shutdown().await;
2935    }
2936
2937    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
2938    async fn bidirectional_32_path_conflict_profile_converges() {
2939        let mut pairs = Vec::new();
2940        for path in 0..32u64 {
2941            let (events, _) = tokio::sync::broadcast::channel(512);
2942            let backend = Arc::new(Live {
2943                value: Arc::new(Mutex::new(
2944                    serde_json::json!({"path": path, "local": 0, "remote": 0}),
2945                )),
2946                events,
2947            });
2948            let handle = start(
2949                backend.clone(),
2950                format!("bidirectional/{path}"),
2951                Config {
2952                    write_policy: WritePolicy::Optimistic,
2953                    conflict_policy: ConflictPolicy::RemoteWins,
2954                    retry: RetryPolicy::Never,
2955                    ..Config::default()
2956                },
2957            );
2958            pairs.push((path, backend, handle));
2959        }
2960        for (_, _, handle) in &pairs {
2961            wait_for_status(handle, SyncStatus::Connected).await;
2962        }
2963        for generation in 1..=100u64 {
2964            let writes = pairs.iter().map(|(_, backend, handle)| async move {
2965                let local = handle.put("", serde_json::json!({"local": generation}));
2966                let remote = backend.patch(
2967                    "",
2968                    serde_json::json!({"remote": generation})
2969                        .as_object()
2970                        .unwrap()
2971                        .clone(),
2972                );
2973                let (local_result, remote_result) = tokio::join!(local, remote);
2974                local_result?;
2975                remote_result
2976            });
2977            for result in futures_util::future::join_all(writes).await {
2978                result.unwrap();
2979            }
2980        }
2981        tokio::time::sleep(Duration::from_millis(50)).await;
2982        for (_, backend, handle) in &pairs {
2983            assert_eq!(handle.snapshot().value, backend.get("").await.unwrap());
2984            assert_eq!(handle.metrics().successful_writes, 100);
2985        }
2986        for (_, _, handle) in pairs {
2987            handle.shutdown().await;
2988        }
2989    }
2990
2991    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
2992    async fn fan_in_eight_local_producers_preserves_acknowledgements() {
2993        let (events, _) = tokio::sync::broadcast::channel(512);
2994        let backend = Arc::new(Live {
2995            value: Arc::new(Mutex::new(serde_json::json!({"remote": 0}))),
2996            events,
2997        });
2998        let handle = start(
2999            backend.clone(),
3000            "fan-in",
3001            Config {
3002                write_policy: WritePolicy::Optimistic,
3003                retry: RetryPolicy::Never,
3004                ..Config::default()
3005            },
3006        );
3007        wait_for_status(&handle, SyncStatus::Connected).await;
3008        let producers = (0..8).map(|producer| {
3009            let handle = &handle;
3010            async move {
3011                for generation in 0..25u64 {
3012                    handle
3013                        .patch(
3014                            "",
3015                            serde_json::json!({ (format!("local-{producer}")): generation }),
3016                        )
3017                        .await
3018                        .unwrap();
3019                }
3020            }
3021        });
3022        let _ = futures_util::future::join_all(producers).await;
3023        backend
3024            .patch(
3025                "",
3026                serde_json::json!({"remote": 1})
3027                    .as_object()
3028                    .unwrap()
3029                    .clone(),
3030            )
3031            .await
3032            .unwrap();
3033        tokio::time::sleep(Duration::from_millis(25)).await;
3034        assert_eq!(handle.metrics().successful_writes, 200);
3035        assert_eq!(handle.snapshot().value, backend.get("").await.unwrap());
3036        handle.shutdown().await;
3037    }
3038
3039    async fn wait_for_status(handle: &SyncHandle, expected: SyncStatus) {
3040        let mut status = handle.subscribe_status();
3041        tokio::time::timeout(Duration::from_secs(5), async {
3042            while status.borrow().clone() != expected {
3043                status.changed().await.unwrap();
3044            }
3045        })
3046        .await
3047        .unwrap();
3048    }
3049
3050    fn test_root(label: &str) -> String {
3051        format!("rtdb-sync-tests/{}/{label}", std::process::id())
3052    }
3053
3054    async fn emulator_put_retry(
3055        backend: &RtdbBackend,
3056        path: &str,
3057        value: Value,
3058    ) -> Result<(), SyncError> {
3059        let mut last = None;
3060        for _ in 0..300 {
3061            match backend.put(path, value.clone()).await {
3062                Ok(()) => return Ok(()),
3063                Err(error) => {
3064                    last = Some(error);
3065                    tokio::time::sleep(Duration::from_millis(100)).await;
3066                }
3067            }
3068        }
3069        Err(last.unwrap_or_else(|| SyncError::Backend("emulator did not accept write".into())))
3070    }
3071
3072    #[test]
3073    fn file_persistence_is_versioned_atomic_and_namespace_isolated() {
3074        let root =
3075            std::env::temp_dir().join(format!("rtdb-sync-persistence-{}", std::process::id()));
3076        let _ = fs::remove_dir_all(&root);
3077        let store = FilePersistence::new(&root).unwrap();
3078        let key = "user-a/path";
3079        store
3080            .store_snapshot(&DurableSnapshot {
3081                format_version: PERSISTENCE_FORMAT_VERSION,
3082                sync_key: key.into(),
3083                generation: 7,
3084                value: serde_json::json!({"ok": true}),
3085                saved_at_ms: now_ms(),
3086            })
3087            .unwrap();
3088        store
3089            .append_mutation(&DurableMutation {
3090                format_version: PERSISTENCE_FORMAT_VERSION,
3091                sync_key: key.into(),
3092                id: 11,
3093                path: "child".into(),
3094                kind: DurableMutationKind::Patch,
3095                payload: serde_json::json!({"count": 1}),
3096                generation: 8,
3097                created_at_ms: now_ms(),
3098            })
3099            .unwrap();
3100        assert_eq!(store.load(key).unwrap().pending.len(), 1);
3101        assert!(store.load("other/path").unwrap().pending.is_empty());
3102        store.acknowledge_mutation(key, 11).unwrap();
3103        store.compact(key).unwrap();
3104        assert!(store.load(key).unwrap().pending.is_empty());
3105        let snapshot_path = store.path(key, "snapshot.json").unwrap();
3106        fs::write(snapshot_path, b"not-json").unwrap();
3107        assert!(matches!(store.load(key), Err(SyncError::Persistence(_))));
3108        store.clear(key).unwrap();
3109        let _ = fs::remove_dir_all(&root);
3110    }
3111
3112    #[tokio::test]
3113    async fn durable_queue_restores_offline_state_and_replays_after_restart() {
3114        let store = Arc::new(MemoryPersistence::default());
3115        let online = Arc::new(std::sync::atomic::AtomicBool::new(false));
3116        let (events, _) = tokio::sync::broadcast::channel(16);
3117        let backend = Arc::new(Recoverable {
3118            online: online.clone(),
3119            value: Arc::new(Mutex::new(serde_json::json!({"count": 0}))),
3120            events,
3121        });
3122        let config = Config {
3123            persistence: Some(store.clone()),
3124            persistence_key: Some("durable-test".into()),
3125            offline_policy: OfflinePolicy::QueueWithLimit { max_pending: 2 },
3126            write_policy: WritePolicy::Optimistic,
3127            retry: RetryPolicy::Exponential {
3128                max_attempts: None,
3129                base: Duration::from_millis(5),
3130                max: Duration::from_millis(20),
3131            },
3132            ..Config::default()
3133        };
3134        let first = start(backend.clone(), "root", config.clone());
3135        tokio::time::timeout(Duration::from_secs(1), async {
3136            loop {
3137                if first.status() == SyncStatus::Offline {
3138                    break;
3139                }
3140                tokio::time::sleep(Duration::from_millis(5)).await;
3141            }
3142        })
3143        .await
3144        .unwrap();
3145        first
3146            .put("", serde_json::json!({"count": 1}))
3147            .await
3148            .unwrap();
3149        assert_eq!(first.metrics().pending_mutations, 1);
3150        first.shutdown().await;
3151        assert_eq!(store.load("durable-test").unwrap().pending.len(), 1);
3152
3153        online.store(true, Ordering::Relaxed);
3154        let second = start(backend.clone(), "root", config);
3155        tokio::time::timeout(Duration::from_secs(2), async {
3156            loop {
3157                if backend.get("root").await.unwrap()["count"] == 1
3158                    && second.metrics().pending_mutations == 0
3159                    && second.status() == SyncStatus::Connected
3160                {
3161                    break;
3162                }
3163                tokio::time::sleep(Duration::from_millis(10)).await;
3164            }
3165        })
3166        .await
3167        .unwrap();
3168        assert_eq!(store.load("durable-test").unwrap().pending.len(), 0);
3169        second.shutdown().await;
3170    }
3171
3172    #[tokio::test]
3173    async fn offline_queue_limit_is_visible_and_missing_store_is_rejected() {
3174        let backend = Arc::new(Failing);
3175        let no_store = start(
3176            backend.clone(),
3177            "root",
3178            Config {
3179                offline_policy: OfflinePolicy::QueueWhileOffline,
3180                ..Config::default()
3181            },
3182        );
3183        tokio::time::timeout(Duration::from_secs(1), async {
3184            loop {
3185                if matches!(
3186                    no_store.status(),
3187                    SyncStatus::Failed(SyncError::Persistence(_))
3188                ) {
3189                    break;
3190                }
3191                tokio::time::sleep(Duration::from_millis(5)).await;
3192            }
3193        })
3194        .await
3195        .unwrap();
3196        no_store.shutdown().await;
3197
3198        let store = Arc::new(MemoryPersistence::default());
3199        let online = Arc::new(std::sync::atomic::AtomicBool::new(false));
3200        let (events, _) = tokio::sync::broadcast::channel(4);
3201        let backend = Arc::new(Recoverable {
3202            online,
3203            value: Arc::new(Mutex::new(Value::Null)),
3204            events,
3205        });
3206        let handle = start(
3207            backend,
3208            "root",
3209            Config {
3210                persistence: Some(store),
3211                persistence_key: Some("limit".into()),
3212                offline_policy: OfflinePolicy::QueueWithLimit { max_pending: 1 },
3213                write_policy: WritePolicy::Optimistic,
3214                retry: RetryPolicy::Exponential {
3215                    max_attempts: None,
3216                    base: Duration::from_millis(5),
3217                    max: Duration::from_millis(20),
3218                },
3219                ..Config::default()
3220            },
3221        );
3222        tokio::time::timeout(Duration::from_secs(1), async {
3223            while handle.status() != SyncStatus::Offline {
3224                tokio::time::sleep(Duration::from_millis(5)).await;
3225            }
3226        })
3227        .await
3228        .unwrap();
3229        handle.put("a", 1).await.unwrap();
3230        assert!(matches!(
3231            handle.put("b", 2).await,
3232            Err(SyncError::Persistence(_))
3233        ));
3234        handle.shutdown().await;
3235    }
3236
3237    #[tokio::test]
3238    async fn restored_pending_mutation_uses_explicit_remote_wins_policy() {
3239        let store = Arc::new(MemoryPersistence::default());
3240        store
3241            .store_snapshot(&DurableSnapshot {
3242                format_version: PERSISTENCE_FORMAT_VERSION,
3243                sync_key: "conflict".into(),
3244                generation: 1,
3245                value: serde_json::json!({"count": 1}),
3246                saved_at_ms: now_ms(),
3247            })
3248            .unwrap();
3249        store
3250            .append_mutation(&DurableMutation {
3251                format_version: PERSISTENCE_FORMAT_VERSION,
3252                sync_key: "conflict".into(),
3253                id: 1,
3254                path: "".into(),
3255                kind: DurableMutationKind::Put,
3256                payload: serde_json::json!({"count": 2}),
3257                generation: 2,
3258                created_at_ms: now_ms(),
3259            })
3260            .unwrap();
3261        let (events, _) = tokio::sync::broadcast::channel(4);
3262        let backend = Arc::new(Recoverable {
3263            online: Arc::new(std::sync::atomic::AtomicBool::new(true)),
3264            value: Arc::new(Mutex::new(serde_json::json!({"count": 9}))),
3265            events,
3266        });
3267        let handle = start(
3268            backend.clone(),
3269            "root",
3270            Config {
3271                persistence: Some(store.clone()),
3272                persistence_key: Some("conflict".into()),
3273                conflict_policy: ConflictPolicy::RemoteWins,
3274                ..Config::default()
3275            },
3276        );
3277        wait_for_status(&handle, SyncStatus::Connected).await;
3278        assert_eq!(backend.get("root").await.unwrap()["count"], 9);
3279        assert_eq!(handle.metrics().conflict_count, 1);
3280        assert_eq!(handle.metrics().pending_mutations, 0);
3281        assert!(store.load("conflict").unwrap().pending.is_empty());
3282        handle.shutdown().await;
3283    }
3284
3285    fn durable_emulator_config(store: Arc<FilePersistence>) -> Config {
3286        Config {
3287            persistence: Some(store),
3288            persistence_key: Some("durable-emulator-acceptance".into()),
3289            offline_policy: OfflinePolicy::QueueWhileOffline,
3290            write_policy: WritePolicy::Optimistic,
3291            retry: RetryPolicy::Exponential {
3292                max_attempts: None,
3293                base: Duration::from_millis(10),
3294                max: Duration::from_millis(50),
3295            },
3296            ..Config::default()
3297        }
3298    }
3299
3300    fn durable_emulator_store() -> Arc<FilePersistence> {
3301        Arc::new(
3302            FilePersistence::new(std::env::var("RTDB_DURABLE_STORE").expect("RTDB_DURABLE_STORE"))
3303                .unwrap(),
3304        )
3305    }
3306
3307    #[tokio::test]
3308    #[ignore = "run through scripts/test-emulator-offline.sh"]
3309    async fn durable_emulator_seed_persists_snapshot() {
3310        let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
3311        let store = durable_emulator_store();
3312        let backend = Arc::new(
3313            RtdbBackend::new(format!("http://{host}"), "")
3314                .with_namespace("demo-rtdb-sync-default-rtdb"),
3315        );
3316        emulator_put_retry(
3317            &backend,
3318            "durable-acceptance",
3319            serde_json::json!({"count": 1}),
3320        )
3321        .await
3322        .unwrap();
3323        let handle = start(
3324            backend,
3325            "durable-acceptance",
3326            durable_emulator_config(store.clone()),
3327        );
3328        wait_for_status(&handle, SyncStatus::Connected).await;
3329        handle.shutdown().await;
3330        assert!(store
3331            .load("durable-emulator-acceptance")
3332            .unwrap()
3333            .snapshot
3334            .is_some());
3335    }
3336
3337    #[tokio::test]
3338    #[ignore = "run through scripts/test-emulator-offline.sh"]
3339    async fn durable_emulator_queues_while_database_is_down() {
3340        let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
3341        let store = durable_emulator_store();
3342        let backend = Arc::new(
3343            RtdbBackend::new(format!("http://{host}"), "")
3344                .with_namespace("demo-rtdb-sync-default-rtdb"),
3345        );
3346        let handle = start(
3347            backend,
3348            "durable-acceptance",
3349            durable_emulator_config(store.clone()),
3350        );
3351        tokio::time::timeout(Duration::from_secs(5), async {
3352            loop {
3353                if handle.status() == SyncStatus::Offline {
3354                    break;
3355                }
3356                tokio::time::sleep(Duration::from_millis(10)).await;
3357            }
3358        })
3359        .await
3360        .unwrap();
3361        handle
3362            .put("", serde_json::json!({"count": 2}))
3363            .await
3364            .unwrap();
3365        if std::env::var_os("RTDB_DURABLE_CRASH_AFTER_QUEUE").is_some() {
3366            std::process::exit(0);
3367        }
3368        handle.shutdown().await;
3369        assert_eq!(
3370            store
3371                .load("durable-emulator-acceptance")
3372                .unwrap()
3373                .pending
3374                .len(),
3375            1
3376        );
3377    }
3378
3379    #[tokio::test]
3380    #[ignore = "run through scripts/test-emulator-offline.sh"]
3381    async fn durable_emulator_replays_after_database_returns() {
3382        let host = std::env::var("FIREBASE_DATABASE_EMULATOR_HOST").expect("emulator host");
3383        let store = durable_emulator_store();
3384        let backend = Arc::new(
3385            RtdbBackend::new(format!("http://{host}"), "")
3386                .with_namespace("demo-rtdb-sync-default-rtdb"),
3387        );
3388        let handle = start(
3389            backend.clone(),
3390            "durable-acceptance",
3391            durable_emulator_config(store.clone()),
3392        );
3393        tokio::time::timeout(Duration::from_secs(10), async {
3394            loop {
3395                if backend.get("durable-acceptance").await.unwrap()["count"] == 2
3396                    && handle.metrics().pending_mutations == 0
3397                    && handle.metrics().replay_successes == 1
3398                {
3399                    break;
3400                }
3401                tokio::time::sleep(Duration::from_millis(20)).await;
3402            }
3403        })
3404        .await
3405        .unwrap();
3406        wait_for_status(&handle, SyncStatus::Connected).await;
3407        assert!(store
3408            .load("durable-emulator-acceptance")
3409            .unwrap()
3410            .pending
3411            .is_empty());
3412        handle.shutdown().await;
3413    }
3414
3415    #[test]
3416    fn retry_policy_is_bounded_and_paths_are_safe() {
3417        assert!(segments("a/../b").is_err());
3418        assert!(matches!(RetryPolicy::Never, RetryPolicy::Never));
3419        let policy = RetryPolicy::Exponential {
3420            max_attempts: Some(3),
3421            base: Duration::from_millis(10),
3422            max: Duration::from_millis(40),
3423        };
3424        assert_eq!(
3425            policy.delay(1, Duration::ZERO),
3426            Some(Duration::from_millis(10))
3427        );
3428        assert!(policy.delay(3, Duration::from_millis(5)).unwrap() <= Duration::from_millis(45));
3429        assert_eq!(policy.delay(4, Duration::ZERO), None);
3430        let mut value = Value::Null;
3431        assert!(apply_event(
3432            "root",
3433            &mut value,
3434            Event::Put {
3435                path: "a/../b".into(),
3436                data: Value::Null
3437            }
3438        )
3439        .is_err());
3440    }
3441
3442    #[test]
3443    fn conflict_policy_is_explicit_and_echoes_are_suppressed() {
3444        let (tx, rx) = watch::channel(Snapshot {
3445            generation: 0,
3446            value: serde_json::json!({"count": 2}),
3447        });
3448        let mut state = serde_json::json!({"count": 2});
3449        let mut pending = vec![PendingMutation {
3450            path: "".into(),
3451            event: Event::Put {
3452                path: "".into(),
3453                data: serde_json::json!({"count": 2}),
3454            },
3455        }];
3456        let mut generation = 0;
3457        reconcile_event(
3458            "root",
3459            &mut state,
3460            &mut generation,
3461            &tx,
3462            &mut pending,
3463            Event::Put {
3464                path: "".into(),
3465                data: serde_json::json!({"count": 2}),
3466            },
3467            ConflictPolicy::Reject,
3468        )
3469        .unwrap();
3470        assert_eq!(generation, 0);
3471        pending.push(PendingMutation {
3472            path: "".into(),
3473            event: Event::Put {
3474                path: "".into(),
3475                data: serde_json::json!({"count": 2}),
3476            },
3477        });
3478        reconcile_event(
3479            "root",
3480            &mut state,
3481            &mut generation,
3482            &tx,
3483            &mut pending,
3484            Event::Put {
3485                path: "".into(),
3486                data: serde_json::json!({"count": 3}),
3487            },
3488            ConflictPolicy::LocalWins,
3489        )
3490        .unwrap();
3491        assert_eq!(state["count"], 2);
3492        assert!(reconcile_event(
3493            "root",
3494            &mut state,
3495            &mut generation,
3496            &tx,
3497            &mut pending,
3498            Event::Put {
3499                path: "".into(),
3500                data: serde_json::json!({"count": 4})
3501            },
3502            ConflictPolicy::Reject
3503        )
3504        .is_err());
3505        drop(rx);
3506    }
3507}