Skip to main content

zero_client/client/
engine.rs

1//! The client engine (`Zero` in `packages/zero-client/src/client/zero.ts`,
2//! with the parts of Replicache it uses): the run loop that keeps a
3//! connection, the replica and its pending mutations, and the query and
4//! mutation bookkeeping. Everything runs synchronously on one thread; see
5//! `event_loop`.
6
7use super::connection_manager::{ConnectionManager, State};
8use super::database::{AggregateWrites, ReplicaDatabase, StorageOp, StoredMutation, base36};
9use super::event_loop::{EventLoop, LoopScheduler};
10use super::ivm_branch::IvmBranch;
11use super::log::Log;
12use super::mutation_tracker::MutationTracker;
13use super::poke_handler::{MergedPoke, PokeHandler};
14use super::query_context::QueryContext;
15use super::query_manager::QueryManager;
16use super::replica_store::{
17    KvDiff, KvMap, KvPatchOp, KvWrite, PendingMutation, ReplicaStore, Snapshot, apply_patch,
18    compare_cookies, diff_all, diff_keys, with_prefix,
19};
20use super::socket::{SocketConnection, SocketEvents, SocketFactory, SocketMessage};
21use super::storage::{LoadError, StorageFailure, ZeroStorage};
22use super::transaction::{Reason, Transaction};
23use super::{
24    ApplicationError, ClientErrorKind, ConnectionError, ConnectionStatus, ConnectionTransition,
25    Deferred, Error, LogLevel, LogSink, Scheduler, StderrLogSink, Timer,
26};
27use crate::decode::{DecodeError, JsonCodable};
28use crate::hash::xxhash32;
29use crate::ivm::{Row, capture};
30use crate::json::{self, Object, Value, number};
31use crate::keys::{self, AGGREGATES_PREFIX, GOT_QUERIES_PREFIX, MUTATIONS_PREFIX};
32use crate::map::OrderedMap;
33use crate::protocol::{
34    ChangeDesiredQueriesBody, ConnectedBody, ConnectionUrl, DeleteClientsBody, Downstream,
35    ErrorBody, ErrorKind, InitConnectionBody, Mutation, PushBody, PushResponseBody,
36    UpQueriesPatchOp, UpdateAuthBody, Upstream, poke_chunk, sec_protocols,
37};
38use crate::schema::Schema;
39use crate::strings;
40use std::cell::{Cell, RefCell};
41use std::collections::{HashMap, HashSet, VecDeque};
42use std::hash::{BuildHasher, Hasher};
43use std::rc::{Rc, Weak};
44use std::sync::Arc;
45
46/// `RUN_LOOP_INTERVAL_MS`: the backoff before retrying a failed connection.
47pub(crate) const RUN_LOOP_INTERVAL_MS: f64 = 5_000.0;
48/// `FAST_RECONNECT_INITIAL_MS`: the longest wait before the first retry
49/// after a transport failure. Each further consecutive failure doubles it,
50/// up to `RUN_LOOP_INTERVAL_MS`.
51const FAST_RECONNECT_INITIAL_MS: f64 = 500.0;
52/// `STABLE_CONNECTION_MS`: how long a connection must stay up before a
53/// later failure starts the fast schedule from the beginning again.
54const STABLE_CONNECTION_MS: f64 = 5_000.0;
55/// `CONNECT_TIMEOUT_MS`: each phase of a connection attempt gets this long.
56const CONNECT_TIMEOUT_MS: f64 = 10_000.0;
57/// `HEARTBEAT_INTERVAL`.
58const HEARTBEAT_INTERVAL_MS: f64 = 60_000.0;
59/// `GC_INTERVAL`, of clients and of client groups.
60const GC_INTERVAL_MS: f64 = 5.0 * 60_000.0;
61/// `CLIENT_MAX_INACTIVE_TIME`.
62const CLIENT_MAX_INACTIVE_MS: f64 = 24.0 * 60.0 * 60_000.0;
63/// `INITIAL_COLLECT_IDB_DELAY`.
64const INITIAL_COLLECT_DATABASES_DELAY_MS: f64 = 5.0 * 60_000.0;
65/// `COLLECT_IDB_INTERVAL`.
66const COLLECT_DATABASES_INTERVAL_MS: f64 = 12.0 * 60.0 * 60_000.0;
67/// `PERSIST_IDLE_TIMEOUT_MS` and `PERSIST_THROTTLE_MS`.
68const PERSIST_IDLE_TIMEOUT_MS: f64 = 1000.0;
69const PERSIST_THROTTLE_MS: f64 = 500.0;
70
71/// `reconnectBackoffMs`: the wait before retrying after `failed_attempts`
72/// consecutive transport failures (1 for the first). The ceiling starts at
73/// `FAST_RECONNECT_INITIAL_MS` and doubles with each failure, and the wait
74/// is between half the ceiling and the ceiling, so that clients that lost
75/// the same server don't all retry at once. Once the ceiling reaches
76/// `max_ms`, the wait is `max_ms`.
77pub(crate) fn reconnect_backoff_ms(failed_attempts: u32, max_ms: f64, random: f64) -> f64 {
78    let ceiling = FAST_RECONNECT_INITIAL_MS * 2f64.powi(failed_attempts as i32 - 1);
79    if ceiling >= max_ms {
80        return max_ms;
81    }
82    (ceiling / 2.0 + random * ceiling / 2.0).round()
83}
84
85thread_local! {
86    static RNG: Cell<u64> = Cell::new({
87        let mut h = std::collections::hash_map::RandomState::new().build_hasher();
88        h.write_u128(
89            std::time::SystemTime::now()
90                .duration_since(std::time::UNIX_EPOCH)
91                .map(|d| d.as_nanos())
92                .unwrap_or(0),
93        );
94        h.finish() | 1
95    });
96}
97
98/// A random 64-bit number (xorshift64*, seeded per thread from the OS).
99fn random_u64() -> u64 {
100    RNG.with(|r| {
101        let mut x = r.get();
102        x ^= x >> 12;
103        x ^= x << 25;
104        x ^= x >> 27;
105        r.set(x);
106        x.wrapping_mul(0x2545_F491_4F6C_DD1D)
107    })
108}
109
110/// `Math.random()`.
111fn random_f64() -> f64 {
112    (random_u64() >> 11) as f64 / (1u64 << 53) as f64
113}
114
115/// A random ID like the JS client's `nanoid`.
116pub(crate) fn random_id() -> String {
117    const ALPHABET: &[u8; 64] = b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-";
118    (0..21)
119        .map(|_| ALPHABET[(random_u64() % 64) as usize] as char)
120        .collect()
121}
122
123/// `Date.prototype.toISOString` of a time in milliseconds since 1970.
124pub(crate) fn iso_date_string(ms: f64) -> String {
125    let t = ms.floor() as i64;
126    let mut days = t.div_euclid(86_400_000);
127    let ms_of_day = t.rem_euclid(86_400_000);
128    // Howard Hinnant's civil_from_days.
129    days += 719_468;
130    let era = days.div_euclid(146_097);
131    let doe = days - era * 146_097;
132    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
133    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
134    let mp = (5 * doy + 2) / 153;
135    let d = doy - (153 * mp + 2) / 5 + 1;
136    let m = if mp < 10 { mp + 3 } else { mp - 9 };
137    let y = yoe + era * 400 + i64::from(m <= 2);
138    format!(
139        "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}.{:03}Z",
140        ms_of_day / 3_600_000,
141        ms_of_day / 60_000 % 60,
142        ms_of_day / 1000 % 60,
143        ms_of_day % 1000
144    )
145}
146
147/// The message valita gives for a message that doesn't parse (`getMessage`
148/// in `packages/shared/src/valita.ts`), for the cases the client can tell
149/// apart.
150fn valita_message(e: &DecodeError, text: &str) -> String {
151    if e.message.starts_with("Unknown message type") || e.message == "Expected [tag, body] tuple" {
152        return format!("Invalid union value: {text}");
153    }
154    e.valita_message()
155}
156
157/// The options of a Zero client (`ZeroOptions` in
158/// `packages/zero-client/src/client/options.ts`).
159#[derive(Clone)]
160pub struct ZeroOptions {
161    /// The zero-cache URL, `http` or `https`. Without one the client runs
162    /// locally and never connects.
163    pub cache_url: Option<String>,
164    /// The user the client syncs for. `None` for a logged-out client.
165    pub user_id: Option<String>,
166    /// The auth token zero-cache and the API server see.
167    pub auth: Option<String>,
168    /// Distinguishes local stores of the same user and URLs.
169    pub storage_key: Option<String>,
170    /// How long the client waits for a pong before it reconnects.
171    pub ping_timeout_ms: f64,
172    /// How long the client tries to reconnect before the state is
173    /// `Disconnected`.
174    pub disconnect_timeout_ms: f64,
175    /// How long query changes are batched before they are sent.
176    pub query_change_throttle_ms: f64,
177    /// The longest `Sec-WebSocket-Protocol` header the client sends.
178    /// `initConnection` is sent as a message instead when it doesn't fit.
179    pub max_header_length: usize,
180    /// The API server's push URL, when it differs from the one zero-cache is
181    /// configured with.
182    pub mutate_url: Option<String>,
183    /// Headers zero-cache sends to `mutate_url`.
184    pub mutate_headers: Option<Vec<(String, String)>>,
185    /// The API server's query URL, when it differs from the one zero-cache is
186    /// configured with.
187    pub query_url: Option<String>,
188    /// Headers zero-cache sends to `query_url`.
189    pub query_headers: Option<Vec<(String, String)>>,
190    /// The server needs a newer client. Defaults to doing nothing.
191    pub on_update_needed: Option<UpdateNeededHandler>,
192    /// The server no longer has the client's state, or the stored replica
193    /// didn't decode and the local store was reset. Without it, a client
194    /// the server lost replaces itself by a new one on the same store, as
195    /// JS reloads the page (see [`Zero::new`]).
196    ///
197    /// [`Zero::new`]: crate::client::Zero::new
198    pub on_client_state_not_found: Option<Rc<dyn Fn()>>,
199    /// Where the client keeps its replica. Defaults to
200    /// [`ZeroStorage::Memory`], which keeps nothing across restarts.
201    pub storage: ZeroStorage,
202    /// The local store failed, so the client runs on what it has in memory
203    /// and persists nothing more (`onStorageFailure`). Called once.
204    pub on_storage_failure: Option<StorageFailureHandler>,
205    /// The least severe messages logged.
206    pub log_level: LogLevel,
207    /// Where messages are logged. Defaults to [`StderrLogSink`].
208    pub log_sink: Option<Rc<dyn LogSink>>,
209    /// How long the app is hidden before the client disconnects
210    /// (`hiddenTabDisconnectDelay`); see [`Zero::set_visible`].
211    ///
212    /// [`Zero::set_visible`]: crate::client::Zero::set_visible
213    pub hidden_disconnect_delay_ms: f64,
214    /// A W3C `traceparent` for each query change and push the client sends
215    /// (`getTraceparent`), so zero-cache and the API server can join the
216    /// app's trace. Called once per message; `None` sends none.
217    pub get_traceparent: Option<Rc<dyn Fn() -> Option<String>>>,
218    /// Wraps every update of the views (`batchViewUpdates`), for a UI that
219    /// renders a batch of state changes at once: each new view's creation,
220    /// and each change that updates the views (a poke, a mutation, the
221    /// first hydration), with its listener calls. It must call the function
222    /// it is given, once, before it returns.
223    pub batch_view_updates: Option<BatchViewUpdates>,
224    /// How long, in ms, a query may take to materialize, including the
225    /// server's and the network's time, before the client logs a warning
226    /// (`slowMaterializeThreshold`). Faster materializations are logged at
227    /// debug level.
228    pub slow_materialize_threshold_ms: f64,
229}
230
231impl Default for ZeroOptions {
232    fn default() -> Self {
233        ZeroOptions {
234            cache_url: None,
235            user_id: None,
236            auth: None,
237            storage_key: None,
238            ping_timeout_ms: 5_000.0,
239            disconnect_timeout_ms: 60_000.0,
240            query_change_throttle_ms: 10.0,
241            max_header_length: 8 * 1024,
242            mutate_url: None,
243            mutate_headers: None,
244            query_url: None,
245            query_headers: None,
246            on_update_needed: None,
247            on_client_state_not_found: None,
248            storage: ZeroStorage::Memory,
249            on_storage_failure: None,
250            log_level: LogLevel::Warn,
251            log_sink: None,
252            hidden_disconnect_delay_ms: 0.0,
253            get_traceparent: None,
254            batch_view_updates: None,
255            slow_materialize_threshold_ms: 5_000.0,
256        }
257    }
258}
259
260/// Called when the local store fails (`onStorageFailure`).
261pub type StorageFailureHandler = Rc<dyn Fn(&StorageFailure)>;
262
263/// Called when the server needs a newer client (`onUpdateNeeded`).
264pub type UpdateNeededHandler = Rc<dyn Fn(&UpdateNeededReason)>;
265
266/// [`ZeroOptions::batch_view_updates`]: called with the function that
267/// updates the views.
268pub type BatchViewUpdates = Rc<dyn Fn(&mut dyn FnMut())>;
269
270/// Why the client must be updated (`UpdateNeededReason`).
271#[derive(Clone, Debug, PartialEq, Eq)]
272pub struct UpdateNeededReason {
273    /// `VersionNotSupported`, `SchemaVersionNotSupported`, or
274    /// `NewClientGroup`.
275    pub type_: String,
276    /// The server's explanation, if it gave one.
277    pub message: Option<String>,
278}
279
280impl UpdateNeededReason {
281    /// The JS reason object.
282    pub fn to_json(&self) -> Value {
283        let mut o = Object::new();
284        o.insert("type", Value::String(self.type_.clone()));
285        if let Some(m) = &self.message {
286            o.insert("message", Value::String(m.clone()));
287        }
288        Value::Object(o)
289    }
290}
291
292/// How a mutation ended, on the client or on the server
293/// (`MutatorResultDetails` in `packages/zero-client/src/client/custom.ts`).
294#[derive(Clone, Debug, PartialEq)]
295pub enum MutationOutcome {
296    /// The mutation ran.
297    Success {
298        /// What the mutator returned (client) or the server's result.
299        data: Option<Value>,
300    },
301    /// The mutator or the API server raised an [`ApplicationError`].
302    AppError {
303        /// The error's message.
304        message: String,
305        /// The error's details.
306        details: Option<Value>,
307    },
308    /// Zero couldn't run the mutation or learn its result, such as while
309    /// offline or when the connection closed. On the server side this
310    /// doesn't mean the mutation wasn't applied: the client sends pending
311    /// mutations again when it reconnects.
312    ZeroError {
313        /// Why.
314        message: String,
315    },
316}
317
318impl MutationOutcome {
319    /// The JS result object.
320    pub fn to_json(&self) -> Value {
321        let mut o = Object::new();
322        match self {
323            MutationOutcome::Success { data } => {
324                o.insert("type", Value::from("success"));
325                if let Some(d) = data {
326                    o.insert("data", d.clone());
327                }
328            }
329            MutationOutcome::AppError { message, details } => {
330                let mut e = Object::new();
331                e.insert("type", Value::from("app"));
332                e.insert("message", Value::String(message.clone()));
333                if let Some(d) = details {
334                    e.insert("details", d.clone());
335                }
336                o.insert("type", Value::from("error"));
337                o.insert("error", Value::Object(e));
338            }
339            MutationOutcome::ZeroError { message } => {
340                let mut e = Object::new();
341                e.insert("type", Value::from("zero"));
342                e.insert("message", Value::String(message.clone()));
343                o.insert("type", Value::from("error"));
344                o.insert("error", Value::Object(e));
345            }
346        }
347        Value::Object(o)
348    }
349}
350
351/// What `mutate` returns: the outcome of the optimistic run on the client,
352/// and the server's.
353#[derive(Clone)]
354pub struct MutatorResult {
355    /// Settles when the mutator has run on the client: what it returned, or
356    /// what it raised.
357    pub client: Deferred<MutationOutcome>,
358    /// Settles when the server has run the mutation, or when the client can't
359    /// learn its result.
360    pub server: Deferred<MutationOutcome>,
361}
362
363/// A mutator bound to the app's context: runs it in a transaction.
364pub(crate) type MutatorFn =
365    Rc<dyn Fn(&Transaction, Option<&Value>) -> Result<Option<Value>, Error>>;
366
367/// A connection's readiness, as the pusher waits on it
368/// (`#connectionReadyResolver`).
369#[derive(Clone, Default)]
370struct ReadyPromise(Rc<RefCell<ReadyState>>);
371
372#[derive(Default)]
373struct ReadyState {
374    settled: Option<bool>,
375    /// Why the connection failed, once it has.
376    error: Option<ConnectionError>,
377    waiters: Vec<Box<dyn FnOnce(bool)>>,
378}
379
380impl ReadyPromise {
381    fn is(&self, other: &ReadyPromise) -> bool {
382        Rc::ptr_eq(&self.0, &other.0)
383    }
384
385    fn settle(&self, value: bool, event_loop: &EventLoop) {
386        let waiters = {
387            let mut s = self.0.borrow_mut();
388            if s.settled.is_some() {
389                return;
390            }
391            s.settled = Some(value);
392            std::mem::take(&mut s.waiters)
393        };
394        for w in waiters {
395            event_loop.microtask(move || w(value));
396        }
397    }
398
399    /// Calls `waiter` in a microtask once settled.
400    fn on_settle(&self, event_loop: &EventLoop, waiter: impl FnOnce(bool) + 'static) {
401        let settled = self.0.borrow().settled;
402        match settled {
403            Some(v) => event_loop.microtask(move || waiter(v)),
404            None => self.0.borrow_mut().waiters.push(Box::new(waiter)),
405        }
406    }
407}
408
409#[derive(Default)]
410struct ConnectAttempt {
411    events: RefCell<Vec<(f64, String)>>,
412    aborted: Cell<bool>,
413    abort_reason: RefCell<Option<ConnectionError>>,
414    timeout: RefCell<Option<Box<dyn Timer>>>,
415    finished: Cell<bool>,
416    on_connected: RefCell<Option<Box<dyn FnOnce()>>>,
417    /// When the socket was opened, for the open and `connected` logs.
418    start: Cell<f64>,
419}
420
421impl ConnectAttempt {
422    fn cancel_timeout(&self) {
423        if let Some(t) = self.timeout.borrow_mut().take() {
424            t.cancel();
425        }
426    }
427}
428
429/// A pending mutation's identity.
430type MutationKey = (String, u64);
431
432fn mutation_key(client_id: &str, id: f64) -> MutationKey {
433    (client_id.to_owned(), id.to_bits())
434}
435
436/// The replica as JS's last persist left it in the store: the snapshot it
437/// persisted (by cookie), its main map over that snapshot, the keys where
438/// the two may differ, and the pending mutations it ran.
439#[derive(Default)]
440struct PersistedState {
441    cookie: Value,
442    main: KvMap,
443    touched: HashSet<String>,
444    mutations: HashSet<MutationKey>,
445}
446
447/// The engine's end of one socket.
448pub(crate) struct SocketHandle {
449    me: Weak<SocketHandle>,
450    engine: Weak<Engine>,
451    connection: RefCell<Option<Box<dyn SocketConnection>>>,
452    closed: Cell<bool>,
453}
454
455impl SocketHandle {
456    fn new(engine: Weak<Engine>) -> Rc<Self> {
457        Rc::new_cyclic(|me| SocketHandle {
458            me: me.clone(),
459            engine,
460            connection: RefCell::new(None),
461            closed: Cell::new(false),
462        })
463    }
464
465    fn send(&self, text: &str) {
466        if self.closed.get() {
467            return;
468        }
469        if let Some(c) = &*self.connection.borrow() {
470            c.send(text);
471        }
472    }
473
474    fn close(&self, code: Option<u16>) {
475        if self.closed.replace(true) {
476            return;
477        }
478        let connection = self.connection.borrow_mut().take();
479        if let Some(c) = connection {
480            c.close(code);
481        }
482    }
483}
484
485impl SocketEvents for SocketHandle {
486    fn did_open(&self) {
487        if self.closed.get() {
488            return;
489        }
490        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
491            e.socket_did_open(&me);
492        }
493    }
494
495    fn did_receive(&self, message: SocketMessage) {
496        if self.closed.get() {
497            return;
498        }
499        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
500            e.socket_did_receive(&me, message);
501        }
502    }
503
504    fn did_close(&self, code: u16, reason: &str, was_clean: bool) {
505        if self.closed.get() {
506            return;
507        }
508        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
509            e.socket_did_close(&me, code, reason, was_clean);
510        }
511    }
512}
513
514type VisibilityWaiter = (bool, u64, Box<dyn FnOnce()>);
515
516/// What an inspector request hears: zero-cache's answer, the `inspect`
517/// body, or why there is none.
518pub(crate) type InspectAnswer = Box<dyn FnOnce(Result<Object, Error>)>;
519
520/// An inspector request (`rpcNoAuthTry` in `lazy-inspector.ts`), waiting
521/// for the connection or, once sent, for the answer with its ID.
522struct InspectRequest {
523    id: String,
524    /// What to send, until it is.
525    body: Option<Object>,
526    sent: bool,
527    /// Waiting for the next connection attempt: the last one's socket
528    /// closed before the request went out.
529    parked: bool,
530    done: InspectAnswer,
531}
532
533pub(crate) struct Engine {
534    me: Weak<Engine>,
535    pub(crate) schema: Arc<Schema>,
536    pub(crate) options: ZeroOptions,
537    pub(crate) client_id: String,
538    pub(crate) client_group_id: String,
539    profile_id: String,
540    pub(crate) user_id: Option<String>,
541    pub(crate) event_loop: Rc<EventLoop>,
542    scheduler: Rc<dyn Scheduler>,
543    pub(crate) log: Log,
544    socket_factory: Option<Rc<dyn SocketFactory>>,
545    socket_origin: Option<String>,
546    mutators: Rc<HashMap<String, MutatorFn>>,
547    client_schema: crate::protocol::ClientSchema,
548
549    pub(crate) store: RefCell<ReplicaStore>,
550    pub(crate) database: Rc<ReplicaDatabase>,
551    lifecycle_timers: RefCell<Vec<Box<dyn Timer>>>,
552    ivm_main: Rc<IvmBranch>,
553    pub(crate) context: Rc<QueryContext>,
554    pub(crate) connection_manager: Rc<ConnectionManager>,
555    mutation_tracker: Rc<MutationTracker>,
556    pub(crate) query_manager: Rc<QueryManager>,
557    poke_handler: RefCell<PokeHandler>,
558
559    /// The current token: `options.auth`, then the last one `connect`
560    /// set; empty for none.
561    pub(crate) auth: RefCell<String>,
562    rep_closed: Cell<bool>,
563    storage_failure_reported: Cell<bool>,
564    socket: RefCell<Option<Rc<SocketHandle>>>,
565    connect_cookie: RefCell<Value>,
566    init_connection_queries: RefCell<Option<OrderedMap<UpQueriesPatchOp>>>,
567    /// Deleted clients still to tell the server about on this connection.
568    deleted_clients: RefCell<Option<DeleteClientsBody>>,
569    last_mutation_id_received: Cell<f64>,
570    last_mutation_ids_sent: RefCell<HashMap<String, f64>>,
571    persisted: RefCell<PersistedState>,
572    persist_timer: RefCell<Option<Box<dyn Timer>>>,
573    last_persist_at: Cell<Option<f64>>,
574    additional_connect_params: RefCell<Option<Vec<(String, String)>>>,
575    /// When the current connection got the server's `connected`.
576    connected_at: Cell<Option<f64>>,
577    /// Consecutive transport failures, which set the reconnect backoff.
578    transport_failures: Cell<u32>,
579    current_attempt: RefCell<Option<Rc<ConnectAttempt>>>,
580    connection_ready: RefCell<ReadyPromise>,
581    recent_sent_messages: RefCell<VecDeque<Value>>,
582    mutation_rejection: RefCell<Option<ConnectionError>>,
583    client_group_disabled: Cell<bool>,
584
585    // The run loop's current wait. Each wait gets a new token, so a wait
586    // that was superseded does nothing when it ends.
587    loop_token: Cell<u64>,
588    abort_ping_timeout: RefCell<Rc<dyn Fn()>>,
589    on_pong: RefCell<Rc<dyn Fn()>>,
590    /// Pings the server now, when the run loop waits while connected.
591    ping_now: RefCell<Rc<dyn Fn()>>,
592    /// The run loop's backoff before it reconnects.
593    backoff: RefCell<Option<(u64, Box<dyn Timer>)>>,
594
595    // The push loop (Replicache's push `ConnectionLoop`).
596    push_timer: RefCell<Option<Box<dyn Timer>>>,
597    push_in_flight: Cell<bool>,
598    push_requested: Cell<bool>,
599    push_request_counter: Cell<u64>,
600
601    /// Trails the app's visibility by `hidden_disconnect_delay_ms` when it
602    /// is hidden (`DocumentVisibilityWatcher`).
603    pub(crate) visible: Cell<bool>,
604    hidden_timer: RefCell<Option<Box<dyn Timer>>>,
605    visibility_waiters: RefCell<Vec<VisibilityWaiter>>,
606    pending_waiters: RefCell<Vec<Box<dyn FnOnce()>>>,
607
608    /// Set while a mutator runs, for a mutation or a rebase: Replicache's
609    /// write lock. Mutations that start meanwhile wait, in order.
610    write_locked: Cell<bool>,
611    waiting_for_write_lock: RefCell<VecDeque<Box<dyn FnOnce()>>>,
612
613    /// What `Zero` does when the client can't sync any longer: by default
614    /// `ZeroOptions::on_client_state_not_found`.
615    pub(crate) on_client_state_not_found: RefCell<Option<Rc<dyn Fn()>>>,
616    /// What `Zero` does when the app must be updated to sync, besides
617    /// calling `ZeroOptions::on_update_needed`.
618    pub(crate) on_update_needed: RefCell<Option<UpdateNeededHandler>>,
619    /// The inspector's requests, oldest first.
620    inspect_requests: RefCell<Vec<InspectRequest>>,
621}
622
623fn noop() -> Rc<dyn Fn()> {
624    Rc::new(|| {})
625}
626
627/// `JSON.stringify(recentSentMessages, null, 2)`.
628fn pretty_recent_messages(messages: &VecDeque<Value>) -> String {
629    let entries: Vec<String> = messages
630        .iter()
631        .map(|m| {
632            let fields: Vec<String> = ["type", "size", "snippet"]
633                .iter()
634                .filter_map(|k| {
635                    m.get(k).map(|v| {
636                        format!(
637                            "    {}: {}",
638                            json::stringify(&Value::from(*k)),
639                            json::stringify(v)
640                        )
641                    })
642                })
643                .collect();
644            format!("  {{\n{}\n  }}", fields.join(",\n"))
645        })
646        .collect();
647    format!("[\n{}\n]", entries.join(",\n"))
648}
649
650fn bad_order_message(name: &str, received: &str, last: &str) -> String {
651    format!(
652        "Received {name} {received} is < than last snapshot {name} {last}; ignoring client view"
653    )
654}
655
656/// The synced aggregate rows `write` wrote, with their values now.
657fn aggregate_writes(write: &KvWrite) -> AggregateWrites {
658    let mut writes: AggregateWrites = write
659        .writes
660        .iter()
661        .filter(|k| k.starts_with(AGGREGATES_PREFIX))
662        .map(|k| (k.clone(), write.get(k).cloned()))
663        .collect();
664    writes.sort_by(|a, b| a.0.cmp(&b.0));
665    writes
666}
667
668impl Engine {
669    pub(crate) fn new(
670        schema: Arc<Schema>,
671        options: ZeroOptions,
672        mutators: Rc<HashMap<String, MutatorFn>>,
673        base: Rc<dyn Scheduler>,
674        socket_factory: Option<Rc<dyn SocketFactory>>,
675        visible: bool,
676    ) -> Result<Rc<Engine>, Error> {
677        if options.user_id.as_deref() == Some("") {
678            return Err(ConnectionError::client(
679                ClientErrorKind::Internal,
680                "ZeroOptions.userID should not be empty. Omit it entirely for logged-out clients.",
681            )
682            .into());
683        }
684        Self::check_auth_valid(options.user_id.as_deref(), options.auth.as_deref())?;
685        let socket_origin = options
686            .cache_url
687            .as_deref()
688            .map(Self::socket_origin)
689            .transpose()?;
690        let event_loop = EventLoop::new(base.clone());
691        let scheduler: Rc<dyn Scheduler> = Rc::new(LoopScheduler {
692            base: base.clone(),
693            event_loop: Rc::downgrade(&event_loop),
694        });
695        let client_id = random_id();
696        let sink = options
697            .log_sink
698            .clone()
699            .unwrap_or_else(|| Rc::new(StderrLogSink));
700        let log = Log::new(options.log_level, sink).with("clientID", &client_id);
701        let client_schema = schema.client_schema();
702        let schema_hash = base36(xxhash32::h64(&json::stringify(
703            &client_schema.normalized().to_json(),
704        )));
705
706        // Open the database and join a client group (`initClientV6`).
707        let name = ReplicaDatabase::name(
708            options.user_id.as_deref(),
709            options.storage_key.as_deref().unwrap_or(""),
710            options.mutate_url.as_deref(),
711            options.query_url.as_deref(),
712            &schema_hash,
713        );
714        let mut mutator_names: Vec<String> = mutators.keys().cloned().collect();
715        mutator_names.sort();
716        let (mut database, mut open_failure, mut corrupt) =
717            ReplicaDatabase::open(&options.storage, &name);
718        let joined = match database.init_client(&client_id, &mutator_names, base.now()) {
719            // What the client group holds doesn't decode, so the client
720            // state is lost: drop the database and start a new one
721            // (`#handleInvalidRefCount`).
722            Err(LoadError::Corrupt(_)) => {
723                corrupt = true;
724                database.drop_database();
725                database.release();
726                (database, open_failure, _) = ReplicaDatabase::open(&options.storage, &name);
727                database.init_client(&client_id, &mutator_names, base.now())
728            }
729            r => r,
730        };
731        let (client_group_id, group) = match joined {
732            Ok(j) => j,
733            // The client group can't be read: run on memory, as when the
734            // open fails.
735            Err(e) => {
736                database.release();
737                database = ReplicaDatabase::fallback(&name, &options.storage);
738                open_failure = Some(match e {
739                    LoadError::Failure(f) => f,
740                    LoadError::Corrupt(m) => {
741                        StorageFailure::new(super::storage::StorageFailureKind::IoError, m)
742                    }
743                });
744                database
745                    .init_client(&client_id, &mutator_names, base.now())
746                    .expect("a memory database reads")
747            }
748        };
749        let profile_id = database.profile_id();
750        // Drop stored mutation IDs that aren't mutation IDs (a corrupt
751        // store) rather than trust them.
752        let is_mutation_id = |id: f64| id >= 0.0 && number::is_safe_integer(id);
753        let last_mutation_ids = group
754            .last_mutation_ids
755            .into_iter()
756            .filter(|(cid, lmid)| {
757                let ok = is_mutation_id(*lmid);
758                if !ok {
759                    log.error(|| {
760                        format!(
761                            "Dropping invalid stored last mutation ID of {cid}: {}",
762                            number::format(*lmid)
763                        )
764                    });
765                }
766                ok
767            })
768            .collect();
769        let mutations: Vec<StoredMutation> = group
770            .mutations
771            .into_iter()
772            .filter(|m| {
773                let ok = is_mutation_id(m.id);
774                if !ok {
775                    log.error(|| {
776                        format!(
777                            "Dropping stored mutation \"{}\" of {} with invalid ID {}",
778                            m.name,
779                            m.client_id,
780                            number::format(m.id)
781                        )
782                    });
783                }
784                ok
785            })
786            .collect();
787        let store = ReplicaStore::new(
788            Snapshot {
789                map: group.entries,
790                cookie: group.cookie,
791                last_mutation_ids,
792            },
793            mutations
794                .iter()
795                .map(|m| PendingMutation {
796                    id: m.id,
797                    client_id: m.client_id.clone(),
798                    name: m.name.clone(),
799                    args: m.args.clone(),
800                    timestamp: m.timestamp,
801                    writes: HashSet::new(),
802                })
803                .collect(),
804        );
805        let stored_aggregate_writes: HashMap<MutationKey, AggregateWrites> = mutations
806            .into_iter()
807            .map(|m| (mutation_key(&m.client_id, m.id), m.aggregate_writes))
808            .collect();
809        let ivm_main = IvmBranch::new(schema.clone());
810        let connection_manager = ConnectionManager::new(
811            options.disconnect_timeout_ms,
812            scheduler.clone(),
813            Rc::downgrade(&event_loop),
814        );
815
816        let engine = Rc::new_cyclic(|me: &Weak<Engine>| {
817            let mutation_tracker = {
818                let (a, f, q) = (me.clone(), me.clone(), me.clone());
819                Rc::new(MutationTracker::new(
820                    client_id.clone(),
821                    Box::new(move |id| {
822                        if let Some(e) = a.upgrade() {
823                            e.send_if_connected(Upstream::AckMutationResponses(id));
824                        }
825                    }),
826                    Box::new(move |error| {
827                        if let Some(e) = f.upgrade() {
828                            e.disconnect(error, None);
829                        }
830                    }),
831                    Box::new(move || {
832                        if let Some(e) = q.upgrade() {
833                            e.query_manager.on_all_mutations_applied();
834                        }
835                    }),
836                ))
837            };
838            let query_manager = {
839                let (s, p) = (me.clone(), me.clone());
840                QueryManager::new(
841                    client_id.clone(),
842                    scheduler.clone(),
843                    options.query_change_throttle_ms,
844                    Box::new(move |mut body| {
845                        if let Some(e) = s.upgrade() {
846                            body.traceparent = e.traceparent();
847                            e.send_if_connected(Upstream::ChangeDesiredQueries(body));
848                        }
849                    }),
850                    Box::new(move || p.upgrade().map_or(0, |e| e.mutation_tracker.size())),
851                    log.clone(),
852                    options.slow_materialize_threshold_ms,
853                )
854            };
855            let context = {
856                let (a, u, c) = (me.clone(), me.clone(), me.clone());
857                let l = event_loop.clone();
858                QueryContext::new(
859                    ivm_main.clone(),
860                    Box::new(move |id, ast, ttl, got| match a.upgrade() {
861                        Some(e) => {
862                            e.query_manager
863                                .add_custom(ast, &id.name, &id.args, ttl, Some(got))
864                        }
865                        None => Box::new(|| {}),
866                    }),
867                    Rc::new(move |id, ttl| {
868                        if let Some(e) = u.upgrade() {
869                            e.query_manager.update_custom(&id.name, &id.args, ttl);
870                        }
871                    }),
872                    Rc::new(move |f| l.microtask(f)),
873                    Some(log.clone()),
874                    Box::new(move || match c.upgrade()?.connection_manager.state() {
875                        State::Closed(r) => Some(r),
876                        _ => None,
877                    }),
878                    true,
879                    options.batch_view_updates.clone(),
880                    Some(query_manager.clone()),
881                )
882            };
883            Engine {
884                me: me.clone(),
885                poke_handler: RefCell::new(PokeHandler::new(client_id.clone(), schema.clone())),
886                schema: schema.clone(),
887                user_id: options.user_id.clone(),
888                // Set now, as JS's constructor does, so a token from
889                // `connect` right after init isn't replaced when the run
890                // loop starts.
891                auth: RefCell::new(options.auth.clone().unwrap_or_default()),
892                options: options.clone(),
893                client_id: client_id.clone(),
894                client_group_id: client_group_id.clone(),
895                profile_id,
896                event_loop: event_loop.clone(),
897                scheduler: scheduler.clone(),
898                log: log.clone(),
899                socket_factory,
900                socket_origin,
901                mutators,
902                client_schema,
903                store: RefCell::new(store),
904                database: database.clone(),
905                lifecycle_timers: RefCell::default(),
906                ivm_main: ivm_main.clone(),
907                context,
908                connection_manager: connection_manager.clone(),
909                mutation_tracker,
910                query_manager,
911                rep_closed: Cell::new(false),
912                storage_failure_reported: Cell::new(false),
913                socket: RefCell::new(None),
914                connect_cookie: RefCell::new(Value::Null),
915                init_connection_queries: RefCell::new(None),
916                deleted_clients: RefCell::new(None),
917                last_mutation_id_received: Cell::new(0.0),
918                last_mutation_ids_sent: RefCell::default(),
919                persisted: RefCell::default(),
920                persist_timer: RefCell::new(None),
921                last_persist_at: Cell::new(None),
922                additional_connect_params: RefCell::new(None),
923                connected_at: Cell::new(None),
924                transport_failures: Cell::new(0),
925                current_attempt: RefCell::new(None),
926                connection_ready: RefCell::default(),
927                recent_sent_messages: RefCell::default(),
928                mutation_rejection: RefCell::new(None),
929                client_group_disabled: Cell::new(false),
930                loop_token: Cell::new(0),
931                abort_ping_timeout: RefCell::new(noop()),
932                on_pong: RefCell::new(noop()),
933                ping_now: RefCell::new(noop()),
934                backoff: RefCell::new(None),
935                push_timer: RefCell::new(None),
936                push_in_flight: Cell::new(false),
937                push_requested: Cell::new(false),
938                push_request_counter: Cell::new(0),
939                visible: Cell::new(visible),
940                hidden_timer: RefCell::new(None),
941                visibility_waiters: RefCell::default(),
942                pending_waiters: RefCell::default(),
943                write_locked: Cell::new(false),
944                waiting_for_write_lock: RefCell::default(),
945                on_client_state_not_found: RefCell::new(None),
946                on_update_needed: RefCell::new(None),
947                inspect_requests: RefCell::default(),
948            }
949        });
950
951        let e = &engine;
952        {
953            let w = Rc::downgrade(e);
954            // The closure is dropped with the manager; no need to keep the
955            // unsubscribe.
956            let _ = connection_manager.subscribe(move |state| {
957                if let (Some(e), State::Closed(reason)) = (w.upgrade(), state) {
958                    e.query_manager.handle_closed(reason);
959                }
960            });
961            let w = Rc::downgrade(e);
962            let _ = connection_manager.subscribe(move |state| {
963                if let Some(e) = w.upgrade() {
964                    e.on_connection_state_change(state);
965                }
966            });
967        }
968        e.on_connection_state_change(&connection_manager.state());
969
970        // Load the snapshot into the sources and replay the pending
971        // mutations on it, with the synced aggregate rows they wrote when
972        // they were last persisted, as JS's stored commits have them.
973        let initial = diff_all(&KvMap::default(), &e.store.borrow().snapshot.map);
974        e.ivm_main.apply(&initial, &HashMap::new());
975        let pending = e.store.borrow().pending.clone();
976        if !pending.is_empty() {
977            let snapshot_map = e.store.borrow().snapshot.map.clone();
978            let (main, replayed, touched, _) = e.replay(
979                &pending,
980                &snapshot_map,
981                e.ivm_main.fork(),
982                &stored_aggregate_writes,
983                &mut |_, _| {},
984            );
985            let diff = {
986                let mut store = e.store.borrow_mut();
987                let snapshot = store.snapshot.clone();
988                store.advance(snapshot, main, replayed, Some(&touched))
989            };
990            e.ivm_main.apply(&diff, &HashMap::new());
991        }
992        {
993            let store = e.store.borrow();
994            *e.persisted.borrow_mut() = PersistedState {
995                cookie: store.snapshot.cookie.clone(),
996                main: store.main.clone(),
997                touched: store.pending_writes(),
998                mutations: store
999                    .pending
1000                    .iter()
1001                    .map(|m| mutation_key(&m.client_id, m.id))
1002                    .collect(),
1003            };
1004        }
1005        {
1006            let w = Rc::downgrade(e);
1007            database.add_failure_listener(
1008                &e.client_id,
1009                Rc::new(move |f| {
1010                    if let Some(e) = w.upgrade() {
1011                        e.storage_failed(f, false);
1012                    }
1013                }),
1014            );
1015        }
1016        // Another client of the group stopped (the `onDelete` of
1017        // `makeActiveClientsManager`): tell the server, if connected.
1018        {
1019            let w = Rc::downgrade(e);
1020            database.client_started(
1021                &e.client_id,
1022                &e.client_group_id,
1023                Rc::new(move |id| {
1024                    if let Some(e) = w.upgrade() {
1025                        let id = id.to_owned();
1026                        e.event_loop.clone().dispatch(|| {
1027                            e.send_if_connected(Upstream::DeleteClients(DeleteClientsBody {
1028                                client_ids: Some(vec![id]),
1029                                client_group_ids: None,
1030                            }))
1031                        });
1032                    }
1033                }),
1034            );
1035        }
1036        if corrupt {
1037            // This client already starts on the new database, which is what
1038            // replacing it would do; the app's callback still hears it, as
1039            // in JS.
1040            e.log.error(|| {
1041                format!("Client state is corrupt; dropped database {name} and started a new one")
1042            });
1043            // Only the app's callback: this client is already on a new
1044            // database, so it isn't replaced (as in Swift).
1045            e.micro(|e| {
1046                if let Some(f) = &e.options.on_client_state_not_found {
1047                    f();
1048                }
1049            });
1050        }
1051        if let Some(f) = open_failure {
1052            e.log.warn(|| {
1053                format!(
1054                    "Local store storage failed ({}) opening {name}; running this instance in memory: {}",
1055                    f.kind.as_str(),
1056                    f.message
1057                )
1058            });
1059            e.micro(move |e| e.storage_failed(&f, true));
1060        }
1061        e.start_lifecycle();
1062
1063        // The replica is loaded: report its got queries and mutation
1064        // results, as the first runs of the store watches do, then build
1065        // the pipelines.
1066        let main = e.store.borrow().main.clone();
1067        e.fire_watchers(&diff_all(&KvMap::default(), &main), true);
1068        e.context.mark_pipelines_ready();
1069        e.micro(|e| e.start_run_loop());
1070        Ok(engine)
1071    }
1072
1073    fn check_auth_valid(user_id: Option<&str>, auth: Option<&str>) -> Result<(), Error> {
1074        if user_id.is_none() && auth.is_some_and(|a| !a.is_empty()) {
1075            return Err(ConnectionError::client(
1076                ClientErrorKind::Internal,
1077                "ZeroOptions.userID is required when auth is set.",
1078            )
1079            .into());
1080        }
1081        Ok(())
1082    }
1083
1084    /// The WebSocket origin of the zero-cache URL (`getServer`,
1085    /// `toWSString`).
1086    fn socket_origin(url: &str) -> Result<String, Error> {
1087        let (scheme, rest) = if let Some(rest) = url.strip_prefix("http://") {
1088            ("ws://", rest)
1089        } else if let Some(rest) = url.strip_prefix("https://") {
1090            ("wss://", rest)
1091        } else {
1092            return Err(ConnectionError::client(
1093                ClientErrorKind::Internal,
1094                "ZeroOptions.cacheURL must use the \"http\" or \"https\" scheme.",
1095            )
1096            .into());
1097        };
1098        let mut rest = rest.to_owned();
1099        if !rest.contains('/') {
1100            rest.push('/');
1101        }
1102        Ok(format!("{scheme}{rest}"))
1103    }
1104
1105    fn micro(&self, f: impl FnOnce(&Engine) + 'static) {
1106        let w = self.me.clone();
1107        self.event_loop.microtask(move || {
1108            if let Some(e) = w.upgrade() {
1109                f(&e);
1110            }
1111        });
1112    }
1113
1114    fn later(&self, ms: f64, f: impl FnOnce(&Engine) + 'static) -> Box<dyn Timer> {
1115        let w = self.me.clone();
1116        self.scheduler.schedule(
1117            ms,
1118            Box::new(move || {
1119                if let Some(e) = w.upgrade() {
1120                    f(&e);
1121                }
1122            }),
1123        )
1124    }
1125
1126    fn status_is(&self, status: ConnectionStatus) -> bool {
1127        self.connection_manager.is(status)
1128    }
1129
1130    // MARK: Sending
1131
1132    /// The app's traceparent for a message about to be sent.
1133    fn traceparent(&self) -> Option<String> {
1134        self.options.get_traceparent.as_ref().and_then(|f| f())
1135    }
1136
1137    fn send_if_connected(&self, message: Upstream) {
1138        if self.status_is(ConnectionStatus::Connected) {
1139            self.send(message);
1140        }
1141    }
1142
1143    fn send(&self, message: Upstream) {
1144        let Some(socket) = self.socket.borrow().clone() else {
1145            return;
1146        };
1147        let text = message.to_text();
1148        if message.tag() != "ping" {
1149            let units: Vec<u16> = text.encode_utf16().collect();
1150            let snippet = if units.len() > 200 {
1151                String::from_utf16(&units[..200]).unwrap_or_else(|_| text.clone())
1152            } else {
1153                text.clone()
1154            };
1155            let mut o = Object::new();
1156            o.insert("type", Value::from(message.tag()));
1157            o.insert("size", Value::Number(units.len() as f64));
1158            o.insert("snippet", Value::String(snippet));
1159            let mut recent = self.recent_sent_messages.borrow_mut();
1160            if recent.len() >= 5 {
1161                recent.pop_front();
1162            }
1163            recent.push_back(Value::Object(o));
1164        }
1165        socket.send(&text);
1166    }
1167
1168    // MARK: The run loop
1169
1170    fn start_run_loop(&self) {
1171        if self.socket_origin.is_none() {
1172            self.connection_manager
1173                .disconnected(ConnectionError::client(
1174                    ClientErrorKind::NoSocketOrigin,
1175                    "No server socket origin provided",
1176                ));
1177            return;
1178        }
1179        self.connection_manager.initialized();
1180        self.iterate();
1181    }
1182
1183    fn next_loop_token(&self) -> u64 {
1184        let t = self.loop_token.get() + 1;
1185        self.loop_token.set(t);
1186        t
1187    }
1188
1189    fn is_token(&self, token: u64) -> bool {
1190        self.loop_token.get() == token
1191    }
1192
1193    /// One turn of the JS run loop: acts on the connection status.
1194    fn iterate(&self) {
1195        if !self.connection_manager.should_continue_run_loop() {
1196            return;
1197        }
1198        let token = self.next_loop_token();
1199        match self.connection_manager.status() {
1200            ConnectionStatus::Connecting | ConnectionStatus::Disconnected => {
1201                if !self.visible.get() {
1202                    // Wait for the app to come back, or for a state change.
1203                    self.on_visibility(true, token, |e| e.begin_attempt());
1204                    self.on_state_change(token, |e, state| e.resume_after_state_change(state));
1205                    return;
1206                }
1207                // The app is visible, which wins the race with a state
1208                // change.
1209                self.micro(move |e| {
1210                    if e.is_token(token) {
1211                        e.begin_attempt();
1212                    }
1213                });
1214            }
1215            ConnectionStatus::Connected => self.wait_while_connected(token),
1216            ConnectionStatus::NeedsAuth | ConnectionStatus::Error => {
1217                let status = self.connection_manager.status();
1218                self.log.info(|| {
1219                    format!(
1220                        "Run loop paused in {} state. Call zero.connection.connect() to resume.",
1221                        status.as_str()
1222                    )
1223                });
1224                self.wait_for_connect_request(token);
1225            }
1226            ConnectionStatus::Closed => {}
1227            ConnectionStatus::Initializing => unreachable!("The run loop runs once initialized"),
1228        }
1229    }
1230
1231    /// Ends the current wait of the run loop with `error` (the `catch` of
1232    /// the JS run loop).
1233    fn loop_failed(&self, error: ConnectionError) {
1234        let (transition, reason) = ConnectionTransition::of(&error);
1235        if !self.status_is(ConnectionStatus::Connected)
1236            && error.client_kind() != Some(ClientErrorKind::ClientClosed)
1237        {
1238            // Only errors that stop the run loop are errors. Reconnects and
1239            // auth failures are routine.
1240            let mut level = match transition {
1241                ConnectionTransition::Error => LogLevel::Error,
1242                ConnectionTransition::Disconnected | ConnectionTransition::Closed => LogLevel::Info,
1243                ConnectionTransition::None | ConnectionTransition::NeedsAuth => LogLevel::Warn,
1244            };
1245            if error.server_kind() == Some(ErrorKind::ClientNotFound) {
1246                level = LogLevel::Warn;
1247            }
1248            self.log
1249                .log(level, || format!("Failed to connect: {}", error.message()));
1250        }
1251        let mut backoff_ms = RUN_LOOP_INTERVAL_MS;
1252        let mut sleep_ms = None;
1253        match transition {
1254            ConnectionTransition::None => {
1255                // A transport failure is retried soon, then less often.
1256                // Errors the server sent keep the full interval, adjusted by
1257                // any backoff they carry.
1258                if reason.is_transport_failure() {
1259                    let failures = self.transport_failures.get() + 1;
1260                    self.transport_failures.set(failures);
1261                    backoff_ms = reconnect_backoff_ms(failures, RUN_LOOP_INTERVAL_MS, random_f64());
1262                }
1263                if let Some(ErrorBody::Backoff {
1264                    min_backoff_ms,
1265                    max_backoff_ms,
1266                    reconnect_params,
1267                    ..
1268                }) = reason.backoff()
1269                {
1270                    if let Some(min) = min_backoff_ms {
1271                        backoff_ms = backoff_ms.max(*min);
1272                    }
1273                    if let Some(max) = max_backoff_ms {
1274                        backoff_ms = backoff_ms.min(*max);
1275                    }
1276                    *self.additional_connect_params.borrow_mut() =
1277                        reconnect_params.as_ref().map(|params| {
1278                            let mut p = params.clone();
1279                            p.sort_by(|a, b| a.0.cmp(&b.0));
1280                            p
1281                        });
1282                }
1283                sleep_ms = Some(backoff_ms);
1284            }
1285            ConnectionTransition::NeedsAuth => self.connection_manager.needs_auth(reason),
1286            ConnectionTransition::Error => self.connection_manager.error(reason),
1287            ConnectionTransition::Disconnected => self.connection_manager.disconnected(reason),
1288            ConnectionTransition::Closed => {}
1289        }
1290        let token = self.next_loop_token();
1291        match sleep_ms {
1292            Some(ms) if ms > 0.0 => {
1293                let timer = self.later(ms, move |e| {
1294                    if e.is_token(token) {
1295                        e.backoff.borrow_mut().take();
1296                        e.iterate();
1297                    }
1298                });
1299                *self.backoff.borrow_mut() = Some((token, timer));
1300            }
1301            _ => self.micro(move |e| {
1302                if e.is_token(token) {
1303                    e.iterate();
1304                }
1305            }),
1306        }
1307    }
1308
1309    /// `throwIfConnectionError`: whether a state the run loop wakes up to
1310    /// stops it. Dropped sockets and timeouts just reconnect.
1311    fn connection_error(state: &State) -> Option<ConnectionError> {
1312        match state {
1313            State::NeedsAuth(r) | State::Error(r) | State::Closed(r) => Some(r.clone()),
1314            State::Connecting {
1315                reason: Some(r), ..
1316            }
1317            | State::Disconnected(r) => match r.client_kind() {
1318                Some(
1319                    ClientErrorKind::ConnectTimeout
1320                    | ClientErrorKind::AbruptClose
1321                    | ClientErrorKind::CleanClose
1322                    | ClientErrorKind::Hidden,
1323                ) => None,
1324                _ => Some(r.clone()),
1325            },
1326            _ => None,
1327        }
1328    }
1329
1330    /// Wakes the loop on the next state change, unless the wait is over.
1331    fn on_state_change(&self, token: u64, body: impl FnOnce(&Engine, State) + 'static) {
1332        let (a, b) = (self.me.clone(), self.me.clone());
1333        self.connection_manager.on_next_state_change(
1334            move || a.upgrade().is_some_and(|e| e.is_token(token)),
1335            move |state| {
1336                if let Some(e) = b.upgrade()
1337                    && e.is_token(token)
1338                {
1339                    body(&e, state);
1340                }
1341            },
1342        );
1343    }
1344
1345    fn resume_after_state_change(&self, state: State) {
1346        match Self::connection_error(&state) {
1347            Some(error) => self.loop_failed(error),
1348            None => self.iterate(),
1349        }
1350    }
1351
1352    /// Connected: ping after `ping_timeout_ms` without poke traffic.
1353    fn wait_while_connected(&self, token: u64) {
1354        let ready = self.connection_ready.borrow().clone();
1355        let timer: Rc<RefCell<Option<Box<dyn Timer>>>> = Rc::default();
1356        let cancel = {
1357            let timer = timer.clone();
1358            move || {
1359                if let Some(t) = timer.borrow_mut().take() {
1360                    t.cancel();
1361                }
1362            }
1363        };
1364        let end: Rc<dyn Fn()> = {
1365            let w = self.me.clone();
1366            Rc::new(move || {
1367                let Some(e) = w.upgrade() else { return };
1368                // The `finally` of the connected case.
1369                if !e.status_is(ConnectionStatus::Connected)
1370                    && e.connection_ready.borrow().is(&ready)
1371                {
1372                    *e.connection_ready.borrow_mut() = ReadyPromise::default();
1373                }
1374            })
1375        };
1376        {
1377            let w = self.me.clone();
1378            let cancel = cancel.clone();
1379            let end = end.clone();
1380            *self.abort_ping_timeout.borrow_mut() = Rc::new(move || {
1381                cancel();
1382                let end = end.clone();
1383                if let Some(e) = w.upgrade() {
1384                    e.micro(move |e| {
1385                        if e.is_token(token) {
1386                            end();
1387                            e.iterate();
1388                        }
1389                    });
1390                }
1391            });
1392        }
1393        {
1394            let end = end.clone();
1395            let t = self.later(self.options.ping_timeout_ms, move |e| {
1396                if e.is_token(token) {
1397                    e.ping(end);
1398                }
1399            });
1400            *timer.borrow_mut() = Some(t);
1401        }
1402        {
1403            let w = self.me.clone();
1404            let cancel = cancel.clone();
1405            let end = end.clone();
1406            *self.ping_now.borrow_mut() = Rc::new(move || {
1407                if let Some(e) = w.upgrade()
1408                    && e.is_token(token)
1409                {
1410                    cancel();
1411                    e.ping(end.clone());
1412                }
1413            });
1414        }
1415        {
1416            let cancel = cancel.clone();
1417            let end = end.clone();
1418            self.on_visibility(false, token, move |e| {
1419                cancel();
1420                e.disconnect(
1421                    ConnectionError::client(
1422                        ClientErrorKind::Hidden,
1423                        "Connection closed because tab was hidden",
1424                    ),
1425                    None,
1426                );
1427                end();
1428                e.iterate();
1429            });
1430        }
1431        self.on_state_change(token, move |e, state| {
1432            end();
1433            e.resume_after_state_change(state);
1434        });
1435    }
1436
1437    fn ping(&self, end: Rc<dyn Fn()>) {
1438        let token = self.next_loop_token();
1439        let t0 = self.scheduler.monotonic_now();
1440        // Before the send, as JS sets `#onPong` first: a transport may
1441        // deliver the pong from inside `send`.
1442        {
1443            let w = self.me.clone();
1444            let end = end.clone();
1445            *self.on_pong.borrow_mut() = Rc::new(move || {
1446                let end = end.clone();
1447                if let Some(e) = w.upgrade() {
1448                    e.micro(move |e| {
1449                        if !e.is_token(token) {
1450                            return;
1451                        }
1452                        let ms = e.scheduler.monotonic_now() - t0;
1453                        e.log
1454                            .debug(|| format!("ping succeeded in {} ms", number::format(ms)));
1455                        end();
1456                        e.iterate();
1457                    });
1458                }
1459            });
1460        }
1461        self.send(Upstream::Ping);
1462        {
1463            let end = end.clone();
1464            // The timer ends with the token; it needn't be cancelled.
1465            let _ = self.later(self.options.ping_timeout_ms, move |e| {
1466                if !e.is_token(token) {
1467                    return;
1468                }
1469                let ms = e.scheduler.monotonic_now() - t0;
1470                e.log
1471                    .info(|| format!("ping failed in {} ms - disconnecting", number::format(ms)));
1472                let error = ConnectionError::client(
1473                    ClientErrorKind::PingTimeout,
1474                    "Server ping request failed",
1475                );
1476                e.disconnect(error.clone(), None);
1477                end();
1478                e.loop_failed(error);
1479            });
1480        }
1481        self.on_state_change(token, move |e, state| {
1482            end();
1483            e.resume_after_state_change(state);
1484        });
1485    }
1486
1487    /// needs-auth and error: wait for `connect()`.
1488    fn wait_for_connect_request(&self, token: u64) {
1489        let w = self.me.clone();
1490        self.connection_manager.on_connect_request(move || {
1491            if let Some(e) = w.upgrade()
1492                && e.is_token(token)
1493            {
1494                e.connection_manager.resume_from_connect_request();
1495                e.iterate();
1496            }
1497        });
1498        self.on_state_change(token, |e, _| e.iterate());
1499    }
1500
1501    // MARK: The app's lifecycle
1502
1503    /// The app was shown, or hidden. Hidden, the client disconnects and
1504    /// waits to be shown again.
1505    pub(crate) fn set_visible(&self, visible: bool) {
1506        if let Some(t) = self.hidden_timer.borrow_mut().take() {
1507            t.cancel();
1508        }
1509        if visible {
1510            self.set_visibility_state(true);
1511        } else {
1512            let t = self.later(self.options.hidden_disconnect_delay_ms, |e| {
1513                e.hidden_timer.borrow_mut().take();
1514                e.set_visibility_state(false);
1515            });
1516            *self.hidden_timer.borrow_mut() = Some(t);
1517        }
1518    }
1519
1520    fn set_visibility_state(&self, visible: bool) {
1521        if visible == self.visible.get() {
1522            return;
1523        }
1524        self.visible.set(visible);
1525        let ready: Vec<VisibilityWaiter> = {
1526            let mut waiters = self.visibility_waiters.borrow_mut();
1527            let (ready, rest) = std::mem::take(&mut *waiters)
1528                .into_iter()
1529                .partition(|(v, _, _)| *v == visible);
1530            *waiters = rest;
1531            ready
1532        };
1533        for (_, _, w) in ready {
1534            w();
1535        }
1536    }
1537
1538    /// Calls `body` in a microtask once the app's visibility is `visible`,
1539    /// unless the run loop's wait `token` is over by then.
1540    fn on_visibility(&self, visible: bool, token: u64, body: impl FnOnce(&Engine) + 'static) {
1541        let w = self.me.clone();
1542        let waiter: Box<dyn FnOnce()> = Box::new(move || {
1543            if let Some(e) = w.upgrade() {
1544                e.micro(move |e| {
1545                    if !e.is_token(token) {
1546                        return;
1547                    }
1548                    // The wait is over: a state change no longer ends it.
1549                    e.next_loop_token();
1550                    body(e);
1551                });
1552            }
1553        });
1554        if self.visible.get() == visible {
1555            waiter();
1556        } else {
1557            let current = self.loop_token.get();
1558            let mut waiters = self.visibility_waiters.borrow_mut();
1559            waiters.retain(|(_, t, _)| *t == current);
1560            waiters.push((visible, token, waiter));
1561        }
1562    }
1563
1564    /// The network may be back: retry now instead of after the backoff,
1565    /// and check that an open connection still works.
1566    pub(crate) fn reconnect_now(&self) {
1567        let backoff = {
1568            let mut b = self.backoff.borrow_mut();
1569            match &*b {
1570                Some((token, _)) if *token == self.loop_token.get() => b.take(),
1571                _ => None,
1572            }
1573        };
1574        if let Some((_, timer)) = backoff {
1575            timer.cancel();
1576            self.iterate();
1577        } else if self.status_is(ConnectionStatus::Connected) {
1578            let ping_now = self.ping_now.borrow().clone();
1579            ping_now();
1580        }
1581    }
1582
1583    fn pending_settled(&self) -> bool {
1584        self.store.borrow().pending.is_empty()
1585            || !self.status_is(ConnectionStatus::Connected)
1586            || self.client_group_disabled.get()
1587            || self.rep_closed.get()
1588    }
1589
1590    /// Calls `done` once the server has confirmed every pending mutation,
1591    /// or right away when it can't: the client isn't connected, or the
1592    /// group is disabled.
1593    pub(crate) fn when_pending_settled(&self, done: impl FnOnce() + 'static) {
1594        if self.pending_settled() {
1595            done();
1596        } else {
1597            self.pending_waiters.borrow_mut().push(Box::new(done));
1598        }
1599    }
1600
1601    fn settle_pending_waiters(&self) {
1602        if self.pending_waiters.borrow().is_empty() || !self.pending_settled() {
1603            return;
1604        }
1605        let waiters = std::mem::take(&mut *self.pending_waiters.borrow_mut());
1606        for w in waiters {
1607            w();
1608        }
1609    }
1610
1611    // MARK: Connecting
1612
1613    fn add_connect_event(&self, attempt: &ConnectAttempt, event: &str) {
1614        attempt
1615            .events
1616            .borrow_mut()
1617            .push((self.scheduler.now(), event.to_owned()));
1618    }
1619
1620    fn arm_connect_timeout(&self, attempt: &Rc<ConnectAttempt>, ack: bool) {
1621        attempt.cancel_timeout();
1622        if attempt.aborted.get() {
1623            return;
1624        }
1625        let weak = Rc::downgrade(attempt);
1626        let t = self.later(CONNECT_TIMEOUT_MS, move |e| {
1627            let Some(attempt) = weak.upgrade() else {
1628                return;
1629            };
1630            let events = Value::Array(
1631                attempt
1632                    .events
1633                    .borrow()
1634                    .iter()
1635                    .map(|(t, ev)| {
1636                        Value::Array(vec![
1637                            Value::String(iso_date_string(*t)),
1638                            Value::String(ev.clone()),
1639                        ])
1640                    })
1641                    .collect(),
1642            );
1643            e.log.debug(|| "Connection attempt timed out".into());
1644            let seconds = number::format(CONNECT_TIMEOUT_MS / 1000.0);
1645            let message = format!(
1646                "{}Connect events: {}",
1647                if ack {
1648                    format!("Server did not acknowledge the connection within {seconds} seconds. ")
1649                } else {
1650                    format!("Connection setup timed out after {seconds} seconds. ")
1651                },
1652                json::stringify(&events)
1653            );
1654            e.disconnect(
1655                ConnectionError::client(ClientErrorKind::ConnectTimeout, message),
1656                None,
1657            );
1658        });
1659        *attempt.timeout.borrow_mut() = Some(t);
1660    }
1661
1662    fn begin_attempt(&self) {
1663        assert!(
1664            self.current_attempt.borrow().is_none(),
1665            "Connection attempt already active"
1666        );
1667        let attempt = Rc::new(ConnectAttempt::default());
1668        *self.current_attempt.borrow_mut() = Some(attempt.clone());
1669        self.add_connect_event(&attempt, "starting the connection");
1670        self.arm_connect_timeout(&attempt, false);
1671        self.connect_attempt(&attempt);
1672    }
1673
1674    fn token(&self) -> Option<String> {
1675        let auth = self.auth.borrow();
1676        (!auth.is_empty()).then(|| auth.clone())
1677    }
1678
1679    /// `#connectAttempt`: opens a socket and waits for `connected`.
1680    fn connect_attempt(&self, attempt: &Rc<ConnectAttempt>) {
1681        if self.rep_closed.get() {
1682            self.finish_attempt(attempt, None);
1683            return;
1684        }
1685        let (Some(socket_origin), Some(socket_factory)) =
1686            (self.socket_origin.clone(), self.socket_factory.clone())
1687        else {
1688            self.finish_attempt(
1689                attempt,
1690                Some(ConnectionError::client(
1691                    ClientErrorKind::Internal,
1692                    "No server provided",
1693                )),
1694            );
1695            return;
1696        };
1697        assert!(
1698            self.status_is(ConnectionStatus::Disconnected)
1699                || self.status_is(ConnectionStatus::Connecting),
1700            "connectAttempt() called from invalid state"
1701        );
1702        let wsid = random_id();
1703        self.log.info(|| "Connecting...".into());
1704        self.connection_manager.connecting(None);
1705        attempt.start.set(self.scheduler.monotonic_now());
1706        self.add_connect_event(attempt, "reading the cookie");
1707        let connect_cookie = self.store.borrow().cookie().clone();
1708        self.add_connect_event(attempt, "reading the client group ID");
1709        self.add_connect_event(attempt, "initializing active clients");
1710        if attempt.aborted.get() {
1711            let reason = attempt.abort_reason.borrow().clone();
1712            self.finish_attempt(attempt, reason);
1713            return;
1714        }
1715
1716        // `createSocket`.
1717        self.add_connect_event(attempt, "reading the profile ID");
1718        let url = ConnectionUrl {
1719            socket_origin,
1720            client_id: self.client_id.clone(),
1721            client_group_id: self.client_group_id.clone(),
1722            user_id: self.user_id.clone(),
1723            base_cookie: match &connect_cookie {
1724                Value::String(s) => Some(s.clone()),
1725                Value::Number(n) => Some(number::format(*n)),
1726                _ => None,
1727            },
1728            timestamp: self.scheduler.monotonic_now(),
1729            last_mutation_id: self.last_mutation_id_received.get(),
1730            wsid,
1731            profile_id: self.profile_id.clone(),
1732            // JS asks for the server's timings when it logs at debug level.
1733            debug_perf: self.options.log_level == LogLevel::Debug,
1734            additional_params: self
1735                .additional_connect_params
1736                .borrow()
1737                .clone()
1738                .unwrap_or_default(),
1739        }
1740        .to_string();
1741        self.add_connect_event(attempt, "reading deleted clients");
1742        let mut deleted = self.deleted_clients_body();
1743        self.add_connect_event(attempt, "reading desired queries");
1744        let mut queries_patch = Some(
1745            self.query_manager
1746                .queries_patch(&self.store.borrow().main, None),
1747        );
1748        let mut active_clients = vec![self.client_id.clone()];
1749        active_clients.extend(
1750            self.database
1751                .active_clients(&self.client_group_id)
1752                .into_iter()
1753                .filter(|c| *c != self.client_id),
1754        );
1755        let init_connection = InitConnectionBody {
1756            desired_queries_patch: queries_patch
1757                .as_ref()
1758                .map(|p| p.values().cloned().collect())
1759                .unwrap_or_default(),
1760            client_schema: connect_cookie.is_null().then(|| self.client_schema.clone()),
1761            deleted: deleted.clone(),
1762            user_push_url: self.options.mutate_url.clone(),
1763            user_push_headers: self.options.mutate_headers.clone(),
1764            user_query_url: self.options.query_url.clone(),
1765            user_query_headers: self.options.query_headers.clone(),
1766            active_clients: Some(active_clients),
1767            traceparent: None,
1768        };
1769        let auth_token = self.token();
1770        let mut sec_protocol = sec_protocols::encode(Some(&init_connection), auth_token.as_deref());
1771        if sec_protocol.len() > self.options.max_header_length {
1772            sec_protocol = sec_protocols::encode(None, auth_token.as_deref());
1773            queries_patch = None;
1774            if sec_protocol.len() > self.options.max_header_length {
1775                let (len, max) = (sec_protocol.len(), self.options.max_header_length);
1776                self.log.warn(|| {
1777                    format!(
1778                        "Encoded auth token length ({len}) exceeds ZeroOptions.maxHeaderLength \
1779                         ({max}). This may cause connection failures."
1780                    )
1781                });
1782            }
1783        } else {
1784            deleted = None;
1785        }
1786        self.log.info(|| format!("Connecting to {url}"));
1787        self.add_connect_event(attempt, "creating the WebSocket");
1788        let handle = SocketHandle::new(self.me.clone());
1789        let connection = socket_factory.open(&url, Some(&sec_protocol), handle.clone());
1790        *handle.connection.borrow_mut() = Some(connection);
1791
1792        if attempt.aborted.get() {
1793            handle.close(None);
1794            let reason = attempt.abort_reason.borrow().clone();
1795            self.finish_attempt(attempt, reason);
1796            return;
1797        }
1798        if self.rep_closed.get() {
1799            handle.close(None);
1800            self.finish_attempt(attempt, None);
1801            return;
1802        }
1803        *self.connect_cookie.borrow_mut() = connect_cookie;
1804        *self.init_connection_queries.borrow_mut() = queries_patch;
1805        *self.deleted_clients.borrow_mut() = deleted;
1806        *self.socket.borrow_mut() = Some(handle);
1807        self.add_connect_event(attempt, "waiting for the server acknowledgement");
1808        self.log
1809            .debug(|| "Waiting for connection to be acknowledged".into());
1810        self.arm_connect_timeout(attempt, true);
1811        let (w, weak_attempt) = (self.me.clone(), Rc::downgrade(attempt));
1812        *attempt.on_connected.borrow_mut() = Some(Box::new(move || {
1813            let (Some(e), Some(attempt)) = (w.upgrade(), weak_attempt.upgrade()) else {
1814                return;
1815            };
1816            if attempt.aborted.get() {
1817                let reason = attempt.abort_reason.borrow().clone();
1818                e.finish_attempt(&attempt, reason);
1819                return;
1820            }
1821            e.mutation_tracker
1822                .on_connected(e.last_mutation_id_received.get());
1823            e.request_push(true);
1824            e.finish_attempt(&attempt, None);
1825        }));
1826    }
1827
1828    /// The run loop's handling of a finished attempt; `error` `None` for
1829    /// success.
1830    fn finish_attempt(&self, attempt: &Rc<ConnectAttempt>, error: Option<ConnectionError>) {
1831        if attempt.finished.replace(true) {
1832            return;
1833        }
1834        attempt.cancel_timeout();
1835        {
1836            let mut current = self.current_attempt.borrow_mut();
1837            if current.as_ref().is_some_and(|c| Rc::ptr_eq(c, attempt)) {
1838                *current = None;
1839            }
1840        }
1841        let ready = self.connection_ready.borrow().clone();
1842        if let Some(error) = error {
1843            if !attempt.aborted.get() {
1844                self.disconnect(error.clone(), None);
1845            }
1846            ready.0.borrow_mut().error.get_or_insert(error.clone());
1847            ready.settle(false, &self.event_loop);
1848            let parked: Vec<InspectRequest> = {
1849                let mut requests = self.inspect_requests.borrow_mut();
1850                let (parked, rest) = std::mem::take(&mut *requests)
1851                    .into_iter()
1852                    .partition(|r| r.parked);
1853                *requests = rest;
1854                parked
1855            };
1856            for r in parked {
1857                (r.done)(Err(error.clone().into()));
1858            }
1859            if self.connection_ready.borrow().is(&ready) {
1860                *self.connection_ready.borrow_mut() = ReadyPromise::default();
1861            }
1862            self.loop_failed(error);
1863            return;
1864        }
1865        ready.settle(true, &self.event_loop);
1866        let parked: Vec<String> = self
1867            .inspect_requests
1868            .borrow_mut()
1869            .iter_mut()
1870            .filter(|r| r.parked)
1871            .map(|r| {
1872                r.parked = false;
1873                r.id.clone()
1874            })
1875            .collect();
1876        for id in parked {
1877            self.inspect_when_connected(id);
1878        }
1879        *self.additional_connect_params.borrow_mut() = None;
1880        match Self::connection_error(&self.connection_manager.state()) {
1881            Some(error) => self.loop_failed(error),
1882            None => self.iterate(),
1883        }
1884    }
1885
1886    /// `#disconnect`: closes the socket and moves the connection status.
1887    pub(crate) fn disconnect(&self, reason: ConnectionError, close_code: Option<u16>) {
1888        let attempt = self.current_attempt.borrow().clone();
1889        if let Some(attempt) = attempt
1890            && !attempt.aborted.get()
1891        {
1892            *attempt.abort_reason.borrow_mut() = Some(reason.clone());
1893            attempt.aborted.set(true);
1894            attempt.cancel_timeout();
1895            let reason = reason.clone();
1896            self.micro(move |e| e.finish_attempt(&attempt, Some(reason)));
1897        }
1898        let status = self.connection_manager.status();
1899        self.log.info(|| {
1900            format!(
1901                "disconnecting: reason={} connectionState={}",
1902                reason.kind_name(),
1903                status.as_str()
1904            )
1905        });
1906        if status == ConnectionStatus::Closed {
1907            self.log.debug(|| "disconnect() called while closed".into());
1908            return;
1909        }
1910        if let Some(at) = self.connected_at.get()
1911            && self.scheduler.monotonic_now() - at >= STABLE_CONNECTION_MS
1912        {
1913            self.transport_failures.set(0);
1914        }
1915        self.connected_at.set(None);
1916        let socket = self.socket.borrow_mut().take();
1917        if let Some(s) = socket {
1918            s.close(close_code);
1919        }
1920        self.last_mutation_ids_sent.borrow_mut().clear();
1921        self.poke_handler.borrow_mut().clear();
1922        self.query_manager.clear_got_queries_authoritative();
1923        let (transition, r) = ConnectionTransition::of(&reason);
1924        match transition {
1925            ConnectionTransition::NeedsAuth => self.connection_manager.needs_auth(r),
1926            ConnectionTransition::Error => self.connection_manager.error(r),
1927            ConnectionTransition::Disconnected => self.connection_manager.disconnected(r),
1928            ConnectionTransition::Closed => self.connection_manager.closed(),
1929            ConnectionTransition::None => {
1930                if !self.connection_manager.is_in_terminal_state() {
1931                    self.connection_manager.connecting(Some(r));
1932                }
1933            }
1934        }
1935        // The answers to the inspector's requests would have come on the
1936        // socket that closed.
1937        let lost: Vec<InspectRequest> = {
1938            let mut requests = self.inspect_requests.borrow_mut();
1939            let (lost, waiting) = std::mem::take(&mut *requests)
1940                .into_iter()
1941                .partition(|r| r.sent);
1942            *requests = waiting;
1943            lost
1944        };
1945        for r in lost {
1946            (r.done)(Err(Error::Message(format!(
1947                "The inspector's request got no answer: the connection to zero-cache closed ({})",
1948                reason.message()
1949            ))));
1950        }
1951    }
1952
1953    // MARK: Messages
1954
1955    fn is_socket(&self, handle: &Rc<SocketHandle>) -> bool {
1956        self.socket
1957            .borrow()
1958            .as_ref()
1959            .is_some_and(|s| Rc::ptr_eq(s, handle))
1960    }
1961
1962    fn socket_did_receive(&self, handle: &Rc<SocketHandle>, message: SocketMessage) {
1963        if !self.is_socket(handle) {
1964            return;
1965        }
1966        let event_loop = self.event_loop.clone();
1967        event_loop.dispatch(|| self.on_message(message));
1968    }
1969
1970    fn socket_did_open(&self, handle: &Rc<SocketHandle>) {
1971        if !self.is_socket(handle) {
1972            return;
1973        }
1974        let Some(attempt) = self.current_attempt.borrow().clone() else {
1975            return;
1976        };
1977        let ms = self.scheduler.monotonic_now() - attempt.start.get();
1978        self.log
1979            .info(|| format!("Got socket open event: timeToOpenMs={}", number::format(ms)));
1980    }
1981
1982    fn socket_did_close(
1983        &self,
1984        handle: &Rc<SocketHandle>,
1985        code: u16,
1986        reason: &str,
1987        was_clean: bool,
1988    ) {
1989        if !self.is_socket(handle) {
1990            return;
1991        }
1992        let event_loop = self.event_loop.clone();
1993        event_loop.dispatch(|| {
1994            // 1005 and 1006 are how platforms report a close without a
1995            // frame.
1996            if code <= 1001 || code == 1005 || code == 1006 {
1997                self.log.info(|| {
1998                    format!("Got socket close event: code={code} reason={reason} wasClean={was_clean}")
1999                });
2000            } else {
2001                self.log.error(|| {
2002                    format!(
2003                        "Got unexpected socket close event: code={code} reason={reason} wasClean={was_clean}"
2004                    )
2005                });
2006            }
2007            if code == 1009 {
2008                let mut message = "A WebSocket message exceeded the server message size limit. \
2009                    This is usually caused by a mutation's args containing a large value such as \
2010                    a base64-encoded image. Consider uploading large files to object storage and \
2011                    storing only the URL in Zero."
2012                    .to_owned();
2013                let recent = self.recent_sent_messages.borrow();
2014                if !recent.is_empty() {
2015                    message.push_str("\nRecent sent messages:\n");
2016                    message.push_str(&pretty_recent_messages(&recent));
2017                }
2018                drop(recent);
2019                self.disconnect(
2020                    ConnectionError::client(ClientErrorKind::InvalidMessage, message),
2021                    None,
2022                );
2023                self.disable_client_group();
2024                self.micro(|e| e.client_state_not_found());
2025            } else if was_clean {
2026                self.disconnect(
2027                    ConnectionError::client(
2028                        ClientErrorKind::CleanClose,
2029                        "WebSocket connection closed cleanly",
2030                    ),
2031                    None,
2032                );
2033            } else {
2034                self.disconnect(
2035                    ConnectionError::client(
2036                        ClientErrorKind::AbruptClose,
2037                        "WebSocket connection closed abruptly",
2038                    ),
2039                    None,
2040                );
2041            }
2042        });
2043    }
2044
2045    fn abort_ping_timeout(&self) {
2046        let f = self.abort_ping_timeout.borrow().clone();
2047        f();
2048    }
2049
2050    /// Disconnects with `error` for a message that doesn't make sense.
2051    fn fail_on_message(&self, error: ConnectionError) {
2052        self.disconnect(error, None);
2053    }
2054
2055    fn on_message(&self, message: SocketMessage) {
2056        if self.rep_closed.get() {
2057            return;
2058        }
2059        let text = match message {
2060            SocketMessage::Binary(bytes) => {
2061                if bytes.first() != Some(&poke_chunk::MESSAGE_TYPE) {
2062                    let t = bytes
2063                        .first()
2064                        .map_or_else(|| "undefined".to_owned(), |b| b.to_string());
2065                    self.fail_on_message(ConnectionError::client(
2066                        ClientErrorKind::InvalidMessage,
2067                        format!("Unknown binary message type: {t}"),
2068                    ));
2069                    return;
2070                }
2071                self.abort_ping_timeout();
2072                let result = self
2073                    .poke_handler
2074                    .borrow_mut()
2075                    .handle_poke_chunk(&bytes[1..]);
2076                if let Err(m) = result {
2077                    self.poke_message_error(m);
2078                }
2079                return;
2080            }
2081            SocketMessage::Text(t) => t,
2082        };
2083        let downstream = match Downstream::from_text(&text) {
2084            Ok(d) => d,
2085            Err(e) => {
2086                self.fail_on_message(ConnectionError::client(
2087                    ClientErrorKind::InvalidMessage,
2088                    format!(
2089                        "Invalid message received from server: {}{text}",
2090                        valita_message(&e, &text)
2091                    ),
2092                ));
2093                return;
2094            }
2095        };
2096        match downstream {
2097            Downstream::Connected(body) => self.micro(move |e| e.handle_connected(body)),
2098            Downstream::Error(body) => self.handle_error_message(body),
2099            Downstream::Pong => {
2100                let f = self.on_pong.borrow().clone();
2101                f();
2102            }
2103            Downstream::PokeStart(body) => {
2104                self.abort_ping_timeout();
2105                let result = self.poke_handler.borrow_mut().handle_poke_start(body);
2106                if let Err(m) = result {
2107                    self.poke_message_error(m);
2108                }
2109            }
2110            Downstream::PokePart(body) => {
2111                self.abort_ping_timeout();
2112                let result = self.poke_handler.borrow_mut().handle_poke_part(body);
2113                match result {
2114                    Ok(Some(lmid)) => self.last_mutation_id_received.set(lmid),
2115                    Ok(None) => {}
2116                    Err(m) => self.poke_message_error(m),
2117                }
2118            }
2119            Downstream::PokeEnd(body) => {
2120                self.abort_ping_timeout();
2121                let result = self.poke_handler.borrow_mut().handle_poke_end(body);
2122                match result {
2123                    Ok(Some(r)) => {
2124                        if let Some(lmid) = r.last_mutation_id_change_for_self {
2125                            self.last_mutation_id_received.set(lmid);
2126                        }
2127                        if !self.poke_handler.borrow().playback_running {
2128                            self.start_playback();
2129                        }
2130                    }
2131                    Ok(None) => {}
2132                    Err(m) => self.poke_message_error(m),
2133                }
2134            }
2135            Downstream::Pull(_) => self.abort_ping_timeout(),
2136            Downstream::DeleteClients(body) => self.database.confirm_deleted_clients(
2137                body.client_ids.as_deref().unwrap_or_default(),
2138                body.client_group_ids.as_deref().unwrap_or_default(),
2139            ),
2140            Downstream::PushResponse(body) => {
2141                if matches!(body, PushResponseBody::Error(_)) {
2142                    self.log.error(|| {
2143                        format!(
2144                            "Received an error response when pushing mutations: {}",
2145                            json::stringify(&body.to_json())
2146                        )
2147                    });
2148                }
2149                if let Err(e) = self.mutation_tracker.process_push_response(&body) {
2150                    self.disconnect(
2151                        ConnectionError::client(ClientErrorKind::Internal, e.message()),
2152                        None,
2153                    );
2154                }
2155            }
2156            Downstream::TransformError(errors) => {
2157                self.query_manager.handle_transform_errors(&errors)
2158            }
2159            Downstream::Inspect(body) => self.inspect_answered(body.0),
2160        }
2161    }
2162
2163    fn handle_connected(&self, _body: ConnectedBody) {
2164        let Some(attempt) = self.current_attempt.borrow().clone() else {
2165            return;
2166        };
2167        let ms = self.scheduler.monotonic_now() - attempt.start.get();
2168        self.log
2169            .info(|| format!("Connected: timeToConnectMs={}", number::format(ms)));
2170        self.add_connect_event(&attempt, "processing the server acknowledgement");
2171        self.connected_at.set(Some(self.scheduler.monotonic_now()));
2172        self.last_mutation_ids_sent.borrow_mut().clear();
2173        let init_queries = self.init_connection_queries.borrow_mut().take();
2174        let patch = self
2175            .query_manager
2176            .queries_patch(&self.store.borrow().main, init_queries.as_ref());
2177        let patch_ops: Vec<UpQueriesPatchOp> = patch.values().cloned().collect();
2178        match init_queries {
2179            Some(_) if !patch.is_empty() => {
2180                self.send(Upstream::ChangeDesiredQueries(ChangeDesiredQueriesBody {
2181                    desired_queries_patch: patch_ops,
2182                    traceparent: self.traceparent(),
2183                }));
2184            }
2185            Some(_) => {}
2186            None => {
2187                let deleted = self.deleted_clients.borrow_mut().take();
2188                let client_schema = self
2189                    .connect_cookie
2190                    .borrow()
2191                    .is_null()
2192                    .then(|| self.client_schema.clone());
2193                self.send(Upstream::InitConnection(InitConnectionBody {
2194                    desired_queries_patch: patch_ops,
2195                    client_schema,
2196                    deleted,
2197                    user_push_url: self.options.mutate_url.clone(),
2198                    user_push_headers: self.options.mutate_headers.clone(),
2199                    user_query_url: self.options.query_url.clone(),
2200                    user_query_headers: self.options.query_headers.clone(),
2201                    active_clients: None,
2202                    traceparent: self.traceparent(),
2203                }));
2204            }
2205        }
2206        let deleted = self.deleted_clients.borrow_mut().take();
2207        if let Some(deleted) = deleted {
2208            self.send(Upstream::DeleteClients(deleted));
2209        }
2210        self.connection_manager.connected();
2211        let on_connected = attempt.on_connected.borrow_mut().take();
2212        if let Some(f) = on_connected {
2213            self.event_loop.microtask(f);
2214        }
2215    }
2216
2217    fn handle_error_message(&self, body: ErrorBody) {
2218        let kind = body.kind();
2219        if kind == ErrorKind::MutationRateLimited {
2220            self.last_mutation_ids_sent.borrow_mut().clear();
2221            self.log.error(|| {
2222                format!(
2223                    "{} Mutation rate limited: {}",
2224                    kind.as_str(),
2225                    body.message()
2226                )
2227            });
2228            return;
2229        }
2230        // Being sent elsewhere or later, a lost client group (which
2231        // recovers), and auth failures are routine; every other kind ends
2232        // the connection.
2233        let message = body.message().to_owned();
2234        let error = ConnectionError::Server(Box::new(body));
2235        let routine =
2236            error.backoff().is_some() || kind == ErrorKind::ClientNotFound || error.is_auth_error();
2237        self.log.log(
2238            if routine {
2239                LogLevel::Warn
2240            } else {
2241                LogLevel::Error
2242            },
2243            || format!("{}:\n\n{message}", kind.as_str()),
2244        );
2245        self.disconnect(error, None);
2246        match kind {
2247            ErrorKind::VersionNotSupported => self.update_needed(UpdateNeededReason {
2248                type_: kind.as_str().to_owned(),
2249                message: Some(message),
2250            }),
2251            ErrorKind::SchemaVersionNotSupported => {
2252                self.disable_client_group();
2253                self.micro(move |e| {
2254                    e.update_needed(UpdateNeededReason {
2255                        type_: "SchemaVersionNotSupported".into(),
2256                        message: Some(message),
2257                    })
2258                });
2259            }
2260            ErrorKind::ClientNotFound => {
2261                self.disable_client_group();
2262                self.micro(|e| e.client_state_not_found());
2263            }
2264            ErrorKind::InvalidConnectionRequestLastMutationID
2265            | ErrorKind::InvalidConnectionRequestBaseCookie => {
2266                // The server is behind the replica: drop it.
2267                self.database.drop_database();
2268                self.micro(|e| e.client_state_not_found());
2269            }
2270            _ => {}
2271        }
2272    }
2273
2274    /// A poke message that doesn't fit the poke being received.
2275    fn poke_message_error(&self, message: String) {
2276        self.poke_handler.borrow_mut().clear();
2277        self.on_poke_error(&message);
2278    }
2279
2280    fn on_poke_error(&self, message: &str) {
2281        if self.status_is(ConnectionStatus::Disconnected) {
2282            return;
2283        }
2284        if message.contains("unexpected base cookie") {
2285            self.log.debug(|| format!("clearing due to {message}"));
2286            self.disconnect(
2287                ConnectionError::client(
2288                    ClientErrorKind::UnexpectedBaseCookie,
2289                    "Server returned unexpected base cookie during sync",
2290                ),
2291                None,
2292            );
2293        } else {
2294            self.log
2295                .error(|| format!("clearing due to unexpected poke error: {message}"));
2296            self.disconnect(
2297                ConnectionError::client(
2298                    ClientErrorKind::Internal,
2299                    format!("Poke processing error: {message}"),
2300                ),
2301                None,
2302            );
2303        }
2304    }
2305
2306    // MARK: Poke playback
2307
2308    fn start_playback(&self) {
2309        self.poke_handler.borrow_mut().playback_running = true;
2310        let _ = self.later(0.0, |e| e.playback_frame());
2311    }
2312
2313    /// Applies the pokes that arrived since the last frame, at most once
2314    /// per scheduler turn, merged (`#rafCallback`).
2315    fn playback_frame(&self) {
2316        if !self.poke_handler.borrow().has_buffered() {
2317            self.poke_handler.borrow_mut().playback_running = false;
2318            return;
2319        }
2320        let _ = self.later(0.0, |e| e.playback_frame());
2321        let merged = self.poke_handler.borrow_mut().take_merged();
2322        let merged = match merged {
2323            Ok(Some(m)) => m,
2324            Ok(None) => return,
2325            Err(e) => {
2326                self.poke_handler.borrow_mut().clear();
2327                self.on_poke_error(&e.0);
2328                return;
2329            }
2330        };
2331        let applied = self.with_write_lock(|| self.apply_poke(&merged));
2332        let result = applied.and_then(|()| {
2333            let lmid = merged
2334                .last_mutation_id_changes
2335                .get(&self.client_id)
2336                .copied();
2337            let advanced = match lmid {
2338                Some(lmid) => self
2339                    .mutation_tracker
2340                    .lmid_advanced(lmid)
2341                    .map_err(|m| format!("Error: {m}")),
2342                None => Ok(()),
2343            };
2344            self.settle_pending_waiters();
2345            advanced
2346        });
2347        if let Err(message) = result {
2348            self.poke_handler.borrow_mut().clear();
2349            self.on_poke_error(&message);
2350        }
2351    }
2352
2353    // MARK: Auth
2354
2355    pub(crate) fn set_auth(&self, auth: Option<String>) {
2356        // A token without a user was rejected at init.
2357        *self.auth.borrow_mut() = auth.clone().unwrap_or_default();
2358        if let Some(auth) = auth
2359            && !auth.is_empty()
2360        {
2361            self.send_if_connected(Upstream::UpdateAuth(UpdateAuthBody { auth }));
2362        }
2363    }
2364
2365    /// `connection.connect()`: resumes from needs-auth or error. `auth`,
2366    /// when set, replaces the token first (`Some(None)` clears it).
2367    pub(crate) fn connect(&self, auth: Option<Option<String>>) {
2368        if let Some(auth) = auth {
2369            // JS's `#setAuth` throws this, so `connect` rejects before it
2370            // does anything.
2371            if let Err(e) = Self::check_auth_valid(self.user_id.as_deref(), auth.as_deref()) {
2372                self.log.error(|| format!("connect(auth:) ignored: {e}"));
2373                return;
2374            }
2375            self.set_auth(auth);
2376        }
2377        if let State::Disconnected(r) = self.connection_manager.state()
2378            && r.client_kind() == Some(ClientErrorKind::NoSocketOrigin)
2379        {
2380            self.log.error(|| {
2381                "connect() called but the connection is disconnected due to a missing cacheURL. \
2382                 No reconnect will be attempted."
2383                    .into()
2384            });
2385            return;
2386        }
2387        if !self.connection_manager.is_in_terminal_state() {
2388            return;
2389        }
2390        self.connection_manager.request_connect();
2391    }
2392
2393    fn client_state_not_found(&self) {
2394        let f = self.on_client_state_not_found.borrow().clone();
2395        match f {
2396            Some(f) => f(),
2397            None => {
2398                if let Some(f) = &self.options.on_client_state_not_found {
2399                    f();
2400                }
2401            }
2402        }
2403    }
2404
2405    fn update_needed(&self, reason: UpdateNeededReason) {
2406        let f = self.on_update_needed.borrow().clone();
2407        if let Some(f) = f {
2408            f(&reason);
2409        }
2410        if let Some(f) = &self.options.on_update_needed {
2411            f(&reason);
2412        }
2413    }
2414
2415    // MARK: Close
2416
2417    pub(crate) fn closed(&self) -> bool {
2418        self.rep_closed.get()
2419    }
2420
2421    pub(crate) fn close(&self) {
2422        if self.rep_closed.get() {
2423            self.log
2424                .debug(|| "close() called on already closed instance".into());
2425            return;
2426        }
2427        self.log.debug(|| "Closing Zero instance".into());
2428        if !self.status_is(ConnectionStatus::Disconnected) {
2429            self.disconnect(
2430                ConnectionError::client(
2431                    ClientErrorKind::ClientClosed,
2432                    "Zero instance closed by user",
2433                ),
2434                Some(1000),
2435            );
2436        }
2437        self.rep_closed.set(true);
2438        if let Some(t) = self.push_timer.borrow_mut().take() {
2439            t.cancel();
2440        }
2441        // JS's close aborts a persist that hasn't run.
2442        if let Some(t) = self.persist_timer.borrow_mut().take() {
2443            t.cancel();
2444        }
2445        for t in std::mem::take(&mut *self.lifecycle_timers.borrow_mut()) {
2446            t.cancel();
2447        }
2448        self.database.client_stopped(&self.client_id);
2449        // Releasing writes what the store had queued; a failure to is
2450        // still this client's to report.
2451        self.database.release();
2452        self.database.remove_failure_listener(&self.client_id);
2453        self.connection_manager.closed();
2454        let requests = std::mem::take(&mut *self.inspect_requests.borrow_mut());
2455        for r in requests {
2456            (r.done)(Err(ConnectionError::client(
2457                ClientErrorKind::ClientClosed,
2458                "Zero instance closed by user",
2459            )
2460            .into()));
2461        }
2462    }
2463
2464    // MARK: The inspector
2465
2466    /// Sends an inspector request (`rpcNoAuthTry`) once connected, with a
2467    /// new ID; `done` hears the answer with that ID, or why there is none.
2468    pub(crate) fn inspect(&self, body: Object, done: InspectAnswer) {
2469        if self.rep_closed.get() {
2470            return done(Err(ConnectionError::client(
2471                ClientErrorKind::ClientClosed,
2472                "Zero instance closed by user",
2473            )
2474            .into()));
2475        }
2476        let id = random_id();
2477        self.inspect_requests.borrow_mut().push(InspectRequest {
2478            id: id.clone(),
2479            body: Some(body),
2480            sent: false,
2481            parked: false,
2482            done,
2483        });
2484        self.inspect_when_connected(id);
2485    }
2486
2487    /// Sends request `id` once the connection is ready (`getSocket`), or
2488    /// fails it when connecting fails.
2489    fn inspect_when_connected(&self, id: String) {
2490        let ready = self.connection_ready.borrow().clone();
2491        let w = self.me.clone();
2492        let r = ready.clone();
2493        ready.on_settle(&self.event_loop, move |connected| {
2494            if let Some(e) = w.upgrade() {
2495                let error = r.0.borrow().error.clone();
2496                e.inspect_ready(id, connected, error);
2497            }
2498        });
2499    }
2500
2501    fn inspect_ready(&self, id: String, connected: bool, error: Option<ConnectionError>) {
2502        let mut requests = self.inspect_requests.borrow_mut();
2503        let Some(i) = requests.iter().position(|r| r.id == id && !r.sent) else {
2504            return;
2505        };
2506        if !connected {
2507            let r = requests.remove(i);
2508            drop(requests);
2509            let error = error.unwrap_or_else(|| {
2510                ConnectionError::client(ClientErrorKind::Internal, "Not connected to zero-cache")
2511            });
2512            return (r.done)(Err(error.into()));
2513        }
2514        let socket = self.socket.borrow().clone();
2515        match socket {
2516            Some(socket) if self.status_is(ConnectionStatus::Connected) => {
2517                let r = &mut requests[i];
2518                r.sent = true;
2519                let mut body = r.body.take().unwrap_or_default();
2520                body.insert("id", Value::from(id.as_str()));
2521                let text = Upstream::Inspect(crate::protocol::InspectUpBody(Value::Object(body)))
2522                    .to_text();
2523                drop(requests);
2524                socket.send(&text);
2525            }
2526            // The connection dropped since: wait for the next attempt.
2527            _ => requests[i].parked = true,
2528        }
2529    }
2530
2531    /// zero-cache answered an inspector request.
2532    fn inspect_answered(&self, body: Value) {
2533        let Value::Object(body) = body else { return };
2534        let Some(id) = body.get("id").and_then(Value::as_str) else {
2535            return;
2536        };
2537        let request = {
2538            let mut requests = self.inspect_requests.borrow_mut();
2539            let Some(i) = requests.iter().position(|r| r.sent && r.id == id) else {
2540                return;
2541            };
2542            requests.remove(i)
2543        };
2544        (request.done)(Ok(body));
2545    }
2546
2547    // MARK: Storage and the client lifecycle
2548
2549    /// Starts the heartbeat and the garbage collection of clients and
2550    /// client groups.
2551    fn start_lifecycle(&self) {
2552        let (h, c, g, d) = (
2553            self.me.clone(),
2554            self.me.clone(),
2555            self.me.clone(),
2556            self.me.clone(),
2557        );
2558        let timers = vec![
2559            self.scheduler.schedule_repeating(
2560                HEARTBEAT_INTERVAL_MS,
2561                Box::new(move || {
2562                    if let Some(e) = h.upgrade()
2563                        && !e.database.heartbeat(&e.client_id, e.scheduler.now())
2564                    {
2565                        e.client_state_not_found();
2566                    }
2567                }),
2568            ),
2569            self.scheduler.schedule_repeating(
2570                GC_INTERVAL_MS,
2571                Box::new(move || {
2572                    let Some(e) = c.upgrade() else { return };
2573                    if let Some(deleted) = e.database.collect_clients(
2574                        &e.client_id,
2575                        e.scheduler.now(),
2576                        CLIENT_MAX_INACTIVE_MS,
2577                    ) {
2578                        // `DeleteClientsManager.onClientsDeleted`.
2579                        let ids = deleted
2580                            .into_iter()
2581                            .filter(|p| p.client_group_id == e.client_group_id)
2582                            .map(|p| p.client_id)
2583                            .collect();
2584                        e.send_if_connected(Upstream::DeleteClients(DeleteClientsBody {
2585                            client_ids: Some(ids),
2586                            client_group_ids: None,
2587                        }));
2588                    }
2589                }),
2590            ),
2591            self.scheduler.schedule_repeating(
2592                GC_INTERVAL_MS,
2593                Box::new(move || {
2594                    if let Some(e) = g.upgrade() {
2595                        e.database.collect_client_groups();
2596                    }
2597                }),
2598            ),
2599            self.scheduler.schedule(
2600                INITIAL_COLLECT_DATABASES_DELAY_MS,
2601                Box::new(move || {
2602                    let Some(e) = d.upgrade() else { return };
2603                    e.collect_databases();
2604                    let w = e.me.clone();
2605                    let t = e.scheduler.schedule_repeating(
2606                        COLLECT_DATABASES_INTERVAL_MS,
2607                        Box::new(move || {
2608                            if let Some(e) = w.upgrade() {
2609                                e.collect_databases();
2610                            }
2611                        }),
2612                    );
2613                    e.lifecycle_timers.borrow_mut().push(t);
2614                }),
2615            ),
2616        ];
2617        *self.lifecycle_timers.borrow_mut() = timers;
2618    }
2619
2620    /// Deletes the other databases of the store that no client has used
2621    /// for twice the time a client may be inactive (`collectIDBDatabases`),
2622    /// such as those of an older schema.
2623    fn collect_databases(&self) {
2624        self.database
2625            .collect_databases(self.scheduler.now(), 2.0 * CLIENT_MAX_INACTIVE_MS);
2626    }
2627
2628    /// The store failed: the client keeps running on what it has in
2629    /// memory, and tells the app once.
2630    fn storage_failed(&self, f: &StorageFailure, opening: bool) {
2631        if self.storage_failure_reported.replace(true) {
2632            return;
2633        }
2634        if !opening {
2635            // A full or failing disk is the device's condition, not a
2636            // defect.
2637            self.log.warn(|| {
2638                format!(
2639                    "Local store storage failed ({}); persistence is stopped for this instance: {}",
2640                    f.kind.as_str(),
2641                    f.message
2642                )
2643            });
2644        }
2645        if let Some(cb) = &self.options.on_storage_failure {
2646            cb(f);
2647        }
2648    }
2649
2650    /// The deleted clients of this client group, for `initConnection`
2651    /// (`convertDeletedClientsToBody`).
2652    fn deleted_clients_body(&self) -> Option<DeleteClientsBody> {
2653        let ids: Vec<String> = self
2654            .database
2655            .deleted_clients()
2656            .into_iter()
2657            .filter(|p| p.client_group_id == self.client_group_id)
2658            .map(|p| p.client_id)
2659            .collect();
2660        (!ids.is_empty()).then_some(DeleteClientsBody {
2661            client_ids: Some(ids),
2662            client_group_ids: None,
2663        })
2664    }
2665
2666    /// The server lost the client group: stop pushing, and keep later
2667    /// clients out of it.
2668    fn disable_client_group(&self) {
2669        self.client_group_disabled.set(true);
2670        self.database.disable_client_group(&self.client_group_id);
2671    }
2672
2673    // MARK: Mutations
2674
2675    /// `MutatorProxy#onConnectionStateChange`.
2676    fn on_connection_state_change(&self, state: &State) {
2677        match state {
2678            State::Disconnected(r) if r.client_kind() == Some(ClientErrorKind::NoSocketOrigin) => {
2679                *self.mutation_rejection.borrow_mut() = None;
2680            }
2681            State::Disconnected(r) | State::Error(r) | State::Closed(r) => {
2682                *self.mutation_rejection.borrow_mut() = Some(r.clone());
2683                self.mutation_tracker.reject_all_outstanding_mutations(r);
2684            }
2685            _ => *self.mutation_rejection.borrow_mut() = None,
2686        }
2687        self.settle_pending_waiters();
2688    }
2689
2690    /// What a mutation's caller hears for an error (`wrapErrorFor`).
2691    fn outcome(error: &Error) -> MutationOutcome {
2692        if let Error::Connection(e) = error
2693            && e.is_zero_error()
2694        {
2695            return MutationOutcome::ZeroError {
2696                message: e.message().to_owned(),
2697            };
2698        }
2699        let app = ApplicationError::wrapping(error);
2700        MutationOutcome::AppError {
2701            message: app.message,
2702            details: app.details,
2703        }
2704    }
2705
2706    fn log_failure(&self, name: &str, outcome: &MutationOutcome, origin: &str) {
2707        match outcome {
2708            MutationOutcome::ZeroError { message } => self
2709                .log
2710                .error(|| format!("Mutator \"{name}\" error on {origin}: {message}")),
2711            MutationOutcome::AppError { message, .. } => self
2712                .log
2713                .error(|| format!("Mutator \"{name}\" app error on {origin}: {message}")),
2714            MutationOutcome::Success { .. } => {}
2715        }
2716    }
2717
2718    /// `zero.mutate`, through the `MutatorProxy` and Replicache's `mutate`.
2719    ///
2720    /// A mutation made while listeners are notified, from a listener, runs
2721    /// once the current task is done, as an async JS mutator commits after
2722    /// the notification: every listener sees the change that notified it
2723    /// before the mutation's, and every view sees both.
2724    pub(crate) fn mutate(&self, name: &str, args: Option<Value>) -> MutatorResult {
2725        let client = Deferred::new();
2726        let server = Deferred::new();
2727        let result = MutatorResult {
2728            client: client.clone(),
2729            server: server.clone(),
2730        };
2731        if self.context.is_notifying() {
2732            let w = self.me.clone();
2733            let name = name.to_owned();
2734            self.event_loop.after_current_task(move || {
2735                let Some(e) = w.upgrade() else { return };
2736                let r = e.mutate(&name, args);
2737                r.client.on_settle(move |o| client.settle(o.clone()));
2738                r.server.on_settle(move |o| server.settle(o.clone()));
2739            });
2740            return result;
2741        }
2742        // JS throws this before it looks at the connection; here both
2743        // outcomes carry it, and nothing is written or pushed.
2744        let Some(mutator) = self.mutators.get(name).cloned() else {
2745            let message = format!(
2746                "Mutator \"{name}\" is not registered. Mutators must be registered with the Zero \
2747                 constructor before use."
2748            );
2749            self.log.error(|| message.clone());
2750            client.resolve(MutationOutcome::ZeroError {
2751                message: message.clone(),
2752            });
2753            server.resolve(MutationOutcome::ZeroError { message });
2754            return result;
2755        };
2756        let rejection = self.mutation_rejection.borrow().clone();
2757        if let Some(rejection) = rejection {
2758            self.log.warn(|| {
2759                format!(
2760                    "Mutation \"{name}\" rejected because Zero is {}. Details: {}.",
2761                    rejection.state_description(),
2762                    rejection.message()
2763                )
2764            });
2765            let message = rejection.message().to_owned();
2766            client.resolve(MutationOutcome::ZeroError {
2767                message: message.clone(),
2768            });
2769            server.resolve(MutationOutcome::ZeroError { message });
2770            return result;
2771        }
2772        let (ephemeral_id, tracked) = self.mutation_tracker.track_mutation();
2773        {
2774            let w = self.me.clone();
2775            let name = name.to_owned();
2776            let server = server.clone();
2777            tracked.on_settle(move |r| match r {
2778                Ok(data) => server.resolve(MutationOutcome::Success { data: data.clone() }),
2779                Err(error) => {
2780                    let outcome = Self::outcome(error);
2781                    if let Some(e) = w.upgrade() {
2782                        e.log_failure(&name, &outcome, "server");
2783                    }
2784                    server.resolve(outcome);
2785                }
2786            });
2787        }
2788        // A mutate from inside a mutator, or while a poke rebases, waits for
2789        // it, as JS's waits for Replicache's write lock.
2790        if self.write_locked.get() || !self.waiting_for_write_lock.borrow().is_empty() {
2791            let w = self.me.clone();
2792            let name = name.to_owned();
2793            self.waiting_for_write_lock
2794                .borrow_mut()
2795                .push_back(Box::new(move || {
2796                    if let Some(e) = w.upgrade() {
2797                        e.with_write_lock(|| {
2798                            e.run_mutation(&mutator, &name, args, ephemeral_id, &client)
2799                        });
2800                    }
2801                }));
2802        } else {
2803            self.with_write_lock(|| self.run_mutation(&mutator, name, args, ephemeral_id, &client));
2804        }
2805        result
2806    }
2807
2808    fn with_write_lock<T>(&self, body: impl FnOnce() -> T) -> T {
2809        self.write_locked.set(true);
2810        let result = body();
2811        self.write_locked.set(false);
2812        if !self.waiting_for_write_lock.borrow().is_empty() {
2813            self.micro(|e| {
2814                if e.write_locked.get() {
2815                    return;
2816                }
2817                let next = e.waiting_for_write_lock.borrow_mut().pop_front();
2818                if let Some(next) = next {
2819                    next();
2820                }
2821            });
2822        }
2823        result
2824    }
2825
2826    fn run_mutation(
2827        &self,
2828        mutator: &MutatorFn,
2829        name: &str,
2830        args: Option<Value>,
2831        ephemeral_id: u64,
2832        client: &Deferred<MutationOutcome>,
2833    ) {
2834        let mutation_id = self.store.borrow().next_mutation_id(&self.client_id);
2835        // Last mutation IDs are checked when they arrive, so this holds.
2836        if !number::is_safe_integer(mutation_id) {
2837            let error: Error = ConnectionError::client(
2838                ClientErrorKind::Internal,
2839                format!("Invalid mutation ID {}", number::format(mutation_id)),
2840            )
2841            .into();
2842            self.mutation_tracker.reject_mutation(ephemeral_id, &error);
2843            client.resolve(Self::outcome(&error));
2844            return;
2845        }
2846        let write = KvWrite::new(self.store.borrow().main.clone());
2847        let branch = self.ivm_main.fork();
2848        self.mutation_tracker
2849            .mutation_id_assigned(ephemeral_id, mutation_id);
2850        let tx = Transaction::new(
2851            self.client_id.clone(),
2852            mutation_id,
2853            Reason::Optimistic,
2854            self.token(),
2855            write,
2856            branch,
2857            self.schema.clone(),
2858        );
2859        // JS forks the views' branch for the transaction, which throws once
2860        // that branch has failed.
2861        let ran = match self.context.branch_failure() {
2862            Some(f) => Err(f.into()),
2863            None => mutator(&tx, args.as_ref()),
2864        };
2865        let data = match ran {
2866            Ok(data) => data,
2867            Err(error) => {
2868                self.mutation_tracker.reject_mutation(ephemeral_id, &error);
2869                let outcome = Self::outcome(&error);
2870                self.log_failure(name, &outcome, "client");
2871                client.resolve(outcome);
2872                return;
2873            }
2874        };
2875        let (write, rows) = tx.finish();
2876        let pending = PendingMutation {
2877            id: mutation_id,
2878            client_id: self.client_id.clone(),
2879            name: name.to_owned(),
2880            args: args.clone().unwrap_or(Value::Null),
2881            timestamp: self.scheduler.monotonic_now(),
2882            writes: write.writes.clone(),
2883        };
2884        let stored = StoredMutation {
2885            client_id: self.client_id.clone(),
2886            id: mutation_id,
2887            name: name.to_owned(),
2888            args: pending.args.clone(),
2889            timestamp: pending.timestamp,
2890            aggregate_writes: aggregate_writes(&write),
2891        };
2892        let diff = self.store.borrow_mut().commit_mutation(pending, &write);
2893        self.database.apply(vec![StorageOp::AddMutation {
2894            group: self.client_group_id.clone(),
2895            mutation: stored,
2896        }]);
2897        if let Err(error) = self.context.process_changes(&diff, &rows) {
2898            // As in JS, the mutation stays committed and pending, but isn't
2899            // pushed now, and its client outcome is the error.
2900            self.fire_watchers(&diff, false);
2901            let outcome = Self::outcome(&error.into());
2902            self.log_failure(name, &outcome, "client");
2903            client.resolve(outcome);
2904            return;
2905        }
2906        self.request_push(false);
2907        self.fire_watchers(&diff, false);
2908        self.schedule_persist();
2909        client.resolve(MutationOutcome::Success { data });
2910    }
2911
2912    /// The store watches: the mutation tracker on `m/<clientID>/`, then the
2913    /// query manager on `g/`. A watch hears only about diffs that touch its
2914    /// prefix, except for its first run.
2915    fn fire_watchers(&self, diff: &KvDiff, initial: bool) {
2916        let mutations = with_prefix(diff, &format!("{MUTATIONS_PREFIX}{}/", self.client_id));
2917        if initial || !mutations.is_empty() {
2918            // A fatal error disconnects from inside the tracker.
2919            let _ = self.mutation_tracker.process_mutation_responses(&mutations);
2920        }
2921        let got = with_prefix(diff, GOT_QUERIES_PREFIX);
2922        if initial || !got.is_empty() {
2923            self.query_manager.process_got_queries(&got);
2924        }
2925    }
2926
2927    // MARK: Pushing
2928
2929    /// `pushConnectionLoop.send(now)`.
2930    fn request_push(&self, now: bool) {
2931        if self.rep_closed.get() {
2932            return;
2933        }
2934        self.push_requested.set(true);
2935        if self.push_in_flight.get() {
2936            return;
2937        }
2938        if now {
2939            if let Some(t) = self.push_timer.borrow_mut().take() {
2940                t.cancel();
2941            }
2942            self.micro(|e| e.invoke_push());
2943        } else if self.push_timer.borrow().is_none() {
2944            let t = self.later(0.0, |e| {
2945                e.push_timer.borrow_mut().take();
2946                e.invoke_push();
2947            });
2948            *self.push_timer.borrow_mut() = Some(t);
2949        }
2950    }
2951
2952    /// `#invokePush`: pushes the pending mutations once connected.
2953    fn invoke_push(&self) {
2954        if self.push_in_flight.get()
2955            || !self.push_requested.get()
2956            || self.rep_closed.get()
2957            || self.client_group_disabled.get()
2958        {
2959            return;
2960        }
2961        self.push_requested.set(false);
2962        let pending = self.store.borrow().pending.clone();
2963        if pending.is_empty() {
2964            return;
2965        }
2966        self.push_in_flight.set(true);
2967        let w = self.me.clone();
2968        let ready = self.connection_ready.borrow().clone();
2969        ready.on_settle(&self.event_loop, move |connected| {
2970            let Some(e) = w.upgrade() else { return };
2971            if connected {
2972                e.pusher(&pending);
2973            }
2974            e.push_in_flight.set(false);
2975            if !connected {
2976                e.push_requested.set(true);
2977            }
2978            if e.push_requested.get() {
2979                e.request_push(false);
2980            }
2981        });
2982    }
2983
2984    /// `#pusher`: one `push` message per mutation not yet sent on this
2985    /// connection.
2986    fn pusher(&self, mutations: &[PendingMutation]) {
2987        let now = self.scheduler.now();
2988        let counter = self.push_request_counter.get() + 1;
2989        self.push_request_counter.set(counter);
2990        let request_id = format!("{}-{counter}", self.client_id);
2991        for m in mutations {
2992            let sent = self
2993                .last_mutation_ids_sent
2994                .borrow()
2995                .get(&m.client_id)
2996                .copied()
2997                .unwrap_or(0.0);
2998            if m.id <= sent {
2999                continue;
3000            }
3001            let timestamp = now - (self.scheduler.monotonic_now() - m.timestamp).round();
3002            self.send(Upstream::Push(PushBody {
3003                client_group_id: self.client_group_id.clone(),
3004                mutations: vec![Mutation::Custom {
3005                    id: m.id,
3006                    client_id: m.client_id.clone(),
3007                    name: m.name.clone(),
3008                    args: vec![m.args.clone()],
3009                    timestamp,
3010                }],
3011                push_version: 1.0,
3012                schema_version: None,
3013                timestamp: now,
3014                request_id: request_id.clone(),
3015                auth: None,
3016                traceparent: self.traceparent(),
3017            }));
3018            self.last_mutation_ids_sent
3019                .borrow_mut()
3020                .insert(m.client_id.clone(), m.id);
3021        }
3022    }
3023
3024    // MARK: Pokes
3025
3026    /// `rep.poke`: applies a merged poke to the snapshot and rebases the
3027    /// pending mutations on it (`handlePullResponseV1` and
3028    /// `maybeEndPull`). Errors are JS's `String(error)`.
3029    fn apply_poke(&self, poke: &MergedPoke) -> Result<(), String> {
3030        let socket_at_start = self.socket.borrow().clone();
3031        let base = self.store.borrow().snapshot.clone();
3032        if poke.base_cookie != base.cookie {
3033            return Err(format!(
3034                "Error: unexpected base cookie for poke: {}",
3035                json::stringify(&poke.to_json())
3036            ));
3037        }
3038        let mut changes: Vec<(&String, &f64)> = poke.last_mutation_id_changes.iter().collect();
3039        changes.sort_by(|a, b| strings::utf16(a.0, b.0));
3040        for (cid, lmid) in changes {
3041            if let Some(last) = base.last_mutation_ids.get(cid)
3042                && lmid < last
3043            {
3044                return Err(format!(
3045                    "Error: {}",
3046                    bad_order_message(
3047                        &format!("{cid} lastMutationID"),
3048                        &number::format(*lmid),
3049                        &number::format(*last)
3050                    )
3051                ));
3052            }
3053        }
3054        if compare_cookies(&poke.cookie, &base.cookie) < 0.0 {
3055            return Err(format!(
3056                "Error: {}",
3057                bad_order_message(
3058                    "cookie",
3059                    &json::stringify(&poke.cookie),
3060                    &json::stringify(&base.cookie)
3061                )
3062            ));
3063        }
3064        if poke.cookie == base.cookie {
3065            // `zero.ts` trusts the got queries once `rep.poke` returns, even
3066            // for a poke that changes nothing.
3067            self.mark_got_queries_authoritative(socket_at_start.as_ref());
3068            return Ok(());
3069        }
3070
3071        let mut snapshot = base;
3072        for (cid, lmid) in &poke.last_mutation_id_changes {
3073            snapshot.last_mutation_ids.insert(cid.clone(), *lmid);
3074        }
3075        snapshot.cookie = poke.cookie.clone();
3076        let mut patch_write = KvWrite::new(snapshot.map.clone());
3077        apply_patch(&poke.patch, &mut patch_write).map_err(|e| format!("Error: {}", e.0))?;
3078        self.check_updated_rows(&poke.patch, &patch_write)?;
3079        snapshot.map = patch_write.map.clone();
3080
3081        let (store_main, pending_writes, pending) = {
3082            let store = self.store.borrow();
3083            (
3084                store.main.clone(),
3085                store.pending_writes(),
3086                store.pending.clone(),
3087            )
3088        };
3089        let mut touched: HashSet<String> =
3090            patch_write.writes.union(&pending_writes).cloned().collect();
3091        let mut main = snapshot.map.clone();
3092        let mut replayed = Vec::new();
3093        let pending: Vec<PendingMutation> = pending
3094            .into_iter()
3095            .filter(|m| {
3096                m.id > snapshot
3097                    .last_mutation_ids
3098                    .get(&m.client_id)
3099                    .copied()
3100                    .unwrap_or(0.0)
3101            })
3102            .collect();
3103        if !pending.is_empty() {
3104            // The rebase runs over a fork of the views' branch moved to the
3105            // new snapshot. JS's fork throws once that branch has failed.
3106            if let Some(f) = self.context.branch_failure() {
3107                return Err(format!("Error: {f}"));
3108            }
3109            let branch = self.ivm_main.fork();
3110            let moved = diff_keys(&store_main, &snapshot.map, &touched);
3111            capture(|| branch.apply(&moved, &HashMap::new())).map_err(|e| format!("Error: {e}"))?;
3112            let (m, r, t, _) =
3113                self.replay(&pending, &main, branch, &HashMap::new(), &mut |_, _| {});
3114            main = m;
3115            replayed = r;
3116            touched.extend(t);
3117        }
3118        let cookie = snapshot.cookie.clone();
3119        let last_mutation_ids = snapshot.last_mutation_ids.clone();
3120        let changes: Vec<(String, Option<Value>)> = {
3121            let mut keys: Vec<&String> = patch_write.writes.iter().collect();
3122            keys.sort();
3123            keys.into_iter()
3124                .filter_map(|k| {
3125                    let v = snapshot.map.get(k).cloned();
3126                    // After a clear, only what the poke put back.
3127                    (!(patch_write.cleared && v.is_none())).then(|| (k.clone(), v))
3128                })
3129                .collect()
3130        };
3131        let diff = self
3132            .store
3133            .borrow_mut()
3134            .advance(snapshot, main, replayed, Some(&touched));
3135        self.database.apply(vec![StorageOp::UpdateSnapshot {
3136            group: self.client_group_id.clone(),
3137            changes,
3138            cleared: patch_write.cleared,
3139            cookie,
3140            last_mutation_ids,
3141        }]);
3142        // A push that fails is a poke error: the client disconnects.
3143        self.context
3144            .process_changes(&diff, &HashMap::new())
3145            .map_err(|e| format!("Error: {e}"))?;
3146        self.fire_watchers(&diff, false);
3147        self.schedule_persist();
3148        self.mark_got_queries_authoritative(socket_at_start.as_ref());
3149        Ok(())
3150    }
3151
3152    /// The server has caught the client up, unless the connection the poke
3153    /// came on went away while it was applied: the disconnect already
3154    /// re-gated trust, and the next connection's first poke restores it.
3155    fn mark_got_queries_authoritative(&self, socket_at_start: Option<&Rc<SocketHandle>>) {
3156        if let Some(s) = socket_at_start
3157            && self.is_socket(s)
3158        {
3159            self.query_manager.mark_got_queries_authoritative();
3160        }
3161    }
3162
3163    /// Runs `pending` again, in order, over `map` and on forks of `branch`
3164    /// (the rebase of `maybeEndPull`). A mutation that fails stays pending
3165    /// without writes. Returns the new main map, the mutations with what
3166    /// they wrote, every key they wrote, and the branch they ran on.
3167    ///
3168    /// At launch, no query has registered the synced aggregates a mutation
3169    /// moves yet, so each run puts back the aggregate rows `restoring` has
3170    /// for the mutation, as JS's stored commit holds them. `ran` hears each
3171    /// mutation with its write, `None` when it failed.
3172    #[allow(clippy::type_complexity)]
3173    fn replay(
3174        &self,
3175        pending: &[PendingMutation],
3176        map: &KvMap,
3177        branch: Rc<IvmBranch>,
3178        restoring: &HashMap<MutationKey, AggregateWrites>,
3179        ran: &mut dyn FnMut(&PendingMutation, Option<&KvWrite>),
3180    ) -> (KvMap, Vec<PendingMutation>, HashSet<String>, Rc<IvmBranch>) {
3181        let mut main = map.clone();
3182        let mut branch = branch;
3183        let mut touched = HashSet::new();
3184        let mut replayed = Vec::new();
3185        for m in pending {
3186            let mut m = m.clone();
3187            let fork = branch.fork();
3188            let tx = Transaction::new(
3189                m.client_id.clone(),
3190                m.id,
3191                Reason::Rebase,
3192                self.token(),
3193                KvWrite::new(main.clone()),
3194                fork.clone(),
3195                self.schema.clone(),
3196            );
3197            let result = match self.mutators.get(&m.name) {
3198                Some(mutator) => mutator(&tx, Some(&m.args).filter(|a| !a.is_null())),
3199                None => Err(ApplicationError::new(format!("Unknown mutator: {}", m.name)).into()),
3200            };
3201            match result {
3202                Ok(_) => {
3203                    let (mut write, _) = tx.finish();
3204                    if let Some(writes) = restoring.get(&mutation_key(&m.client_id, m.id)) {
3205                        for (key, value) in writes {
3206                            match value {
3207                                Some(v) => write.set(key, v.clone()),
3208                                None => {
3209                                    write.delete(key);
3210                                }
3211                            }
3212                        }
3213                    }
3214                    main = write.map.clone();
3215                    branch = fork;
3216                    m.writes = write.writes.clone();
3217                    touched.extend(write.writes.iter().cloned());
3218                    ran(&m, Some(&write));
3219                }
3220                Err(_) => {
3221                    // The mutation stays pending; its writes are dropped.
3222                    m.writes = HashSet::new();
3223                    ran(&m, None);
3224                }
3225            }
3226            replayed.push(m);
3227        }
3228        (main, replayed, touched, branch)
3229    }
3230
3231    /// Replicache's persist scheduler (`ProcessScheduler`): after a
3232    /// mutation or a poke, a persist runs once the last one is
3233    /// `PERSIST_THROTTLE_MS` old, then after `PERSIST_IDLE_TIMEOUT_MS`;
3234    /// requests while one waits join it.
3235    fn schedule_persist(&self) {
3236        if self.persist_timer.borrow().is_some() || self.rep_closed.get() {
3237            return;
3238        }
3239        let now = self.scheduler.monotonic_now();
3240        let throttled = self
3241            .last_persist_at
3242            .get()
3243            .map_or(0.0, |at| (at + PERSIST_THROTTLE_MS - now).max(0.0));
3244        let t = self.later(throttled + PERSIST_IDLE_TIMEOUT_MS, |e| e.persist());
3245        *self.persist_timer.borrow_mut() = Some(t);
3246    }
3247
3248    /// `persistDD31`, for the synced aggregate rows the store keeps of each
3249    /// pending mutation. The snapshot and the mutations are written through
3250    /// as they come, and the mutations replayed at launch; JS persists its
3251    /// commits now and then, running the mutations again for it, so the
3252    /// aggregates they move are the ones registered at that persist, not
3253    /// when they ran. A persist runs the mutations it hasn't persisted over
3254    /// what it persisted before, or, when the snapshot is newer, all of
3255    /// them over it, and stores the aggregate rows each wrote.
3256    fn persist(&self) {
3257        self.persist_timer.borrow_mut().take();
3258        self.last_persist_at
3259            .set(Some(self.scheduler.monotonic_now()));
3260        let (pending, snapshot_cookie, snapshot_map, store_main, pending_writes) = {
3261            let store = self.store.borrow();
3262            (
3263                store.pending.clone(),
3264                store.snapshot.cookie.clone(),
3265                store.snapshot.map.clone(),
3266                store.main.clone(),
3267                store.pending_writes(),
3268            )
3269        };
3270        let to_run: Vec<PendingMutation> = {
3271            let mut persisted = self.persisted.borrow_mut();
3272            let to_run = if persisted.cookie != snapshot_cookie {
3273                persisted.cookie = snapshot_cookie;
3274                persisted.main = snapshot_map;
3275                persisted.touched.clear();
3276                pending.clone()
3277            } else {
3278                pending
3279                    .iter()
3280                    .filter(|m| {
3281                        !persisted
3282                            .mutations
3283                            .contains(&mutation_key(&m.client_id, m.id))
3284                    })
3285                    .cloned()
3286                    .collect()
3287            };
3288            persisted.mutations = pending
3289                .iter()
3290                .map(|m| mutation_key(&m.client_id, m.id))
3291                .collect();
3292            to_run
3293        };
3294        // The persist runs over a fork of the views' branch moved to what
3295        // it persisted. JS's fork throws once that branch has failed, and
3296        // the persist with it; the branch's sources no longer follow the
3297        // replica.
3298        if to_run.is_empty() || self.context.branch_failure().is_some() {
3299            return;
3300        }
3301        let (persisted_main, persisted_touched) = {
3302            let p = self.persisted.borrow();
3303            (p.main.clone(), p.touched.clone())
3304        };
3305        let branch = self.ivm_main.fork();
3306        let keys: HashSet<String> = pending_writes.union(&persisted_touched).cloned().collect();
3307        let moved = diff_keys(&store_main, &persisted_main, &keys);
3308        if capture(|| branch.apply(&moved, &HashMap::new())).is_err() {
3309            return;
3310        }
3311        let mut ops = Vec::new();
3312        let group = self.client_group_id.clone();
3313        let (main, _, touched, _) = self.replay(
3314            &to_run,
3315            &persisted_main,
3316            branch,
3317            &HashMap::new(),
3318            &mut |m, write| {
3319                ops.push(StorageOp::SetAggregateWrites {
3320                    group: group.clone(),
3321                    client_id: m.client_id.clone(),
3322                    id: m.id,
3323                    writes: write.map(aggregate_writes).unwrap_or_default(),
3324                });
3325            },
3326        );
3327        {
3328            let mut p = self.persisted.borrow_mut();
3329            p.main = main;
3330            p.touched.extend(touched);
3331        }
3332        self.database.apply(ops);
3333    }
3334
3335    /// Rejects a poke whose `update` ops leave a row under a key that isn't
3336    /// its primary key: one whose merge drops or changes the key's columns.
3337    /// The table's source would then hold rows that disagree with the
3338    /// replica, which JS's `MemorySource` reports by throwing ("Row already
3339    /// exists", "Row not found") and the poke handler by disconnecting.
3340    fn check_updated_rows(&self, patch: &[KvPatchOp], write: &KvWrite) -> Result<(), String> {
3341        let mut keyless: HashMap<String, Vec<Row>> = HashMap::new();
3342        for op in patch {
3343            let KvPatchOp::Update { key, .. } = op else {
3344                continue;
3345            };
3346            let Some(value) = write.get(key) else {
3347                continue;
3348            };
3349            let table = keys::source_name(key);
3350            let Some(t) = self.schema.tables.get(&table) else {
3351                continue;
3352            };
3353            let row = value.as_object().cloned().unwrap_or_default();
3354            if let Ok(row_key) = keys::primary_key_string(&table, &t.primary_key, &row) {
3355                if row_key != *key {
3356                    return Err(format!(
3357                        "Error: update of {key} changes its primary key to {row_key}"
3358                    ));
3359                }
3360                continue;
3361            }
3362            // JS keeps a row whose merge leaves it without a usable primary
3363            // key, and fails once a second one collides with it in the
3364            // source.
3365            let Some(source) = self.ivm_main.source(&table) else {
3366                continue;
3367            };
3368            let row: Row = Rc::new(row);
3369            let previous = self
3370                .store
3371                .borrow()
3372                .main
3373                .get(key)
3374                .map(|v| Rc::new(v.as_object().cloned().unwrap_or_default()));
3375            let replaces_itself = previous.is_some_and(|p| source.same_key(&p, &row));
3376            let rows = keyless.entry(table).or_default();
3377            if (!replaces_itself && source.contains(&row))
3378                || rows.iter().any(|r| source.same_key(r, &row))
3379            {
3380                return Err(format!("Error: Row already exists: update of {key}"));
3381            }
3382            rows.push(row);
3383        }
3384        Ok(())
3385    }
3386}
3387
3388impl Drop for Engine {
3389    fn drop(&mut self) {
3390        self.event_loop.stop();
3391        for t in self.lifecycle_timers.get_mut().drain(..) {
3392            t.cancel();
3393        }
3394        if !self.rep_closed.get() {
3395            self.database.client_stopped(&self.client_id);
3396            self.database.remove_failure_listener(&self.client_id);
3397            self.database.release();
3398        }
3399    }
3400}
3401
3402#[cfg(test)]
3403mod tests {
3404    use super::*;
3405
3406    #[test]
3407    fn iso_dates() {
3408        assert_eq!(
3409            iso_date_string(1_700_000_000_000.0),
3410            "2023-11-14T22:13:20.000Z"
3411        );
3412        assert_eq!(iso_date_string(0.0), "1970-01-01T00:00:00.000Z");
3413        assert_eq!(iso_date_string(-1.0), "1969-12-31T23:59:59.999Z");
3414    }
3415
3416    #[test]
3417    fn backoff() {
3418        assert_eq!(reconnect_backoff_ms(1, 5000.0, 0.0), 250.0);
3419        assert_eq!(reconnect_backoff_ms(1, 5000.0, 1.0), 500.0);
3420        assert_eq!(reconnect_backoff_ms(4, 5000.0, 0.5), 3000.0);
3421        assert_eq!(reconnect_backoff_ms(5, 5000.0, 0.5), 5000.0);
3422    }
3423}