Skip to main content

zero_client/client/
engine.rs

1//! The client engine (`Zero` in `packages/zero-client/src/client/zero.ts`,
2//! with the parts of Replicache it uses): the run loop that keeps a
3//! connection, the replica and its pending mutations, and the query and
4//! mutation bookkeeping. Everything runs synchronously on one thread; see
5//! `event_loop`.
6
7use super::connection_manager::{ConnectionManager, State};
8use super::database::{AggregateWrites, ReplicaDatabase, StorageOp, StoredMutation, base36};
9use super::event_loop::{EventLoop, LoopScheduler};
10use super::ivm_branch::IvmBranch;
11use super::log::Log;
12use super::mutation_tracker::{MutationTracker, OutcomeListeners};
13use super::poke_handler::{MergedPoke, PokeHandler};
14use super::query_context::QueryContext;
15use super::query_manager::QueryManager;
16use super::replica_store::{
17    KvDiff, KvMap, KvPatchOp, KvWrite, PendingMutation, ReplicaStore, Snapshot, apply_patch,
18    compare_cookies, diff_all, diff_keys, with_prefix,
19};
20use super::socket::{SocketConnection, SocketEvents, SocketFactory, SocketMessage};
21use super::storage::{LoadError, StorageFailure, ZeroStorage};
22use super::transaction::{Reason, Transaction};
23use super::{
24    ApplicationError, ClientErrorKind, ConnectionError, ConnectionStatus, ConnectionTransition,
25    Deferred, Error, LogLevel, LogSink, Scheduler, StderrLogSink, Timer,
26};
27use crate::decode::{DecodeError, JsonCodable};
28use crate::hash::xxhash32;
29use crate::ivm::{Row, capture};
30use crate::json::{self, Object, Value, number};
31use crate::keys::{self, AGGREGATES_PREFIX, GOT_QUERIES_PREFIX, MUTATIONS_PREFIX};
32use crate::map::OrderedMap;
33use crate::protocol::{
34    ChangeDesiredQueriesBody, ConnectedBody, ConnectionUrl, DeleteClientsBody, Downstream,
35    ErrorBody, ErrorKind, InitConnectionBody, Mutation, PushBody, PushResponseBody,
36    UpQueriesPatchOp, UpdateAuthBody, Upstream, poke_chunk, sec_protocols,
37};
38use crate::schema::Schema;
39use crate::strings;
40use std::cell::{Cell, RefCell};
41use std::collections::{HashMap, HashSet, VecDeque};
42use std::hash::{BuildHasher, Hasher};
43use std::rc::{Rc, Weak};
44use std::sync::Arc;
45
46/// `RUN_LOOP_INTERVAL_MS`: the backoff before retrying a failed connection.
47pub(crate) const RUN_LOOP_INTERVAL_MS: f64 = 5_000.0;
48/// `FAST_RECONNECT_INITIAL_MS`: the longest wait before the first retry
49/// after a transport failure. Each further consecutive failure doubles it,
50/// up to `RUN_LOOP_INTERVAL_MS`.
51const FAST_RECONNECT_INITIAL_MS: f64 = 500.0;
52/// `STABLE_CONNECTION_MS`: how long a connection must stay up before a
53/// later failure starts the fast schedule from the beginning again.
54const STABLE_CONNECTION_MS: f64 = 5_000.0;
55/// `CONNECT_TIMEOUT_MS`: each phase of a connection attempt gets this long.
56const CONNECT_TIMEOUT_MS: f64 = 10_000.0;
57/// `HEARTBEAT_INTERVAL`.
58const HEARTBEAT_INTERVAL_MS: f64 = 60_000.0;
59/// `GC_INTERVAL`, of clients and of client groups.
60const GC_INTERVAL_MS: f64 = 5.0 * 60_000.0;
61/// `CLIENT_MAX_INACTIVE_TIME`.
62const CLIENT_MAX_INACTIVE_MS: f64 = 24.0 * 60.0 * 60_000.0;
63/// `INITIAL_COLLECT_IDB_DELAY`.
64const INITIAL_COLLECT_DATABASES_DELAY_MS: f64 = 5.0 * 60_000.0;
65/// `COLLECT_IDB_INTERVAL`.
66const COLLECT_DATABASES_INTERVAL_MS: f64 = 12.0 * 60.0 * 60_000.0;
67/// `PERSIST_IDLE_TIMEOUT_MS` and `PERSIST_THROTTLE_MS`.
68const PERSIST_IDLE_TIMEOUT_MS: f64 = 1000.0;
69const PERSIST_THROTTLE_MS: f64 = 500.0;
70
71/// `reconnectBackoffMs`: the wait before retrying after `failed_attempts`
72/// consecutive transport failures (1 for the first). The ceiling starts at
73/// `FAST_RECONNECT_INITIAL_MS` and doubles with each failure, and the wait
74/// is between half the ceiling and the ceiling, so that clients that lost
75/// the same server don't all retry at once. Once the ceiling reaches
76/// `max_ms`, the wait is `max_ms`.
77pub(crate) fn reconnect_backoff_ms(failed_attempts: u32, max_ms: f64, random: f64) -> f64 {
78    let ceiling = FAST_RECONNECT_INITIAL_MS * 2f64.powi(failed_attempts as i32 - 1);
79    if ceiling >= max_ms {
80        return max_ms;
81    }
82    (ceiling / 2.0 + random * ceiling / 2.0).round()
83}
84
85thread_local! {
86    static RNG: Cell<u64> = Cell::new({
87        let mut h = std::collections::hash_map::RandomState::new().build_hasher();
88        h.write_u128(
89            std::time::SystemTime::now()
90                .duration_since(std::time::UNIX_EPOCH)
91                .map(|d| d.as_nanos())
92                .unwrap_or(0),
93        );
94        h.finish() | 1
95    });
96}
97
98/// A random 64-bit number (xorshift64*, seeded per thread from the OS).
99fn random_u64() -> u64 {
100    RNG.with(|r| {
101        let mut x = r.get();
102        x ^= x >> 12;
103        x ^= x << 25;
104        x ^= x >> 27;
105        r.set(x);
106        x.wrapping_mul(0x2545_F491_4F6C_DD1D)
107    })
108}
109
110/// `Math.random()`.
111fn random_f64() -> f64 {
112    (random_u64() >> 11) as f64 / (1u64 << 53) as f64
113}
114
115/// A random ID like the JS client's `nanoid`.
116pub(crate) fn random_id() -> String {
117    const ALPHABET: &[u8; 64] = b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-";
118    (0..21)
119        .map(|_| ALPHABET[(random_u64() % 64) as usize] as char)
120        .collect()
121}
122
123/// `Date.prototype.toISOString` of a time in milliseconds since 1970.
124pub(crate) fn iso_date_string(ms: f64) -> String {
125    let t = ms.floor() as i64;
126    let mut days = t.div_euclid(86_400_000);
127    let ms_of_day = t.rem_euclid(86_400_000);
128    // Howard Hinnant's civil_from_days.
129    days += 719_468;
130    let era = days.div_euclid(146_097);
131    let doe = days - era * 146_097;
132    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
133    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
134    let mp = (5 * doy + 2) / 153;
135    let d = doy - (153 * mp + 2) / 5 + 1;
136    let m = if mp < 10 { mp + 3 } else { mp - 9 };
137    let y = yoe + era * 400 + i64::from(m <= 2);
138    format!(
139        "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}.{:03}Z",
140        ms_of_day / 3_600_000,
141        ms_of_day / 60_000 % 60,
142        ms_of_day / 1000 % 60,
143        ms_of_day % 1000
144    )
145}
146
147/// The message valita gives for a message that doesn't parse (`getMessage`
148/// in `packages/shared/src/valita.ts`), for the cases the client can tell
149/// apart.
150fn valita_message(e: &DecodeError, text: &str) -> String {
151    if e.message.starts_with("Unknown message type") || e.message == "Expected [tag, body] tuple" {
152        return format!("Invalid union value: {text}");
153    }
154    e.valita_message()
155}
156
157/// The options of a Zero client (`ZeroOptions` in
158/// `packages/zero-client/src/client/options.ts`).
159#[derive(Clone)]
160pub struct ZeroOptions {
161    /// The zero-cache URL, `http` or `https`. Without one the client runs
162    /// locally and never connects.
163    pub cache_url: Option<String>,
164    /// The user the client syncs for. `None` for a logged-out client.
165    pub user_id: Option<String>,
166    /// The auth token zero-cache and the API server see.
167    pub auth: Option<String>,
168    /// Distinguishes local stores of the same user and URLs.
169    pub storage_key: Option<String>,
170    /// How long the client waits for a pong before it reconnects.
171    pub ping_timeout_ms: f64,
172    /// How long the client tries to reconnect before the state is
173    /// `Disconnected`.
174    pub disconnect_timeout_ms: f64,
175    /// How long query changes are batched before they are sent.
176    pub query_change_throttle_ms: f64,
177    /// The longest `Sec-WebSocket-Protocol` header the client sends.
178    /// `initConnection` is sent as a message instead when it doesn't fit.
179    pub max_header_length: usize,
180    /// The API server's push URL, when it differs from the one zero-cache is
181    /// configured with.
182    pub mutate_url: Option<String>,
183    /// Headers zero-cache sends to `mutate_url`.
184    pub mutate_headers: Option<Vec<(String, String)>>,
185    /// The API server's query URL, when it differs from the one zero-cache is
186    /// configured with.
187    pub query_url: Option<String>,
188    /// Headers zero-cache sends to `query_url`.
189    pub query_headers: Option<Vec<(String, String)>>,
190    /// The server needs a newer client. Defaults to doing nothing.
191    pub on_update_needed: Option<UpdateNeededHandler>,
192    /// The server no longer has the client's state, or the stored replica
193    /// didn't decode and the local store was reset. Without it, a client
194    /// the server lost replaces itself by a new one on the same store, as
195    /// JS reloads the page (see [`Zero::new`]).
196    ///
197    /// [`Zero::new`]: crate::client::Zero::new
198    pub on_client_state_not_found: Option<Rc<dyn Fn()>>,
199    /// Where the client keeps its replica. Defaults to
200    /// [`ZeroStorage::Memory`], which keeps nothing across restarts.
201    pub storage: ZeroStorage,
202    /// The local store failed, so the client runs on what it has in memory
203    /// and persists nothing more (`onStorageFailure`). Called once.
204    pub on_storage_failure: Option<StorageFailureHandler>,
205    /// The least severe messages logged.
206    pub log_level: LogLevel,
207    /// Where messages are logged. Defaults to [`StderrLogSink`].
208    pub log_sink: Option<Rc<dyn LogSink>>,
209    /// How long the app is hidden before the client disconnects
210    /// (`hiddenTabDisconnectDelay`); see [`Zero::set_visible`]. 5 minutes
211    /// by default, as in JS. A mobile app, which the system suspends soon
212    /// after it goes to the background, can set 0 to disconnect as soon as
213    /// it is hidden.
214    ///
215    /// [`Zero::set_visible`]: crate::client::Zero::set_visible
216    pub hidden_disconnect_delay_ms: f64,
217    /// A W3C `traceparent` for each query change and push the client sends
218    /// (`getTraceparent`), so zero-cache and the API server can join the
219    /// app's trace. Called once per message; `None` sends none.
220    pub get_traceparent: Option<Rc<dyn Fn() -> Option<String>>>,
221    /// Wraps every update of the views (`batchViewUpdates`), for a UI that
222    /// renders a batch of state changes at once: each new view's creation,
223    /// and each change that updates the views (a poke, a mutation, the
224    /// first hydration), with its listener calls. It must call the function
225    /// it is given, once, before it returns.
226    pub batch_view_updates: Option<BatchViewUpdates>,
227    /// How long, in ms, a query may take to materialize, including the
228    /// server's and the network's time, before the client logs a warning
229    /// (`slowMaterializeThreshold`). Faster materializations are logged at
230    /// debug level.
231    pub slow_materialize_threshold_ms: f64,
232}
233
234impl Default for ZeroOptions {
235    fn default() -> Self {
236        ZeroOptions {
237            cache_url: None,
238            user_id: None,
239            auth: None,
240            storage_key: None,
241            ping_timeout_ms: 5_000.0,
242            disconnect_timeout_ms: 60_000.0,
243            query_change_throttle_ms: 10.0,
244            max_header_length: 8 * 1024,
245            mutate_url: None,
246            mutate_headers: None,
247            query_url: None,
248            query_headers: None,
249            on_update_needed: None,
250            on_client_state_not_found: None,
251            storage: ZeroStorage::Memory,
252            on_storage_failure: None,
253            log_level: LogLevel::Warn,
254            log_sink: None,
255            hidden_disconnect_delay_ms: 300_000.0,
256            get_traceparent: None,
257            batch_view_updates: None,
258            slow_materialize_threshold_ms: 5_000.0,
259        }
260    }
261}
262
263/// Called when the local store fails (`onStorageFailure`).
264pub type StorageFailureHandler = Rc<dyn Fn(&StorageFailure)>;
265
266/// Called when the server needs a newer client (`onUpdateNeeded`).
267pub type UpdateNeededHandler = Rc<dyn Fn(&UpdateNeededReason)>;
268
269/// [`ZeroOptions::batch_view_updates`]: called with the function that
270/// updates the views.
271pub type BatchViewUpdates = Rc<dyn Fn(&mut dyn FnMut())>;
272
273/// Why the client must be updated (`UpdateNeededReason`).
274#[derive(Clone, Debug, PartialEq, Eq)]
275pub struct UpdateNeededReason {
276    /// `VersionNotSupported`, `SchemaVersionNotSupported`, or
277    /// `NewClientGroup`.
278    pub type_: String,
279    /// The server's explanation, if it gave one.
280    pub message: Option<String>,
281}
282
283impl UpdateNeededReason {
284    /// The JS reason object.
285    pub fn to_json(&self) -> Value {
286        let mut o = Object::new();
287        o.insert("type", Value::String(self.type_.clone()));
288        if let Some(m) = &self.message {
289            o.insert("message", Value::String(m.clone()));
290        }
291        Value::Object(o)
292    }
293}
294
295/// How a mutation ended, on the client or on the server
296/// (`MutatorResultDetails` in `packages/zero-client/src/client/custom.ts`).
297#[derive(Clone, Debug, PartialEq)]
298pub enum MutationOutcome {
299    /// The mutation ran.
300    Success {
301        /// What the mutator returned (client) or the server's result.
302        data: Option<Value>,
303    },
304    /// The mutator or the API server raised an [`ApplicationError`].
305    AppError {
306        /// The error's message.
307        message: String,
308        /// The error's details.
309        details: Option<Value>,
310    },
311    /// Zero couldn't run the mutation or learn its result, such as while
312    /// offline or when the connection closed. On the server side this
313    /// doesn't mean the mutation wasn't applied: the client sends pending
314    /// mutations again when it reconnects.
315    ZeroError {
316        /// Why.
317        message: String,
318    },
319}
320
321impl MutationOutcome {
322    /// The JS result object.
323    pub fn to_json(&self) -> Value {
324        let mut o = Object::new();
325        match self {
326            MutationOutcome::Success { data } => {
327                o.insert("type", Value::from("success"));
328                if let Some(d) = data {
329                    o.insert("data", d.clone());
330                }
331            }
332            MutationOutcome::AppError { message, details } => {
333                let mut e = Object::new();
334                e.insert("type", Value::from("app"));
335                e.insert("message", Value::String(message.clone()));
336                if let Some(d) = details {
337                    e.insert("details", d.clone());
338                }
339                o.insert("type", Value::from("error"));
340                o.insert("error", Value::Object(e));
341            }
342            MutationOutcome::ZeroError { message } => {
343                let mut e = Object::new();
344                e.insert("type", Value::from("zero"));
345                e.insert("message", Value::String(message.clone()));
346                o.insert("type", Value::from("error"));
347                o.insert("error", Value::Object(e));
348            }
349        }
350        Value::Object(o)
351    }
352}
353
354/// The server's outcome of a mutation this client made, reported by
355/// [`Zero::add_mutation_outcome_listener`](super::Zero::add_mutation_outcome_listener).
356/// Swift's `MutationServerOutcome`: the JS client has no counterpart.
357#[derive(Clone, Debug, PartialEq)]
358pub struct MutationServerOutcome {
359    /// The mutation's ID, counted per client from 1.
360    pub mutation_id: u64,
361    /// The mutator's name, such as `issue.create`.
362    pub name: String,
363    /// The mutator's arguments: `None` when it was given none.
364    pub args: Option<Value>,
365    /// `Success` or `AppError`, with the server's data or message; or
366    /// `ZeroError` when the server reported a mutation it couldn't process.
367    pub outcome: MutationOutcome,
368}
369
370/// What `mutate` returns: the outcome of the optimistic run on the client,
371/// and the server's.
372#[derive(Clone)]
373pub struct MutatorResult {
374    /// Settles when the mutator has run on the client: what it returned, or
375    /// what it raised.
376    pub client: Deferred<MutationOutcome>,
377    /// Settles when the server has run the mutation, or when the client can't
378    /// learn its result.
379    pub server: Deferred<MutationOutcome>,
380}
381
382/// A mutator bound to the app's context: runs it in a transaction.
383pub(crate) type MutatorFn =
384    Rc<dyn Fn(&Transaction, Option<&Value>) -> Result<Option<Value>, Error>>;
385
386/// A connection's readiness, as the pusher waits on it
387/// (`#connectionReadyResolver`).
388#[derive(Clone, Default)]
389struct ReadyPromise(Rc<RefCell<ReadyState>>);
390
391#[derive(Default)]
392struct ReadyState {
393    settled: Option<bool>,
394    /// Why the connection failed, once it has.
395    error: Option<ConnectionError>,
396    waiters: Vec<Box<dyn FnOnce(bool)>>,
397}
398
399impl ReadyPromise {
400    fn is(&self, other: &ReadyPromise) -> bool {
401        Rc::ptr_eq(&self.0, &other.0)
402    }
403
404    fn settle(&self, value: bool, event_loop: &EventLoop) {
405        let waiters = {
406            let mut s = self.0.borrow_mut();
407            if s.settled.is_some() {
408                return;
409            }
410            s.settled = Some(value);
411            std::mem::take(&mut s.waiters)
412        };
413        for w in waiters {
414            event_loop.microtask(move || w(value));
415        }
416    }
417
418    /// Calls `waiter` in a microtask once settled.
419    fn on_settle(&self, event_loop: &EventLoop, waiter: impl FnOnce(bool) + 'static) {
420        let settled = self.0.borrow().settled;
421        match settled {
422            Some(v) => event_loop.microtask(move || waiter(v)),
423            None => self.0.borrow_mut().waiters.push(Box::new(waiter)),
424        }
425    }
426}
427
428#[derive(Default)]
429struct ConnectAttempt {
430    events: RefCell<Vec<(f64, String)>>,
431    aborted: Cell<bool>,
432    abort_reason: RefCell<Option<ConnectionError>>,
433    timeout: RefCell<Option<Box<dyn Timer>>>,
434    finished: Cell<bool>,
435    on_connected: RefCell<Option<Box<dyn FnOnce()>>>,
436    /// When the socket was opened, for the open and `connected` logs.
437    start: Cell<f64>,
438}
439
440impl ConnectAttempt {
441    fn cancel_timeout(&self) {
442        if let Some(t) = self.timeout.borrow_mut().take() {
443            t.cancel();
444        }
445    }
446}
447
448/// A pending mutation's identity.
449type MutationKey = (String, u64);
450
451fn mutation_key(client_id: &str, id: f64) -> MutationKey {
452    (client_id.to_owned(), id.to_bits())
453}
454
455/// The replica as JS's last persist left it in the store: the snapshot it
456/// persisted (by cookie), its main map over that snapshot, the keys where
457/// the two may differ, and the pending mutations it ran.
458#[derive(Default)]
459struct PersistedState {
460    cookie: Value,
461    main: KvMap,
462    touched: HashSet<String>,
463    mutations: HashSet<MutationKey>,
464}
465
466/// The engine's end of one socket.
467pub(crate) struct SocketHandle {
468    me: Weak<SocketHandle>,
469    engine: Weak<Engine>,
470    connection: RefCell<Option<Box<dyn SocketConnection>>>,
471    closed: Cell<bool>,
472}
473
474impl SocketHandle {
475    fn new(engine: Weak<Engine>) -> Rc<Self> {
476        Rc::new_cyclic(|me| SocketHandle {
477            me: me.clone(),
478            engine,
479            connection: RefCell::new(None),
480            closed: Cell::new(false),
481        })
482    }
483
484    fn send(&self, text: &str) {
485        if self.closed.get() {
486            return;
487        }
488        if let Some(c) = &*self.connection.borrow() {
489            c.send(text);
490        }
491    }
492
493    fn close(&self, code: Option<u16>) {
494        if self.closed.replace(true) {
495            return;
496        }
497        let connection = self.connection.borrow_mut().take();
498        if let Some(c) = connection {
499            c.close(code);
500        }
501    }
502}
503
504impl SocketEvents for SocketHandle {
505    fn did_open(&self) {
506        if self.closed.get() {
507            return;
508        }
509        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
510            e.socket_did_open(&me);
511        }
512    }
513
514    fn did_receive(&self, message: SocketMessage) {
515        if self.closed.get() {
516            return;
517        }
518        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
519            e.socket_did_receive(&me, message);
520        }
521    }
522
523    fn did_close(&self, code: u16, reason: &str, was_clean: bool) {
524        if self.closed.get() {
525            return;
526        }
527        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
528            e.socket_did_close(&me, code, reason, was_clean);
529        }
530    }
531}
532
533type VisibilityWaiter = (bool, u64, Box<dyn FnOnce()>);
534
535/// What an inspector request hears: zero-cache's answer, the `inspect`
536/// body, or why there is none.
537pub(crate) type InspectAnswer = Box<dyn FnOnce(Result<Object, Error>)>;
538
539/// An inspector request (`rpcNoAuthTry` in `lazy-inspector.ts`), waiting
540/// for the connection or, once sent, for the answer with its ID.
541struct InspectRequest {
542    id: String,
543    /// What to send, until it is.
544    body: Option<Object>,
545    sent: bool,
546    /// Waiting for the next connection attempt: the last one's socket
547    /// closed before the request went out.
548    parked: bool,
549    done: InspectAnswer,
550}
551
552pub(crate) struct Engine {
553    me: Weak<Engine>,
554    pub(crate) schema: Arc<Schema>,
555    pub(crate) options: ZeroOptions,
556    pub(crate) client_id: String,
557    pub(crate) client_group_id: String,
558    profile_id: String,
559    pub(crate) user_id: Option<String>,
560    pub(crate) event_loop: Rc<EventLoop>,
561    scheduler: Rc<dyn Scheduler>,
562    pub(crate) log: Log,
563    socket_factory: Option<Rc<dyn SocketFactory>>,
564    socket_origin: Option<String>,
565    mutators: Rc<HashMap<String, MutatorFn>>,
566    client_schema: crate::protocol::ClientSchema,
567
568    pub(crate) store: RefCell<ReplicaStore>,
569    pub(crate) database: Rc<ReplicaDatabase>,
570    lifecycle_timers: RefCell<Vec<Box<dyn Timer>>>,
571    ivm_main: Rc<IvmBranch>,
572    pub(crate) context: Rc<QueryContext>,
573    pub(crate) connection_manager: Rc<ConnectionManager>,
574    mutation_tracker: Rc<MutationTracker>,
575    pub(crate) query_manager: Rc<QueryManager>,
576    poke_handler: RefCell<PokeHandler>,
577
578    /// The current token: `options.auth`, then the last one `connect`
579    /// set; empty for none.
580    pub(crate) auth: RefCell<String>,
581    rep_closed: Cell<bool>,
582    storage_failure_reported: Cell<bool>,
583    socket: RefCell<Option<Rc<SocketHandle>>>,
584    connect_cookie: RefCell<Value>,
585    init_connection_queries: RefCell<Option<OrderedMap<UpQueriesPatchOp>>>,
586    /// Deleted clients still to tell the server about on this connection.
587    deleted_clients: RefCell<Option<DeleteClientsBody>>,
588    last_mutation_id_received: Cell<f64>,
589    last_mutation_ids_sent: RefCell<HashMap<String, f64>>,
590    persisted: RefCell<PersistedState>,
591    persist_timer: RefCell<Option<Box<dyn Timer>>>,
592    last_persist_at: Cell<Option<f64>>,
593    additional_connect_params: RefCell<Option<Vec<(String, String)>>>,
594    /// When the current connection got the server's `connected`.
595    connected_at: Cell<Option<f64>>,
596    /// Consecutive transport failures, which set the reconnect backoff.
597    transport_failures: Cell<u32>,
598    current_attempt: RefCell<Option<Rc<ConnectAttempt>>>,
599    connection_ready: RefCell<ReadyPromise>,
600    recent_sent_messages: RefCell<VecDeque<Value>>,
601    mutation_rejection: RefCell<Option<ConnectionError>>,
602    client_group_disabled: Cell<bool>,
603
604    // The run loop's current wait. Each wait gets a new token, so a wait
605    // that was superseded does nothing when it ends.
606    loop_token: Cell<u64>,
607    abort_ping_timeout: RefCell<Rc<dyn Fn()>>,
608    on_pong: RefCell<Rc<dyn Fn()>>,
609    /// Pings the server now, when the run loop waits while connected.
610    ping_now: RefCell<Rc<dyn Fn()>>,
611    /// The run loop's backoff before it reconnects.
612    backoff: RefCell<Option<(u64, Box<dyn Timer>)>>,
613
614    // The push loop (Replicache's push `ConnectionLoop`).
615    push_timer: RefCell<Option<Box<dyn Timer>>>,
616    push_in_flight: Cell<bool>,
617    push_requested: Cell<bool>,
618    push_request_counter: Cell<u64>,
619
620    /// Trails the app's visibility by `hidden_disconnect_delay_ms` when it
621    /// is hidden (`DocumentVisibilityWatcher`).
622    pub(crate) visible: Cell<bool>,
623    hidden_timer: RefCell<Option<Box<dyn Timer>>>,
624    visibility_waiters: RefCell<Vec<VisibilityWaiter>>,
625    pending_waiters: RefCell<Vec<Box<dyn FnOnce()>>>,
626
627    /// Set while a mutator runs, for a mutation or a rebase: Replicache's
628    /// write lock. Mutations that start meanwhile wait, in order.
629    write_locked: Cell<bool>,
630    waiting_for_write_lock: RefCell<VecDeque<Box<dyn FnOnce()>>>,
631
632    /// What `Zero` does when the client can't sync any longer: by default
633    /// `ZeroOptions::on_client_state_not_found`.
634    pub(crate) on_client_state_not_found: RefCell<Option<Rc<dyn Fn()>>>,
635    /// What `Zero` does when the app must be updated to sync, besides
636    /// calling `ZeroOptions::on_update_needed`.
637    pub(crate) on_update_needed: RefCell<Option<UpdateNeededHandler>>,
638    /// The inspector's requests, oldest first.
639    inspect_requests: RefCell<Vec<InspectRequest>>,
640}
641
642fn noop() -> Rc<dyn Fn()> {
643    Rc::new(|| {})
644}
645
646/// `JSON.stringify(recentSentMessages, null, 2)`.
647fn pretty_recent_messages(messages: &VecDeque<Value>) -> String {
648    let entries: Vec<String> = messages
649        .iter()
650        .map(|m| {
651            let fields: Vec<String> = ["type", "size", "snippet"]
652                .iter()
653                .filter_map(|k| {
654                    m.get(k).map(|v| {
655                        format!(
656                            "    {}: {}",
657                            json::stringify(&Value::from(*k)),
658                            json::stringify(v)
659                        )
660                    })
661                })
662                .collect();
663            format!("  {{\n{}\n  }}", fields.join(",\n"))
664        })
665        .collect();
666    format!("[\n{}\n]", entries.join(",\n"))
667}
668
669fn bad_order_message(name: &str, received: &str, last: &str) -> String {
670    format!(
671        "Received {name} {received} is < than last snapshot {name} {last}; ignoring client view"
672    )
673}
674
675/// The synced aggregate rows `write` wrote, with their values now.
676fn aggregate_writes(write: &KvWrite) -> AggregateWrites {
677    let mut writes: AggregateWrites = write
678        .writes
679        .iter()
680        .filter(|k| k.starts_with(AGGREGATES_PREFIX))
681        .map(|k| (k.clone(), write.get(k).cloned()))
682        .collect();
683    writes.sort_by(|a, b| a.0.cmp(&b.0));
684    writes
685}
686
687impl Engine {
688    pub(crate) fn new(
689        schema: Arc<Schema>,
690        options: ZeroOptions,
691        mutators: Rc<HashMap<String, MutatorFn>>,
692        base: Rc<dyn Scheduler>,
693        socket_factory: Option<Rc<dyn SocketFactory>>,
694        visible: bool,
695    ) -> Result<Rc<Engine>, Error> {
696        if options.user_id.as_deref() == Some("") {
697            return Err(ConnectionError::client(
698                ClientErrorKind::Internal,
699                "ZeroOptions.userID should not be empty. Omit it entirely for logged-out clients.",
700            )
701            .into());
702        }
703        Self::check_auth_valid(options.user_id.as_deref(), options.auth.as_deref())?;
704        let socket_origin = options
705            .cache_url
706            .as_deref()
707            .map(Self::socket_origin)
708            .transpose()?;
709        let event_loop = EventLoop::new(base.clone());
710        let scheduler: Rc<dyn Scheduler> = Rc::new(LoopScheduler {
711            base: base.clone(),
712            event_loop: Rc::downgrade(&event_loop),
713        });
714        let client_id = random_id();
715        let sink = options
716            .log_sink
717            .clone()
718            .unwrap_or_else(|| Rc::new(StderrLogSink));
719        let log = Log::new(options.log_level, sink).with("clientID", &client_id);
720        let client_schema = schema.client_schema();
721        let schema_hash = base36(xxhash32::h64(&json::stringify(
722            &client_schema.normalized().to_json(),
723        )));
724
725        // Open the database and join a client group (`initClientV6`).
726        let name = ReplicaDatabase::name(
727            options.user_id.as_deref(),
728            options.storage_key.as_deref().unwrap_or(""),
729            options.mutate_url.as_deref(),
730            options.query_url.as_deref(),
731            &schema_hash,
732        );
733        let mut mutator_names: Vec<String> = mutators.keys().cloned().collect();
734        mutator_names.sort();
735        let (mut database, mut open_failure, mut corrupt) =
736            ReplicaDatabase::open(&options.storage, &name);
737        let joined = match database.init_client(&client_id, &mutator_names, base.now()) {
738            // What the client group holds doesn't decode, so the client
739            // state is lost: drop the database and start a new one
740            // (`#handleInvalidRefCount`).
741            Err(LoadError::Corrupt(_)) => {
742                corrupt = true;
743                database.drop_database();
744                database.release();
745                (database, open_failure, _) = ReplicaDatabase::open(&options.storage, &name);
746                database.init_client(&client_id, &mutator_names, base.now())
747            }
748            r => r,
749        };
750        let (client_group_id, group) = match joined {
751            Ok(j) => j,
752            // The client group can't be read: run on memory, as when the
753            // open fails.
754            Err(e) => {
755                database.release();
756                database = ReplicaDatabase::fallback(&name, &options.storage);
757                open_failure = Some(match e {
758                    LoadError::Failure(f) => f,
759                    LoadError::Corrupt(m) => {
760                        StorageFailure::new(super::storage::StorageFailureKind::IoError, m)
761                    }
762                });
763                database
764                    .init_client(&client_id, &mutator_names, base.now())
765                    .expect("a memory database reads")
766            }
767        };
768        let profile_id = database.profile_id();
769        // Drop stored mutation IDs that aren't mutation IDs (a corrupt
770        // store) rather than trust them.
771        let is_mutation_id = |id: f64| id >= 0.0 && number::is_safe_integer(id);
772        let last_mutation_ids = group
773            .last_mutation_ids
774            .into_iter()
775            .filter(|(cid, lmid)| {
776                let ok = is_mutation_id(*lmid);
777                if !ok {
778                    log.error(|| {
779                        format!(
780                            "Dropping invalid stored last mutation ID of {cid}: {}",
781                            number::format(*lmid)
782                        )
783                    });
784                }
785                ok
786            })
787            .collect();
788        let mutations: Vec<StoredMutation> = group
789            .mutations
790            .into_iter()
791            .filter(|m| {
792                let ok = is_mutation_id(m.id);
793                if !ok {
794                    log.error(|| {
795                        format!(
796                            "Dropping stored mutation \"{}\" of {} with invalid ID {}",
797                            m.name,
798                            m.client_id,
799                            number::format(m.id)
800                        )
801                    });
802                }
803                ok
804            })
805            .collect();
806        let store = ReplicaStore::new(
807            Snapshot {
808                map: group.entries,
809                cookie: group.cookie,
810                last_mutation_ids,
811            },
812            mutations
813                .iter()
814                .map(|m| PendingMutation {
815                    id: m.id,
816                    client_id: m.client_id.clone(),
817                    name: m.name.clone(),
818                    args: m.args.clone(),
819                    timestamp: m.timestamp,
820                    writes: HashSet::new(),
821                })
822                .collect(),
823        );
824        let stored_aggregate_writes: HashMap<MutationKey, AggregateWrites> = mutations
825            .into_iter()
826            .map(|m| (mutation_key(&m.client_id, m.id), m.aggregate_writes))
827            .collect();
828        let ivm_main = IvmBranch::new(schema.clone());
829        let connection_manager = ConnectionManager::new(
830            options.disconnect_timeout_ms,
831            scheduler.clone(),
832            Rc::downgrade(&event_loop),
833        );
834
835        let engine = Rc::new_cyclic(|me: &Weak<Engine>| {
836            let mutation_tracker = {
837                let (a, f, q) = (me.clone(), me.clone(), me.clone());
838                Rc::new(MutationTracker::new(
839                    client_id.clone(),
840                    Box::new(move |id| {
841                        if let Some(e) = a.upgrade() {
842                            e.send_if_connected(Upstream::AckMutationResponses(id));
843                        }
844                    }),
845                    Box::new(move |error| {
846                        if let Some(e) = f.upgrade() {
847                            e.disconnect(error, None);
848                        }
849                    }),
850                    Box::new(move || {
851                        if let Some(e) = q.upgrade() {
852                            e.query_manager.on_all_mutations_applied();
853                        }
854                    }),
855                ))
856            };
857            let query_manager = {
858                let (s, p) = (me.clone(), me.clone());
859                QueryManager::new(
860                    client_id.clone(),
861                    scheduler.clone(),
862                    options.query_change_throttle_ms,
863                    Box::new(move |mut body| {
864                        if let Some(e) = s.upgrade() {
865                            body.traceparent = e.traceparent();
866                            e.send_if_connected(Upstream::ChangeDesiredQueries(body));
867                        }
868                    }),
869                    Box::new(move || p.upgrade().map_or(0, |e| e.mutation_tracker.size())),
870                    log.clone(),
871                    options.slow_materialize_threshold_ms,
872                )
873            };
874            let context = {
875                let (a, u, c) = (me.clone(), me.clone(), me.clone());
876                let l = event_loop.clone();
877                QueryContext::new(
878                    ivm_main.clone(),
879                    Box::new(move |id, ast, ttl, got| match a.upgrade() {
880                        Some(e) => {
881                            e.query_manager
882                                .add_custom(ast, &id.name, &id.args, ttl, Some(got))
883                        }
884                        None => Box::new(|| {}),
885                    }),
886                    Rc::new(move |id, ttl| {
887                        if let Some(e) = u.upgrade() {
888                            e.query_manager.update_custom(&id.name, &id.args, ttl);
889                        }
890                    }),
891                    Rc::new(move |f| l.microtask(f)),
892                    Some(log.clone()),
893                    Box::new(move || match c.upgrade()?.connection_manager.state() {
894                        State::Closed(r) => Some(r),
895                        _ => None,
896                    }),
897                    true,
898                    options.batch_view_updates.clone(),
899                    Some(query_manager.clone()),
900                )
901            };
902            Engine {
903                me: me.clone(),
904                poke_handler: RefCell::new(PokeHandler::new(client_id.clone(), schema.clone())),
905                schema: schema.clone(),
906                user_id: options.user_id.clone(),
907                // Set now, as JS's constructor does, so a token from
908                // `connect` right after init isn't replaced when the run
909                // loop starts.
910                auth: RefCell::new(options.auth.clone().unwrap_or_default()),
911                options: options.clone(),
912                client_id: client_id.clone(),
913                client_group_id: client_group_id.clone(),
914                profile_id,
915                event_loop: event_loop.clone(),
916                scheduler: scheduler.clone(),
917                log: log.clone(),
918                socket_factory,
919                socket_origin,
920                mutators,
921                client_schema,
922                store: RefCell::new(store),
923                database: database.clone(),
924                lifecycle_timers: RefCell::default(),
925                ivm_main: ivm_main.clone(),
926                context,
927                connection_manager: connection_manager.clone(),
928                mutation_tracker,
929                query_manager,
930                rep_closed: Cell::new(false),
931                storage_failure_reported: Cell::new(false),
932                socket: RefCell::new(None),
933                connect_cookie: RefCell::new(Value::Null),
934                init_connection_queries: RefCell::new(None),
935                deleted_clients: RefCell::new(None),
936                last_mutation_id_received: Cell::new(0.0),
937                last_mutation_ids_sent: RefCell::default(),
938                persisted: RefCell::default(),
939                persist_timer: RefCell::new(None),
940                last_persist_at: Cell::new(None),
941                additional_connect_params: RefCell::new(None),
942                connected_at: Cell::new(None),
943                transport_failures: Cell::new(0),
944                current_attempt: RefCell::new(None),
945                connection_ready: RefCell::default(),
946                recent_sent_messages: RefCell::default(),
947                mutation_rejection: RefCell::new(None),
948                client_group_disabled: Cell::new(false),
949                loop_token: Cell::new(0),
950                abort_ping_timeout: RefCell::new(noop()),
951                on_pong: RefCell::new(noop()),
952                ping_now: RefCell::new(noop()),
953                backoff: RefCell::new(None),
954                push_timer: RefCell::new(None),
955                push_in_flight: Cell::new(false),
956                push_requested: Cell::new(false),
957                push_request_counter: Cell::new(0),
958                visible: Cell::new(visible),
959                hidden_timer: RefCell::new(None),
960                visibility_waiters: RefCell::default(),
961                pending_waiters: RefCell::default(),
962                write_locked: Cell::new(false),
963                waiting_for_write_lock: RefCell::default(),
964                on_client_state_not_found: RefCell::new(None),
965                on_update_needed: RefCell::new(None),
966                inspect_requests: RefCell::default(),
967            }
968        });
969
970        let e = &engine;
971        {
972            let w = Rc::downgrade(e);
973            // The closure is dropped with the manager; no need to keep the
974            // unsubscribe.
975            let _ = connection_manager.subscribe(move |state| {
976                if let (Some(e), State::Closed(reason)) = (w.upgrade(), state) {
977                    e.query_manager.handle_closed(reason);
978                }
979            });
980            let w = Rc::downgrade(e);
981            let _ = connection_manager.subscribe(move |state| {
982                if let Some(e) = w.upgrade() {
983                    e.on_connection_state_change(state);
984                }
985            });
986        }
987        e.on_connection_state_change(&connection_manager.state());
988
989        // Load the snapshot into the sources and replay the pending
990        // mutations on it, with the synced aggregate rows they wrote when
991        // they were last persisted, as JS's stored commits have them.
992        let initial = diff_all(&KvMap::default(), &e.store.borrow().snapshot.map);
993        e.ivm_main.apply(&initial, &HashMap::new());
994        let pending = e.store.borrow().pending.clone();
995        if !pending.is_empty() {
996            let snapshot_map = e.store.borrow().snapshot.map.clone();
997            let (main, replayed, touched, _) = e.replay(
998                &pending,
999                &snapshot_map,
1000                e.ivm_main.fork(),
1001                &stored_aggregate_writes,
1002                &mut |_, _| {},
1003            );
1004            let diff = {
1005                let mut store = e.store.borrow_mut();
1006                let snapshot = store.snapshot.clone();
1007                store.advance(snapshot, main, replayed, Some(&touched))
1008            };
1009            e.ivm_main.apply(&diff, &HashMap::new());
1010        }
1011        {
1012            let store = e.store.borrow();
1013            *e.persisted.borrow_mut() = PersistedState {
1014                cookie: store.snapshot.cookie.clone(),
1015                main: store.main.clone(),
1016                touched: store.pending_writes(),
1017                mutations: store
1018                    .pending
1019                    .iter()
1020                    .map(|m| mutation_key(&m.client_id, m.id))
1021                    .collect(),
1022            };
1023        }
1024        {
1025            let w = Rc::downgrade(e);
1026            database.add_failure_listener(
1027                &e.client_id,
1028                Rc::new(move |f| {
1029                    if let Some(e) = w.upgrade() {
1030                        e.storage_failed(f, false);
1031                    }
1032                }),
1033            );
1034        }
1035        // Another client of the group stopped (the `onDelete` of
1036        // `makeActiveClientsManager`): tell the server, if connected.
1037        {
1038            let w = Rc::downgrade(e);
1039            database.client_started(
1040                &e.client_id,
1041                &e.client_group_id,
1042                Rc::new(move |id| {
1043                    if let Some(e) = w.upgrade() {
1044                        let id = id.to_owned();
1045                        e.event_loop.clone().dispatch(|| {
1046                            e.send_if_connected(Upstream::DeleteClients(DeleteClientsBody {
1047                                client_ids: Some(vec![id]),
1048                                client_group_ids: None,
1049                            }))
1050                        });
1051                    }
1052                }),
1053            );
1054        }
1055        if corrupt {
1056            // This client already starts on the new database, which is what
1057            // replacing it would do; the app's callback still hears it, as
1058            // in JS.
1059            e.log.error(|| {
1060                format!("Client state is corrupt; dropped database {name} and started a new one")
1061            });
1062            // Only the app's callback: this client is already on a new
1063            // database, so it isn't replaced (as in Swift).
1064            e.micro(|e| {
1065                if let Some(f) = &e.options.on_client_state_not_found {
1066                    f();
1067                }
1068            });
1069        }
1070        if let Some(f) = open_failure {
1071            e.log.warn(|| {
1072                format!(
1073                    "Local store storage failed ({}) opening {name}; running this instance in memory: {}",
1074                    f.kind.as_str(),
1075                    f.message
1076                )
1077            });
1078            e.micro(move |e| e.storage_failed(&f, true));
1079        }
1080        e.start_lifecycle();
1081
1082        // The replica is loaded: report its got queries and mutation
1083        // results, as the first runs of the store watches do, then build
1084        // the pipelines.
1085        let main = e.store.borrow().main.clone();
1086        e.fire_watchers(&diff_all(&KvMap::default(), &main), true);
1087        e.context.mark_pipelines_ready();
1088        e.micro(|e| e.start_run_loop());
1089        Ok(engine)
1090    }
1091
1092    fn check_auth_valid(user_id: Option<&str>, auth: Option<&str>) -> Result<(), Error> {
1093        if user_id.is_none() && auth.is_some_and(|a| !a.is_empty()) {
1094            return Err(ConnectionError::client(
1095                ClientErrorKind::Internal,
1096                "ZeroOptions.userID is required when auth is set.",
1097            )
1098            .into());
1099        }
1100        Ok(())
1101    }
1102
1103    /// The WebSocket origin of the zero-cache URL (`getServer`,
1104    /// `toWSString`).
1105    fn socket_origin(url: &str) -> Result<String, Error> {
1106        let (scheme, rest) = if let Some(rest) = url.strip_prefix("http://") {
1107            ("ws://", rest)
1108        } else if let Some(rest) = url.strip_prefix("https://") {
1109            ("wss://", rest)
1110        } else {
1111            return Err(ConnectionError::client(
1112                ClientErrorKind::Internal,
1113                "ZeroOptions.cacheURL must use the \"http\" or \"https\" scheme.",
1114            )
1115            .into());
1116        };
1117        let mut rest = rest.to_owned();
1118        if !rest.contains('/') {
1119            rest.push('/');
1120        }
1121        Ok(format!("{scheme}{rest}"))
1122    }
1123
1124    fn micro(&self, f: impl FnOnce(&Engine) + 'static) {
1125        let w = self.me.clone();
1126        self.event_loop.microtask(move || {
1127            if let Some(e) = w.upgrade() {
1128                f(&e);
1129            }
1130        });
1131    }
1132
1133    fn later(&self, ms: f64, f: impl FnOnce(&Engine) + 'static) -> Box<dyn Timer> {
1134        let w = self.me.clone();
1135        self.scheduler.schedule(
1136            ms,
1137            Box::new(move || {
1138                if let Some(e) = w.upgrade() {
1139                    f(&e);
1140                }
1141            }),
1142        )
1143    }
1144
1145    fn status_is(&self, status: ConnectionStatus) -> bool {
1146        self.connection_manager.is(status)
1147    }
1148
1149    // MARK: Sending
1150
1151    /// The app's traceparent for a message about to be sent.
1152    fn traceparent(&self) -> Option<String> {
1153        self.options.get_traceparent.as_ref().and_then(|f| f())
1154    }
1155
1156    fn send_if_connected(&self, message: Upstream) {
1157        if self.status_is(ConnectionStatus::Connected) {
1158            self.send(message);
1159        }
1160    }
1161
1162    fn send(&self, message: Upstream) {
1163        let Some(socket) = self.socket.borrow().clone() else {
1164            return;
1165        };
1166        let text = message.to_text();
1167        if message.tag() != "ping" {
1168            let units: Vec<u16> = text.encode_utf16().collect();
1169            let snippet = if units.len() > 200 {
1170                String::from_utf16(&units[..200]).unwrap_or_else(|_| text.clone())
1171            } else {
1172                text.clone()
1173            };
1174            let mut o = Object::new();
1175            o.insert("type", Value::from(message.tag()));
1176            o.insert("size", Value::Number(units.len() as f64));
1177            o.insert("snippet", Value::String(snippet));
1178            let mut recent = self.recent_sent_messages.borrow_mut();
1179            if recent.len() >= 5 {
1180                recent.pop_front();
1181            }
1182            recent.push_back(Value::Object(o));
1183        }
1184        socket.send(&text);
1185    }
1186
1187    // MARK: The run loop
1188
1189    fn start_run_loop(&self) {
1190        if self.socket_origin.is_none() {
1191            self.connection_manager
1192                .disconnected(ConnectionError::client(
1193                    ClientErrorKind::NoSocketOrigin,
1194                    "No server socket origin provided",
1195                ));
1196            return;
1197        }
1198        self.connection_manager.initialized();
1199        self.iterate();
1200    }
1201
1202    fn next_loop_token(&self) -> u64 {
1203        let t = self.loop_token.get() + 1;
1204        self.loop_token.set(t);
1205        t
1206    }
1207
1208    fn is_token(&self, token: u64) -> bool {
1209        self.loop_token.get() == token
1210    }
1211
1212    /// One turn of the JS run loop: acts on the connection status.
1213    fn iterate(&self) {
1214        if !self.connection_manager.should_continue_run_loop() {
1215            return;
1216        }
1217        let token = self.next_loop_token();
1218        match self.connection_manager.status() {
1219            ConnectionStatus::Connecting | ConnectionStatus::Disconnected => {
1220                if !self.visible.get() {
1221                    // Wait for the app to come back, or for a state change.
1222                    self.on_visibility(true, token, |e| e.begin_attempt());
1223                    self.on_state_change(token, |e, state| e.resume_after_state_change(state));
1224                    return;
1225                }
1226                // The app is visible, which wins the race with a state
1227                // change.
1228                self.micro(move |e| {
1229                    if e.is_token(token) {
1230                        e.begin_attempt();
1231                    }
1232                });
1233            }
1234            ConnectionStatus::Connected => self.wait_while_connected(token),
1235            ConnectionStatus::NeedsAuth | ConnectionStatus::Error => {
1236                let status = self.connection_manager.status();
1237                self.log.info(|| {
1238                    format!(
1239                        "Run loop paused in {} state. Call zero.connection.connect() to resume.",
1240                        status.as_str()
1241                    )
1242                });
1243                self.wait_for_connect_request(token);
1244            }
1245            ConnectionStatus::Closed => {}
1246            ConnectionStatus::Initializing => unreachable!("The run loop runs once initialized"),
1247        }
1248    }
1249
1250    /// Ends the current wait of the run loop with `error` (the `catch` of
1251    /// the JS run loop).
1252    fn loop_failed(&self, error: ConnectionError) {
1253        let (transition, reason) = ConnectionTransition::of(&error);
1254        if !self.status_is(ConnectionStatus::Connected)
1255            && error.client_kind() != Some(ClientErrorKind::ClientClosed)
1256        {
1257            // Only errors that stop the run loop are errors. Reconnects and
1258            // auth failures are routine.
1259            let mut level = match transition {
1260                ConnectionTransition::Error => LogLevel::Error,
1261                ConnectionTransition::Disconnected | ConnectionTransition::Closed => LogLevel::Info,
1262                ConnectionTransition::None | ConnectionTransition::NeedsAuth => LogLevel::Warn,
1263            };
1264            if error.server_kind() == Some(ErrorKind::ClientNotFound) {
1265                level = LogLevel::Warn;
1266            }
1267            self.log
1268                .log(level, || format!("Failed to connect: {}", error.message()));
1269        }
1270        let mut backoff_ms = RUN_LOOP_INTERVAL_MS;
1271        let mut sleep_ms = None;
1272        match transition {
1273            ConnectionTransition::None => {
1274                // A transport failure is retried soon, then less often.
1275                // Errors the server sent keep the full interval, adjusted by
1276                // any backoff they carry.
1277                if reason.is_transport_failure() {
1278                    let failures = self.transport_failures.get() + 1;
1279                    self.transport_failures.set(failures);
1280                    backoff_ms = reconnect_backoff_ms(failures, RUN_LOOP_INTERVAL_MS, random_f64());
1281                }
1282                if let Some(ErrorBody::Backoff {
1283                    min_backoff_ms,
1284                    max_backoff_ms,
1285                    reconnect_params,
1286                    ..
1287                }) = reason.backoff()
1288                {
1289                    if let Some(min) = min_backoff_ms {
1290                        backoff_ms = backoff_ms.max(*min);
1291                    }
1292                    if let Some(max) = max_backoff_ms {
1293                        backoff_ms = backoff_ms.min(*max);
1294                    }
1295                    *self.additional_connect_params.borrow_mut() =
1296                        reconnect_params.as_ref().map(|params| {
1297                            let mut p = params.clone();
1298                            p.sort_by(|a, b| a.0.cmp(&b.0));
1299                            p
1300                        });
1301                }
1302                sleep_ms = Some(backoff_ms);
1303            }
1304            ConnectionTransition::NeedsAuth => self.connection_manager.needs_auth(reason),
1305            ConnectionTransition::Error => self.connection_manager.error(reason),
1306            ConnectionTransition::Disconnected => self.connection_manager.disconnected(reason),
1307            ConnectionTransition::Closed => {}
1308        }
1309        let token = self.next_loop_token();
1310        match sleep_ms {
1311            Some(ms) if ms > 0.0 => {
1312                let timer = self.later(ms, move |e| {
1313                    if e.is_token(token) {
1314                        e.backoff.borrow_mut().take();
1315                        e.iterate();
1316                    }
1317                });
1318                *self.backoff.borrow_mut() = Some((token, timer));
1319            }
1320            _ => self.micro(move |e| {
1321                if e.is_token(token) {
1322                    e.iterate();
1323                }
1324            }),
1325        }
1326    }
1327
1328    /// `throwIfConnectionError`: whether a state the run loop wakes up to
1329    /// stops it. Dropped sockets and timeouts just reconnect.
1330    fn connection_error(state: &State) -> Option<ConnectionError> {
1331        match state {
1332            State::NeedsAuth(r) | State::Error(r) | State::Closed(r) => Some(r.clone()),
1333            State::Connecting {
1334                reason: Some(r), ..
1335            }
1336            | State::Disconnected(r) => match r.client_kind() {
1337                Some(
1338                    ClientErrorKind::ConnectTimeout
1339                    | ClientErrorKind::AbruptClose
1340                    | ClientErrorKind::CleanClose
1341                    | ClientErrorKind::Hidden,
1342                ) => None,
1343                _ => Some(r.clone()),
1344            },
1345            _ => None,
1346        }
1347    }
1348
1349    /// Wakes the loop on the next state change, unless the wait is over.
1350    fn on_state_change(&self, token: u64, body: impl FnOnce(&Engine, State) + 'static) {
1351        let (a, b) = (self.me.clone(), self.me.clone());
1352        self.connection_manager.on_next_state_change(
1353            move || a.upgrade().is_some_and(|e| e.is_token(token)),
1354            move |state| {
1355                if let Some(e) = b.upgrade()
1356                    && e.is_token(token)
1357                {
1358                    body(&e, state);
1359                }
1360            },
1361        );
1362    }
1363
1364    fn resume_after_state_change(&self, state: State) {
1365        match Self::connection_error(&state) {
1366            Some(error) => self.loop_failed(error),
1367            None => self.iterate(),
1368        }
1369    }
1370
1371    /// Connected: ping after `ping_timeout_ms` without poke traffic.
1372    fn wait_while_connected(&self, token: u64) {
1373        let ready = self.connection_ready.borrow().clone();
1374        let timer: Rc<RefCell<Option<Box<dyn Timer>>>> = Rc::default();
1375        let cancel = {
1376            let timer = timer.clone();
1377            move || {
1378                if let Some(t) = timer.borrow_mut().take() {
1379                    t.cancel();
1380                }
1381            }
1382        };
1383        let end: Rc<dyn Fn()> = {
1384            let w = self.me.clone();
1385            Rc::new(move || {
1386                let Some(e) = w.upgrade() else { return };
1387                // The `finally` of the connected case.
1388                if !e.status_is(ConnectionStatus::Connected)
1389                    && e.connection_ready.borrow().is(&ready)
1390                {
1391                    *e.connection_ready.borrow_mut() = ReadyPromise::default();
1392                }
1393            })
1394        };
1395        {
1396            let w = self.me.clone();
1397            let cancel = cancel.clone();
1398            let end = end.clone();
1399            *self.abort_ping_timeout.borrow_mut() = Rc::new(move || {
1400                cancel();
1401                let end = end.clone();
1402                if let Some(e) = w.upgrade() {
1403                    e.micro(move |e| {
1404                        if e.is_token(token) {
1405                            end();
1406                            e.iterate();
1407                        }
1408                    });
1409                }
1410            });
1411        }
1412        {
1413            let end = end.clone();
1414            let t = self.later(self.options.ping_timeout_ms, move |e| {
1415                if e.is_token(token) {
1416                    e.ping(end);
1417                }
1418            });
1419            *timer.borrow_mut() = Some(t);
1420        }
1421        {
1422            let w = self.me.clone();
1423            let cancel = cancel.clone();
1424            let end = end.clone();
1425            *self.ping_now.borrow_mut() = Rc::new(move || {
1426                if let Some(e) = w.upgrade()
1427                    && e.is_token(token)
1428                {
1429                    cancel();
1430                    e.ping(end.clone());
1431                }
1432            });
1433        }
1434        {
1435            let cancel = cancel.clone();
1436            let end = end.clone();
1437            self.on_visibility(false, token, move |e| {
1438                cancel();
1439                e.disconnect(
1440                    ConnectionError::client(
1441                        ClientErrorKind::Hidden,
1442                        "Connection closed because tab was hidden",
1443                    ),
1444                    None,
1445                );
1446                end();
1447                e.iterate();
1448            });
1449        }
1450        self.on_state_change(token, move |e, state| {
1451            end();
1452            e.resume_after_state_change(state);
1453        });
1454    }
1455
1456    fn ping(&self, end: Rc<dyn Fn()>) {
1457        let token = self.next_loop_token();
1458        let t0 = self.scheduler.monotonic_now();
1459        // Before the send, as JS sets `#onPong` first: a transport may
1460        // deliver the pong from inside `send`.
1461        {
1462            let w = self.me.clone();
1463            let end = end.clone();
1464            *self.on_pong.borrow_mut() = Rc::new(move || {
1465                let end = end.clone();
1466                if let Some(e) = w.upgrade() {
1467                    e.micro(move |e| {
1468                        if !e.is_token(token) {
1469                            return;
1470                        }
1471                        let ms = e.scheduler.monotonic_now() - t0;
1472                        e.log
1473                            .debug(|| format!("ping succeeded in {} ms", number::format(ms)));
1474                        end();
1475                        e.iterate();
1476                    });
1477                }
1478            });
1479        }
1480        self.send(Upstream::Ping);
1481        {
1482            let end = end.clone();
1483            // The timer ends with the token; it needn't be cancelled.
1484            let _ = self.later(self.options.ping_timeout_ms, move |e| {
1485                if !e.is_token(token) {
1486                    return;
1487                }
1488                let ms = e.scheduler.monotonic_now() - t0;
1489                e.log
1490                    .info(|| format!("ping failed in {} ms - disconnecting", number::format(ms)));
1491                let error = ConnectionError::client(
1492                    ClientErrorKind::PingTimeout,
1493                    "Server ping request failed",
1494                );
1495                e.disconnect(error.clone(), None);
1496                end();
1497                e.loop_failed(error);
1498            });
1499        }
1500        self.on_state_change(token, move |e, state| {
1501            end();
1502            e.resume_after_state_change(state);
1503        });
1504    }
1505
1506    /// needs-auth and error: wait for `connect()`.
1507    fn wait_for_connect_request(&self, token: u64) {
1508        let w = self.me.clone();
1509        self.connection_manager.on_connect_request(move || {
1510            if let Some(e) = w.upgrade()
1511                && e.is_token(token)
1512            {
1513                e.connection_manager.resume_from_connect_request();
1514                e.iterate();
1515            }
1516        });
1517        self.on_state_change(token, |e, _| e.iterate());
1518    }
1519
1520    // MARK: The app's lifecycle
1521
1522    /// The app was shown, or hidden. Hidden, the client disconnects and
1523    /// waits to be shown again.
1524    pub(crate) fn set_visible(&self, visible: bool) {
1525        if let Some(t) = self.hidden_timer.borrow_mut().take() {
1526            t.cancel();
1527        }
1528        if visible {
1529            self.set_visibility_state(true);
1530        } else {
1531            let t = self.later(self.options.hidden_disconnect_delay_ms, |e| {
1532                e.hidden_timer.borrow_mut().take();
1533                e.set_visibility_state(false);
1534            });
1535            *self.hidden_timer.borrow_mut() = Some(t);
1536        }
1537    }
1538
1539    fn set_visibility_state(&self, visible: bool) {
1540        if visible == self.visible.get() {
1541            return;
1542        }
1543        self.visible.set(visible);
1544        let ready: Vec<VisibilityWaiter> = {
1545            let mut waiters = self.visibility_waiters.borrow_mut();
1546            let (ready, rest) = std::mem::take(&mut *waiters)
1547                .into_iter()
1548                .partition(|(v, _, _)| *v == visible);
1549            *waiters = rest;
1550            ready
1551        };
1552        for (_, _, w) in ready {
1553            w();
1554        }
1555    }
1556
1557    /// Calls `body` in a microtask once the app's visibility is `visible`,
1558    /// unless the run loop's wait `token` is over by then.
1559    fn on_visibility(&self, visible: bool, token: u64, body: impl FnOnce(&Engine) + 'static) {
1560        let w = self.me.clone();
1561        let waiter: Box<dyn FnOnce()> = Box::new(move || {
1562            if let Some(e) = w.upgrade() {
1563                e.micro(move |e| {
1564                    if !e.is_token(token) {
1565                        return;
1566                    }
1567                    // The wait is over: a state change no longer ends it.
1568                    e.next_loop_token();
1569                    body(e);
1570                });
1571            }
1572        });
1573        if self.visible.get() == visible {
1574            waiter();
1575        } else {
1576            let current = self.loop_token.get();
1577            let mut waiters = self.visibility_waiters.borrow_mut();
1578            waiters.retain(|(_, t, _)| *t == current);
1579            waiters.push((visible, token, waiter));
1580        }
1581    }
1582
1583    /// The network may be back: retry now instead of after the backoff,
1584    /// and check that an open connection still works.
1585    pub(crate) fn reconnect_now(&self) {
1586        let backoff = {
1587            let mut b = self.backoff.borrow_mut();
1588            match &*b {
1589                Some((token, _)) if *token == self.loop_token.get() => b.take(),
1590                _ => None,
1591            }
1592        };
1593        if let Some((_, timer)) = backoff {
1594            timer.cancel();
1595            self.iterate();
1596        } else if self.status_is(ConnectionStatus::Connected) {
1597            let ping_now = self.ping_now.borrow().clone();
1598            ping_now();
1599        }
1600    }
1601
1602    fn pending_settled(&self) -> bool {
1603        self.store.borrow().pending.is_empty()
1604            || !self.status_is(ConnectionStatus::Connected)
1605            || self.client_group_disabled.get()
1606            || self.rep_closed.get()
1607    }
1608
1609    /// Calls `done` once the server has confirmed every pending mutation,
1610    /// or right away when it can't: the client isn't connected, or the
1611    /// group is disabled.
1612    pub(crate) fn when_pending_settled(&self, done: impl FnOnce() + 'static) {
1613        if self.pending_settled() {
1614            done();
1615        } else {
1616            self.pending_waiters.borrow_mut().push(Box::new(done));
1617        }
1618    }
1619
1620    /// Reports the server's outcome of each mutation this engine makes to
1621    /// `listeners`, which `Zero` keeps (`onMutationOutcome`).
1622    pub(crate) fn set_mutation_outcome_listeners(&self, listeners: Rc<OutcomeListeners>) {
1623        self.mutation_tracker.set_outcome_listeners(listeners);
1624    }
1625
1626    fn settle_pending_waiters(&self) {
1627        if self.pending_waiters.borrow().is_empty() || !self.pending_settled() {
1628            return;
1629        }
1630        let waiters = std::mem::take(&mut *self.pending_waiters.borrow_mut());
1631        for w in waiters {
1632            w();
1633        }
1634    }
1635
1636    // MARK: Connecting
1637
1638    fn add_connect_event(&self, attempt: &ConnectAttempt, event: &str) {
1639        attempt
1640            .events
1641            .borrow_mut()
1642            .push((self.scheduler.now(), event.to_owned()));
1643    }
1644
1645    fn arm_connect_timeout(&self, attempt: &Rc<ConnectAttempt>, ack: bool) {
1646        attempt.cancel_timeout();
1647        if attempt.aborted.get() {
1648            return;
1649        }
1650        let weak = Rc::downgrade(attempt);
1651        let t = self.later(CONNECT_TIMEOUT_MS, move |e| {
1652            let Some(attempt) = weak.upgrade() else {
1653                return;
1654            };
1655            let events = Value::Array(
1656                attempt
1657                    .events
1658                    .borrow()
1659                    .iter()
1660                    .map(|(t, ev)| {
1661                        Value::Array(vec![
1662                            Value::String(iso_date_string(*t)),
1663                            Value::String(ev.clone()),
1664                        ])
1665                    })
1666                    .collect(),
1667            );
1668            e.log.debug(|| "Connection attempt timed out".into());
1669            let seconds = number::format(CONNECT_TIMEOUT_MS / 1000.0);
1670            let message = format!(
1671                "{}Connect events: {}",
1672                if ack {
1673                    format!("Server did not acknowledge the connection within {seconds} seconds. ")
1674                } else {
1675                    format!("Connection setup timed out after {seconds} seconds. ")
1676                },
1677                json::stringify(&events)
1678            );
1679            e.disconnect(
1680                ConnectionError::client(ClientErrorKind::ConnectTimeout, message),
1681                None,
1682            );
1683        });
1684        *attempt.timeout.borrow_mut() = Some(t);
1685    }
1686
1687    fn begin_attempt(&self) {
1688        assert!(
1689            self.current_attempt.borrow().is_none(),
1690            "Connection attempt already active"
1691        );
1692        let attempt = Rc::new(ConnectAttempt::default());
1693        *self.current_attempt.borrow_mut() = Some(attempt.clone());
1694        self.add_connect_event(&attempt, "starting the connection");
1695        self.arm_connect_timeout(&attempt, false);
1696        self.connect_attempt(&attempt);
1697    }
1698
1699    fn token(&self) -> Option<String> {
1700        let auth = self.auth.borrow();
1701        (!auth.is_empty()).then(|| auth.clone())
1702    }
1703
1704    /// `#connectAttempt`: opens a socket and waits for `connected`.
1705    fn connect_attempt(&self, attempt: &Rc<ConnectAttempt>) {
1706        if self.rep_closed.get() {
1707            self.finish_attempt(attempt, None);
1708            return;
1709        }
1710        let (Some(socket_origin), Some(socket_factory)) =
1711            (self.socket_origin.clone(), self.socket_factory.clone())
1712        else {
1713            self.finish_attempt(
1714                attempt,
1715                Some(ConnectionError::client(
1716                    ClientErrorKind::Internal,
1717                    "No server provided",
1718                )),
1719            );
1720            return;
1721        };
1722        assert!(
1723            self.status_is(ConnectionStatus::Disconnected)
1724                || self.status_is(ConnectionStatus::Connecting),
1725            "connectAttempt() called from invalid state"
1726        );
1727        let wsid = random_id();
1728        self.log.info(|| "Connecting...".into());
1729        self.connection_manager.connecting(None);
1730        attempt.start.set(self.scheduler.monotonic_now());
1731        self.add_connect_event(attempt, "reading the cookie");
1732        let connect_cookie = self.store.borrow().cookie().clone();
1733        self.add_connect_event(attempt, "reading the client group ID");
1734        self.add_connect_event(attempt, "initializing active clients");
1735        if attempt.aborted.get() {
1736            let reason = attempt.abort_reason.borrow().clone();
1737            self.finish_attempt(attempt, reason);
1738            return;
1739        }
1740
1741        // `createSocket`.
1742        self.add_connect_event(attempt, "reading the profile ID");
1743        let url = ConnectionUrl {
1744            socket_origin,
1745            client_id: self.client_id.clone(),
1746            client_group_id: self.client_group_id.clone(),
1747            user_id: self.user_id.clone(),
1748            base_cookie: match &connect_cookie {
1749                Value::String(s) => Some(s.clone()),
1750                Value::Number(n) => Some(number::format(*n)),
1751                _ => None,
1752            },
1753            timestamp: self.scheduler.monotonic_now(),
1754            last_mutation_id: self.last_mutation_id_received.get(),
1755            wsid,
1756            profile_id: self.profile_id.clone(),
1757            // JS asks for the server's timings when it logs at debug level.
1758            debug_perf: self.options.log_level == LogLevel::Debug,
1759            additional_params: self
1760                .additional_connect_params
1761                .borrow()
1762                .clone()
1763                .unwrap_or_default(),
1764        }
1765        .to_string();
1766        self.add_connect_event(attempt, "reading deleted clients");
1767        let mut deleted = self.deleted_clients_body();
1768        self.add_connect_event(attempt, "reading desired queries");
1769        let mut queries_patch = Some(
1770            self.query_manager
1771                .queries_patch(&self.store.borrow().main, None),
1772        );
1773        let mut active_clients = vec![self.client_id.clone()];
1774        active_clients.extend(
1775            self.database
1776                .active_clients(&self.client_group_id)
1777                .into_iter()
1778                .filter(|c| *c != self.client_id),
1779        );
1780        let init_connection = InitConnectionBody {
1781            desired_queries_patch: queries_patch
1782                .as_ref()
1783                .map(|p| p.values().cloned().collect())
1784                .unwrap_or_default(),
1785            client_schema: connect_cookie.is_null().then(|| self.client_schema.clone()),
1786            deleted: deleted.clone(),
1787            user_push_url: self.options.mutate_url.clone(),
1788            user_push_headers: self.options.mutate_headers.clone(),
1789            user_query_url: self.options.query_url.clone(),
1790            user_query_headers: self.options.query_headers.clone(),
1791            active_clients: Some(active_clients),
1792            traceparent: None,
1793        };
1794        let auth_token = self.token();
1795        let mut sec_protocol = sec_protocols::encode(Some(&init_connection), auth_token.as_deref());
1796        if sec_protocol.len() > self.options.max_header_length {
1797            sec_protocol = sec_protocols::encode(None, auth_token.as_deref());
1798            queries_patch = None;
1799            if sec_protocol.len() > self.options.max_header_length {
1800                let (len, max) = (sec_protocol.len(), self.options.max_header_length);
1801                self.log.warn(|| {
1802                    format!(
1803                        "Encoded auth token length ({len}) exceeds ZeroOptions.maxHeaderLength \
1804                         ({max}). This may cause connection failures."
1805                    )
1806                });
1807            }
1808        } else {
1809            deleted = None;
1810        }
1811        self.log.info(|| format!("Connecting to {url}"));
1812        self.add_connect_event(attempt, "creating the WebSocket");
1813        let handle = SocketHandle::new(self.me.clone());
1814        let connection = socket_factory.open(&url, Some(&sec_protocol), handle.clone());
1815        *handle.connection.borrow_mut() = Some(connection);
1816
1817        if attempt.aborted.get() {
1818            handle.close(None);
1819            let reason = attempt.abort_reason.borrow().clone();
1820            self.finish_attempt(attempt, reason);
1821            return;
1822        }
1823        if self.rep_closed.get() {
1824            handle.close(None);
1825            self.finish_attempt(attempt, None);
1826            return;
1827        }
1828        *self.connect_cookie.borrow_mut() = connect_cookie;
1829        *self.init_connection_queries.borrow_mut() = queries_patch;
1830        *self.deleted_clients.borrow_mut() = deleted;
1831        *self.socket.borrow_mut() = Some(handle);
1832        self.add_connect_event(attempt, "waiting for the server acknowledgement");
1833        self.log
1834            .debug(|| "Waiting for connection to be acknowledged".into());
1835        self.arm_connect_timeout(attempt, true);
1836        let (w, weak_attempt) = (self.me.clone(), Rc::downgrade(attempt));
1837        *attempt.on_connected.borrow_mut() = Some(Box::new(move || {
1838            let (Some(e), Some(attempt)) = (w.upgrade(), weak_attempt.upgrade()) else {
1839                return;
1840            };
1841            if attempt.aborted.get() {
1842                let reason = attempt.abort_reason.borrow().clone();
1843                e.finish_attempt(&attempt, reason);
1844                return;
1845            }
1846            e.mutation_tracker
1847                .on_connected(e.last_mutation_id_received.get());
1848            e.request_push(true);
1849            e.finish_attempt(&attempt, None);
1850        }));
1851    }
1852
1853    /// The run loop's handling of a finished attempt; `error` `None` for
1854    /// success.
1855    fn finish_attempt(&self, attempt: &Rc<ConnectAttempt>, error: Option<ConnectionError>) {
1856        if attempt.finished.replace(true) {
1857            return;
1858        }
1859        attempt.cancel_timeout();
1860        {
1861            let mut current = self.current_attempt.borrow_mut();
1862            if current.as_ref().is_some_and(|c| Rc::ptr_eq(c, attempt)) {
1863                *current = None;
1864            }
1865        }
1866        let ready = self.connection_ready.borrow().clone();
1867        if let Some(error) = error {
1868            if !attempt.aborted.get() {
1869                self.disconnect(error.clone(), None);
1870            }
1871            ready.0.borrow_mut().error.get_or_insert(error.clone());
1872            ready.settle(false, &self.event_loop);
1873            let parked: Vec<InspectRequest> = {
1874                let mut requests = self.inspect_requests.borrow_mut();
1875                let (parked, rest) = std::mem::take(&mut *requests)
1876                    .into_iter()
1877                    .partition(|r| r.parked);
1878                *requests = rest;
1879                parked
1880            };
1881            for r in parked {
1882                (r.done)(Err(error.clone().into()));
1883            }
1884            if self.connection_ready.borrow().is(&ready) {
1885                *self.connection_ready.borrow_mut() = ReadyPromise::default();
1886            }
1887            self.loop_failed(error);
1888            return;
1889        }
1890        ready.settle(true, &self.event_loop);
1891        let parked: Vec<String> = self
1892            .inspect_requests
1893            .borrow_mut()
1894            .iter_mut()
1895            .filter(|r| r.parked)
1896            .map(|r| {
1897                r.parked = false;
1898                r.id.clone()
1899            })
1900            .collect();
1901        for id in parked {
1902            self.inspect_when_connected(id);
1903        }
1904        *self.additional_connect_params.borrow_mut() = None;
1905        match Self::connection_error(&self.connection_manager.state()) {
1906            Some(error) => self.loop_failed(error),
1907            None => self.iterate(),
1908        }
1909    }
1910
1911    /// `#disconnect`: closes the socket and moves the connection status.
1912    pub(crate) fn disconnect(&self, reason: ConnectionError, close_code: Option<u16>) {
1913        let attempt = self.current_attempt.borrow().clone();
1914        if let Some(attempt) = attempt
1915            && !attempt.aborted.get()
1916        {
1917            *attempt.abort_reason.borrow_mut() = Some(reason.clone());
1918            attempt.aborted.set(true);
1919            attempt.cancel_timeout();
1920            let reason = reason.clone();
1921            self.micro(move |e| e.finish_attempt(&attempt, Some(reason)));
1922        }
1923        let status = self.connection_manager.status();
1924        self.log.info(|| {
1925            format!(
1926                "disconnecting: reason={} connectionState={}",
1927                reason.kind_name(),
1928                status.as_str()
1929            )
1930        });
1931        if status == ConnectionStatus::Closed {
1932            self.log.debug(|| "disconnect() called while closed".into());
1933            return;
1934        }
1935        if let Some(at) = self.connected_at.get()
1936            && self.scheduler.monotonic_now() - at >= STABLE_CONNECTION_MS
1937        {
1938            self.transport_failures.set(0);
1939        }
1940        self.connected_at.set(None);
1941        let socket = self.socket.borrow_mut().take();
1942        if let Some(s) = socket {
1943            s.close(close_code);
1944        }
1945        self.last_mutation_ids_sent.borrow_mut().clear();
1946        self.poke_handler.borrow_mut().clear();
1947        self.query_manager.clear_got_queries_authoritative();
1948        let (transition, r) = ConnectionTransition::of(&reason);
1949        match transition {
1950            ConnectionTransition::NeedsAuth => self.connection_manager.needs_auth(r),
1951            ConnectionTransition::Error => self.connection_manager.error(r),
1952            ConnectionTransition::Disconnected => self.connection_manager.disconnected(r),
1953            ConnectionTransition::Closed => self.connection_manager.closed(),
1954            ConnectionTransition::None => {
1955                if !self.connection_manager.is_in_terminal_state() {
1956                    self.connection_manager.connecting(Some(r));
1957                }
1958            }
1959        }
1960        // The answers to the inspector's requests would have come on the
1961        // socket that closed.
1962        let lost: Vec<InspectRequest> = {
1963            let mut requests = self.inspect_requests.borrow_mut();
1964            let (lost, waiting) = std::mem::take(&mut *requests)
1965                .into_iter()
1966                .partition(|r| r.sent);
1967            *requests = waiting;
1968            lost
1969        };
1970        for r in lost {
1971            (r.done)(Err(Error::Message(format!(
1972                "The inspector's request got no answer: the connection to zero-cache closed ({})",
1973                reason.message()
1974            ))));
1975        }
1976    }
1977
1978    // MARK: Messages
1979
1980    fn is_socket(&self, handle: &Rc<SocketHandle>) -> bool {
1981        self.socket
1982            .borrow()
1983            .as_ref()
1984            .is_some_and(|s| Rc::ptr_eq(s, handle))
1985    }
1986
1987    fn socket_did_receive(&self, handle: &Rc<SocketHandle>, message: SocketMessage) {
1988        if !self.is_socket(handle) {
1989            return;
1990        }
1991        let event_loop = self.event_loop.clone();
1992        event_loop.dispatch(|| self.on_message(message));
1993    }
1994
1995    fn socket_did_open(&self, handle: &Rc<SocketHandle>) {
1996        if !self.is_socket(handle) {
1997            return;
1998        }
1999        let Some(attempt) = self.current_attempt.borrow().clone() else {
2000            return;
2001        };
2002        let ms = self.scheduler.monotonic_now() - attempt.start.get();
2003        self.log
2004            .info(|| format!("Got socket open event: timeToOpenMs={}", number::format(ms)));
2005    }
2006
2007    fn socket_did_close(
2008        &self,
2009        handle: &Rc<SocketHandle>,
2010        code: u16,
2011        reason: &str,
2012        was_clean: bool,
2013    ) {
2014        if !self.is_socket(handle) {
2015            return;
2016        }
2017        let event_loop = self.event_loop.clone();
2018        event_loop.dispatch(|| {
2019            // 1005 and 1006 are how platforms report a close without a
2020            // frame.
2021            if code <= 1001 || code == 1005 || code == 1006 {
2022                self.log.info(|| {
2023                    format!("Got socket close event: code={code} reason={reason} wasClean={was_clean}")
2024                });
2025            } else {
2026                self.log.error(|| {
2027                    format!(
2028                        "Got unexpected socket close event: code={code} reason={reason} wasClean={was_clean}"
2029                    )
2030                });
2031            }
2032            if code == 1009 {
2033                let mut message = "A WebSocket message exceeded the server message size limit. \
2034                    This is usually caused by a mutation's args containing a large value such as \
2035                    a base64-encoded image. Consider uploading large files to object storage and \
2036                    storing only the URL in Zero."
2037                    .to_owned();
2038                let recent = self.recent_sent_messages.borrow();
2039                if !recent.is_empty() {
2040                    message.push_str("\nRecent sent messages:\n");
2041                    message.push_str(&pretty_recent_messages(&recent));
2042                }
2043                drop(recent);
2044                self.disconnect(
2045                    ConnectionError::client(ClientErrorKind::InvalidMessage, message),
2046                    None,
2047                );
2048                self.disable_client_group();
2049                self.micro(|e| e.client_state_not_found());
2050            } else if was_clean {
2051                self.disconnect(
2052                    ConnectionError::client(
2053                        ClientErrorKind::CleanClose,
2054                        "WebSocket connection closed cleanly",
2055                    ),
2056                    None,
2057                );
2058            } else {
2059                self.disconnect(
2060                    ConnectionError::client(
2061                        ClientErrorKind::AbruptClose,
2062                        "WebSocket connection closed abruptly",
2063                    ),
2064                    None,
2065                );
2066            }
2067        });
2068    }
2069
2070    fn abort_ping_timeout(&self) {
2071        let f = self.abort_ping_timeout.borrow().clone();
2072        f();
2073    }
2074
2075    /// Disconnects with `error` for a message that doesn't make sense.
2076    fn fail_on_message(&self, error: ConnectionError) {
2077        self.disconnect(error, None);
2078    }
2079
2080    fn on_message(&self, message: SocketMessage) {
2081        if self.rep_closed.get() {
2082            return;
2083        }
2084        let text = match message {
2085            SocketMessage::Binary(bytes) => {
2086                if bytes.first() != Some(&poke_chunk::MESSAGE_TYPE) {
2087                    let t = bytes
2088                        .first()
2089                        .map_or_else(|| "undefined".to_owned(), |b| b.to_string());
2090                    self.fail_on_message(ConnectionError::client(
2091                        ClientErrorKind::InvalidMessage,
2092                        format!("Unknown binary message type: {t}"),
2093                    ));
2094                    return;
2095                }
2096                self.abort_ping_timeout();
2097                let result = self
2098                    .poke_handler
2099                    .borrow_mut()
2100                    .handle_poke_chunk(&bytes[1..]);
2101                if let Err(m) = result {
2102                    self.poke_message_error(m);
2103                }
2104                return;
2105            }
2106            SocketMessage::Text(t) => t,
2107        };
2108        let downstream = match Downstream::from_text(&text) {
2109            Ok(d) => d,
2110            Err(e) => {
2111                self.fail_on_message(ConnectionError::client(
2112                    ClientErrorKind::InvalidMessage,
2113                    format!(
2114                        "Invalid message received from server: {}{text}",
2115                        valita_message(&e, &text)
2116                    ),
2117                ));
2118                return;
2119            }
2120        };
2121        match downstream {
2122            Downstream::Connected(body) => self.micro(move |e| e.handle_connected(body)),
2123            Downstream::Error(body) => self.handle_error_message(body),
2124            Downstream::Pong => {
2125                let f = self.on_pong.borrow().clone();
2126                f();
2127            }
2128            Downstream::PokeStart(body) => {
2129                self.abort_ping_timeout();
2130                let result = self.poke_handler.borrow_mut().handle_poke_start(body);
2131                if let Err(m) = result {
2132                    self.poke_message_error(m);
2133                }
2134            }
2135            Downstream::PokePart(body) => {
2136                self.abort_ping_timeout();
2137                let result = self.poke_handler.borrow_mut().handle_poke_part(body);
2138                match result {
2139                    Ok(Some(lmid)) => self.last_mutation_id_received.set(lmid),
2140                    Ok(None) => {}
2141                    Err(m) => self.poke_message_error(m),
2142                }
2143            }
2144            Downstream::PokeEnd(body) => {
2145                self.abort_ping_timeout();
2146                let result = self.poke_handler.borrow_mut().handle_poke_end(body);
2147                match result {
2148                    Ok(Some(r)) => {
2149                        if let Some(lmid) = r.last_mutation_id_change_for_self {
2150                            self.last_mutation_id_received.set(lmid);
2151                        }
2152                        if !self.poke_handler.borrow().playback_running {
2153                            self.start_playback();
2154                        }
2155                    }
2156                    Ok(None) => {}
2157                    Err(m) => self.poke_message_error(m),
2158                }
2159            }
2160            Downstream::Pull(_) => self.abort_ping_timeout(),
2161            Downstream::DeleteClients(body) => self.database.confirm_deleted_clients(
2162                body.client_ids.as_deref().unwrap_or_default(),
2163                body.client_group_ids.as_deref().unwrap_or_default(),
2164            ),
2165            Downstream::PushResponse(body) => {
2166                if matches!(body, PushResponseBody::Error(_)) {
2167                    self.log.error(|| {
2168                        format!(
2169                            "Received an error response when pushing mutations: {}",
2170                            json::stringify(&body.to_json())
2171                        )
2172                    });
2173                }
2174                if let Err(e) = self.mutation_tracker.process_push_response(&body) {
2175                    self.disconnect(
2176                        ConnectionError::client(ClientErrorKind::Internal, e.message()),
2177                        None,
2178                    );
2179                }
2180            }
2181            Downstream::TransformError(errors) => {
2182                self.query_manager.handle_transform_errors(&errors)
2183            }
2184            Downstream::Inspect(body) => self.inspect_answered(body.0),
2185        }
2186    }
2187
2188    fn handle_connected(&self, _body: ConnectedBody) {
2189        let Some(attempt) = self.current_attempt.borrow().clone() else {
2190            return;
2191        };
2192        let ms = self.scheduler.monotonic_now() - attempt.start.get();
2193        self.log
2194            .info(|| format!("Connected: timeToConnectMs={}", number::format(ms)));
2195        self.add_connect_event(&attempt, "processing the server acknowledgement");
2196        self.connected_at.set(Some(self.scheduler.monotonic_now()));
2197        self.last_mutation_ids_sent.borrow_mut().clear();
2198        let init_queries = self.init_connection_queries.borrow_mut().take();
2199        let patch = self
2200            .query_manager
2201            .queries_patch(&self.store.borrow().main, init_queries.as_ref());
2202        let patch_ops: Vec<UpQueriesPatchOp> = patch.values().cloned().collect();
2203        match init_queries {
2204            Some(_) if !patch.is_empty() => {
2205                self.send(Upstream::ChangeDesiredQueries(ChangeDesiredQueriesBody {
2206                    desired_queries_patch: patch_ops,
2207                    traceparent: self.traceparent(),
2208                }));
2209            }
2210            Some(_) => {}
2211            None => {
2212                let deleted = self.deleted_clients.borrow_mut().take();
2213                let client_schema = self
2214                    .connect_cookie
2215                    .borrow()
2216                    .is_null()
2217                    .then(|| self.client_schema.clone());
2218                self.send(Upstream::InitConnection(InitConnectionBody {
2219                    desired_queries_patch: patch_ops,
2220                    client_schema,
2221                    deleted,
2222                    user_push_url: self.options.mutate_url.clone(),
2223                    user_push_headers: self.options.mutate_headers.clone(),
2224                    user_query_url: self.options.query_url.clone(),
2225                    user_query_headers: self.options.query_headers.clone(),
2226                    active_clients: None,
2227                    traceparent: self.traceparent(),
2228                }));
2229            }
2230        }
2231        let deleted = self.deleted_clients.borrow_mut().take();
2232        if let Some(deleted) = deleted {
2233            self.send(Upstream::DeleteClients(deleted));
2234        }
2235        self.connection_manager.connected();
2236        let on_connected = attempt.on_connected.borrow_mut().take();
2237        if let Some(f) = on_connected {
2238            self.event_loop.microtask(f);
2239        }
2240    }
2241
2242    fn handle_error_message(&self, body: ErrorBody) {
2243        let kind = body.kind();
2244        if kind == ErrorKind::MutationRateLimited {
2245            self.last_mutation_ids_sent.borrow_mut().clear();
2246            self.log.error(|| {
2247                format!(
2248                    "{} Mutation rate limited: {}",
2249                    kind.as_str(),
2250                    body.message()
2251                )
2252            });
2253            return;
2254        }
2255        // Being sent elsewhere or later, a lost client group (which
2256        // recovers), and auth failures are routine; every other kind ends
2257        // the connection.
2258        let message = body.message().to_owned();
2259        let error = ConnectionError::Server(Box::new(body));
2260        let routine =
2261            error.backoff().is_some() || kind == ErrorKind::ClientNotFound || error.is_auth_error();
2262        self.log.log(
2263            if routine {
2264                LogLevel::Warn
2265            } else {
2266                LogLevel::Error
2267            },
2268            || format!("{}:\n\n{message}", kind.as_str()),
2269        );
2270        self.disconnect(error, None);
2271        match kind {
2272            ErrorKind::VersionNotSupported => self.update_needed(UpdateNeededReason {
2273                type_: kind.as_str().to_owned(),
2274                message: Some(message),
2275            }),
2276            ErrorKind::SchemaVersionNotSupported => {
2277                self.disable_client_group();
2278                self.micro(move |e| {
2279                    e.update_needed(UpdateNeededReason {
2280                        type_: "SchemaVersionNotSupported".into(),
2281                        message: Some(message),
2282                    })
2283                });
2284            }
2285            ErrorKind::ClientNotFound => {
2286                self.disable_client_group();
2287                self.micro(|e| e.client_state_not_found());
2288            }
2289            ErrorKind::InvalidConnectionRequestLastMutationID
2290            | ErrorKind::InvalidConnectionRequestBaseCookie => {
2291                // The server is behind the replica: drop it.
2292                self.database.drop_database();
2293                self.micro(|e| e.client_state_not_found());
2294            }
2295            _ => {}
2296        }
2297    }
2298
2299    /// A poke message that doesn't fit the poke being received.
2300    fn poke_message_error(&self, message: String) {
2301        self.poke_handler.borrow_mut().clear();
2302        self.on_poke_error(&message);
2303    }
2304
2305    fn on_poke_error(&self, message: &str) {
2306        if self.status_is(ConnectionStatus::Disconnected) {
2307            return;
2308        }
2309        if message.contains("unexpected base cookie") {
2310            self.log.debug(|| format!("clearing due to {message}"));
2311            self.disconnect(
2312                ConnectionError::client(
2313                    ClientErrorKind::UnexpectedBaseCookie,
2314                    "Server returned unexpected base cookie during sync",
2315                ),
2316                None,
2317            );
2318        } else {
2319            self.log
2320                .error(|| format!("clearing due to unexpected poke error: {message}"));
2321            self.disconnect(
2322                ConnectionError::client(
2323                    ClientErrorKind::Internal,
2324                    format!("Poke processing error: {message}"),
2325                ),
2326                None,
2327            );
2328        }
2329    }
2330
2331    // MARK: Poke playback
2332
2333    fn start_playback(&self) {
2334        self.poke_handler.borrow_mut().playback_running = true;
2335        let _ = self.later(0.0, |e| e.playback_frame());
2336    }
2337
2338    /// Applies the pokes that arrived since the last frame, at most once
2339    /// per scheduler turn, merged (`#rafCallback`).
2340    fn playback_frame(&self) {
2341        if !self.poke_handler.borrow().has_buffered() {
2342            self.poke_handler.borrow_mut().playback_running = false;
2343            return;
2344        }
2345        let _ = self.later(0.0, |e| e.playback_frame());
2346        let merged = self.poke_handler.borrow_mut().take_merged();
2347        let merged = match merged {
2348            Ok(Some(m)) => m,
2349            Ok(None) => return,
2350            Err(e) => {
2351                self.poke_handler.borrow_mut().clear();
2352                self.on_poke_error(&e.0);
2353                return;
2354            }
2355        };
2356        let applied = self.with_write_lock(|| self.apply_poke(&merged));
2357        let result = applied.and_then(|()| {
2358            let lmid = merged
2359                .last_mutation_id_changes
2360                .get(&self.client_id)
2361                .copied();
2362            let advanced = match lmid {
2363                Some(lmid) => self
2364                    .mutation_tracker
2365                    .lmid_advanced(lmid)
2366                    .map_err(|m| format!("Error: {m}")),
2367                None => Ok(()),
2368            };
2369            self.settle_pending_waiters();
2370            advanced
2371        });
2372        if let Err(message) = result {
2373            self.poke_handler.borrow_mut().clear();
2374            self.on_poke_error(&message);
2375        }
2376    }
2377
2378    // MARK: Auth
2379
2380    pub(crate) fn set_auth(&self, auth: Option<String>) {
2381        // A token without a user was rejected at init.
2382        *self.auth.borrow_mut() = auth.clone().unwrap_or_default();
2383        if let Some(auth) = auth
2384            && !auth.is_empty()
2385        {
2386            self.send_if_connected(Upstream::UpdateAuth(UpdateAuthBody { auth }));
2387        }
2388    }
2389
2390    /// `connection.connect()`: resumes from needs-auth or error. `auth`,
2391    /// when set, replaces the token first (`Some(None)` clears it).
2392    pub(crate) fn connect(&self, auth: Option<Option<String>>) -> Result<(), Error> {
2393        if let Some(auth) = auth {
2394            // JS's `#setAuth` throws this, so `connect` rejects before it
2395            // does anything.
2396            Self::check_auth_valid(self.user_id.as_deref(), auth.as_deref())?;
2397            self.set_auth(auth);
2398        }
2399        if let State::Disconnected(r) = self.connection_manager.state()
2400            && r.client_kind() == Some(ClientErrorKind::NoSocketOrigin)
2401        {
2402            self.log.error(|| {
2403                "connect() called but the connection is disconnected due to a missing cacheURL. \
2404                 No reconnect will be attempted."
2405                    .into()
2406            });
2407            return Ok(());
2408        }
2409        if self.connection_manager.is_in_terminal_state() {
2410            self.connection_manager.request_connect();
2411        }
2412        Ok(())
2413    }
2414
2415    fn client_state_not_found(&self) {
2416        let f = self.on_client_state_not_found.borrow().clone();
2417        match f {
2418            Some(f) => f(),
2419            None => {
2420                if let Some(f) = &self.options.on_client_state_not_found {
2421                    f();
2422                }
2423            }
2424        }
2425    }
2426
2427    fn update_needed(&self, reason: UpdateNeededReason) {
2428        let f = self.on_update_needed.borrow().clone();
2429        if let Some(f) = f {
2430            f(&reason);
2431        }
2432        if let Some(f) = &self.options.on_update_needed {
2433            f(&reason);
2434        }
2435    }
2436
2437    // MARK: Close
2438
2439    pub(crate) fn closed(&self) -> bool {
2440        self.rep_closed.get()
2441    }
2442
2443    pub(crate) fn close(&self) {
2444        if self.rep_closed.get() {
2445            self.log
2446                .debug(|| "close() called on already closed instance".into());
2447            return;
2448        }
2449        self.log.debug(|| "Closing Zero instance".into());
2450        if !self.status_is(ConnectionStatus::Disconnected) {
2451            self.disconnect(
2452                ConnectionError::client(
2453                    ClientErrorKind::ClientClosed,
2454                    "Zero instance closed by user",
2455                ),
2456                Some(1000),
2457            );
2458        }
2459        self.rep_closed.set(true);
2460        if let Some(t) = self.push_timer.borrow_mut().take() {
2461            t.cancel();
2462        }
2463        // JS's close aborts a persist that hasn't run.
2464        if let Some(t) = self.persist_timer.borrow_mut().take() {
2465            t.cancel();
2466        }
2467        for t in std::mem::take(&mut *self.lifecycle_timers.borrow_mut()) {
2468            t.cancel();
2469        }
2470        self.database.client_stopped(&self.client_id);
2471        // Releasing writes what the store had queued; a failure to is
2472        // still this client's to report.
2473        self.database.release();
2474        self.database.remove_failure_listener(&self.client_id);
2475        self.connection_manager.closed();
2476        let requests = std::mem::take(&mut *self.inspect_requests.borrow_mut());
2477        for r in requests {
2478            (r.done)(Err(ConnectionError::client(
2479                ClientErrorKind::ClientClosed,
2480                "Zero instance closed by user",
2481            )
2482            .into()));
2483        }
2484    }
2485
2486    // MARK: The inspector
2487
2488    /// Sends an inspector request (`rpcNoAuthTry`) once connected, with a
2489    /// new ID; `done` hears the answer with that ID, or why there is none.
2490    pub(crate) fn inspect(&self, body: Object, done: InspectAnswer) {
2491        if self.rep_closed.get() {
2492            return done(Err(ConnectionError::client(
2493                ClientErrorKind::ClientClosed,
2494                "Zero instance closed by user",
2495            )
2496            .into()));
2497        }
2498        let id = random_id();
2499        self.inspect_requests.borrow_mut().push(InspectRequest {
2500            id: id.clone(),
2501            body: Some(body),
2502            sent: false,
2503            parked: false,
2504            done,
2505        });
2506        self.inspect_when_connected(id);
2507    }
2508
2509    /// Sends request `id` once the connection is ready (`getSocket`), or
2510    /// fails it when connecting fails.
2511    fn inspect_when_connected(&self, id: String) {
2512        let ready = self.connection_ready.borrow().clone();
2513        let w = self.me.clone();
2514        let r = ready.clone();
2515        ready.on_settle(&self.event_loop, move |connected| {
2516            if let Some(e) = w.upgrade() {
2517                let error = r.0.borrow().error.clone();
2518                e.inspect_ready(id, connected, error);
2519            }
2520        });
2521    }
2522
2523    fn inspect_ready(&self, id: String, connected: bool, error: Option<ConnectionError>) {
2524        let mut requests = self.inspect_requests.borrow_mut();
2525        let Some(i) = requests.iter().position(|r| r.id == id && !r.sent) else {
2526            return;
2527        };
2528        if !connected {
2529            let r = requests.remove(i);
2530            drop(requests);
2531            let error = error.unwrap_or_else(|| {
2532                ConnectionError::client(ClientErrorKind::Internal, "Not connected to zero-cache")
2533            });
2534            return (r.done)(Err(error.into()));
2535        }
2536        let socket = self.socket.borrow().clone();
2537        match socket {
2538            Some(socket) if self.status_is(ConnectionStatus::Connected) => {
2539                let r = &mut requests[i];
2540                r.sent = true;
2541                let mut body = r.body.take().unwrap_or_default();
2542                body.insert("id", Value::from(id.as_str()));
2543                let text = Upstream::Inspect(crate::protocol::InspectUpBody(Value::Object(body)))
2544                    .to_text();
2545                drop(requests);
2546                socket.send(&text);
2547            }
2548            // The connection dropped since: wait for the next attempt.
2549            _ => requests[i].parked = true,
2550        }
2551    }
2552
2553    /// zero-cache answered an inspector request.
2554    fn inspect_answered(&self, body: Value) {
2555        let Value::Object(body) = body else { return };
2556        let Some(id) = body.get("id").and_then(Value::as_str) else {
2557            return;
2558        };
2559        let request = {
2560            let mut requests = self.inspect_requests.borrow_mut();
2561            let Some(i) = requests.iter().position(|r| r.sent && r.id == id) else {
2562                return;
2563            };
2564            requests.remove(i)
2565        };
2566        (request.done)(Ok(body));
2567    }
2568
2569    // MARK: Storage and the client lifecycle
2570
2571    /// Starts the heartbeat and the garbage collection of clients and
2572    /// client groups.
2573    fn start_lifecycle(&self) {
2574        let (h, c, g, d) = (
2575            self.me.clone(),
2576            self.me.clone(),
2577            self.me.clone(),
2578            self.me.clone(),
2579        );
2580        let timers = vec![
2581            self.scheduler.schedule_repeating(
2582                HEARTBEAT_INTERVAL_MS,
2583                Box::new(move || {
2584                    if let Some(e) = h.upgrade()
2585                        && !e.database.heartbeat(&e.client_id, e.scheduler.now())
2586                    {
2587                        e.client_state_not_found();
2588                    }
2589                }),
2590            ),
2591            self.scheduler.schedule_repeating(
2592                GC_INTERVAL_MS,
2593                Box::new(move || {
2594                    let Some(e) = c.upgrade() else { return };
2595                    if let Some(deleted) = e.database.collect_clients(
2596                        &e.client_id,
2597                        e.scheduler.now(),
2598                        CLIENT_MAX_INACTIVE_MS,
2599                    ) {
2600                        // `DeleteClientsManager.onClientsDeleted`.
2601                        let ids = deleted
2602                            .into_iter()
2603                            .filter(|p| p.client_group_id == e.client_group_id)
2604                            .map(|p| p.client_id)
2605                            .collect();
2606                        e.send_if_connected(Upstream::DeleteClients(DeleteClientsBody {
2607                            client_ids: Some(ids),
2608                            client_group_ids: None,
2609                        }));
2610                    }
2611                }),
2612            ),
2613            self.scheduler.schedule_repeating(
2614                GC_INTERVAL_MS,
2615                Box::new(move || {
2616                    if let Some(e) = g.upgrade() {
2617                        e.database.collect_client_groups();
2618                    }
2619                }),
2620            ),
2621            self.scheduler.schedule(
2622                INITIAL_COLLECT_DATABASES_DELAY_MS,
2623                Box::new(move || {
2624                    let Some(e) = d.upgrade() else { return };
2625                    e.collect_databases();
2626                    let w = e.me.clone();
2627                    let t = e.scheduler.schedule_repeating(
2628                        COLLECT_DATABASES_INTERVAL_MS,
2629                        Box::new(move || {
2630                            if let Some(e) = w.upgrade() {
2631                                e.collect_databases();
2632                            }
2633                        }),
2634                    );
2635                    e.lifecycle_timers.borrow_mut().push(t);
2636                }),
2637            ),
2638        ];
2639        *self.lifecycle_timers.borrow_mut() = timers;
2640    }
2641
2642    /// Deletes the other databases of the store that no client has used
2643    /// for twice the time a client may be inactive (`collectIDBDatabases`),
2644    /// such as those of an older schema.
2645    fn collect_databases(&self) {
2646        self.database
2647            .collect_databases(self.scheduler.now(), 2.0 * CLIENT_MAX_INACTIVE_MS);
2648    }
2649
2650    /// The store failed: the client keeps running on what it has in
2651    /// memory, and tells the app once.
2652    fn storage_failed(&self, f: &StorageFailure, opening: bool) {
2653        if self.storage_failure_reported.replace(true) {
2654            return;
2655        }
2656        if !opening {
2657            // A full or failing disk is the device's condition, not a
2658            // defect.
2659            self.log.warn(|| {
2660                format!(
2661                    "Local store storage failed ({}); persistence is stopped for this instance: {}",
2662                    f.kind.as_str(),
2663                    f.message
2664                )
2665            });
2666        }
2667        if let Some(cb) = &self.options.on_storage_failure {
2668            cb(f);
2669        }
2670    }
2671
2672    /// The deleted clients of this client group, for `initConnection`
2673    /// (`convertDeletedClientsToBody`).
2674    fn deleted_clients_body(&self) -> Option<DeleteClientsBody> {
2675        let ids: Vec<String> = self
2676            .database
2677            .deleted_clients()
2678            .into_iter()
2679            .filter(|p| p.client_group_id == self.client_group_id)
2680            .map(|p| p.client_id)
2681            .collect();
2682        (!ids.is_empty()).then_some(DeleteClientsBody {
2683            client_ids: Some(ids),
2684            client_group_ids: None,
2685        })
2686    }
2687
2688    /// The server lost the client group: stop pushing, and keep later
2689    /// clients out of it.
2690    fn disable_client_group(&self) {
2691        self.client_group_disabled.set(true);
2692        self.database.disable_client_group(&self.client_group_id);
2693    }
2694
2695    // MARK: Mutations
2696
2697    /// `MutatorProxy#onConnectionStateChange`.
2698    fn on_connection_state_change(&self, state: &State) {
2699        match state {
2700            State::Disconnected(r) if r.client_kind() == Some(ClientErrorKind::NoSocketOrigin) => {
2701                *self.mutation_rejection.borrow_mut() = None;
2702            }
2703            State::Disconnected(r) | State::Error(r) | State::Closed(r) => {
2704                *self.mutation_rejection.borrow_mut() = Some(r.clone());
2705                self.mutation_tracker.reject_all_outstanding_mutations(r);
2706            }
2707            _ => *self.mutation_rejection.borrow_mut() = None,
2708        }
2709        self.settle_pending_waiters();
2710    }
2711
2712    /// What a mutation's caller hears for an error (`wrapErrorFor`).
2713    pub(crate) fn outcome(error: &Error) -> MutationOutcome {
2714        if let Error::Connection(e) = error
2715            && e.is_zero_error()
2716        {
2717            return MutationOutcome::ZeroError {
2718                message: e.message().to_owned(),
2719            };
2720        }
2721        let app = ApplicationError::wrapping(error);
2722        MutationOutcome::AppError {
2723            message: app.message,
2724            details: app.details,
2725        }
2726    }
2727
2728    fn log_failure(&self, name: &str, outcome: &MutationOutcome, origin: &str) {
2729        match outcome {
2730            MutationOutcome::ZeroError { message } => self
2731                .log
2732                .error(|| format!("Mutator \"{name}\" error on {origin}: {message}")),
2733            MutationOutcome::AppError { message, .. } => self
2734                .log
2735                .error(|| format!("Mutator \"{name}\" app error on {origin}: {message}")),
2736            MutationOutcome::Success { .. } => {}
2737        }
2738    }
2739
2740    /// `zero.mutate`, through the `MutatorProxy` and Replicache's `mutate`.
2741    ///
2742    /// A mutation made while listeners are notified, from a listener, runs
2743    /// once the current task is done, as an async JS mutator commits after
2744    /// the notification: every listener sees the change that notified it
2745    /// before the mutation's, and every view sees both.
2746    pub(crate) fn mutate(&self, name: &str, args: Option<Value>) -> MutatorResult {
2747        let client = Deferred::new();
2748        let server = Deferred::new();
2749        let result = MutatorResult {
2750            client: client.clone(),
2751            server: server.clone(),
2752        };
2753        if self.context.is_notifying() {
2754            let w = self.me.clone();
2755            let name = name.to_owned();
2756            self.event_loop.after_current_task(move || {
2757                let Some(e) = w.upgrade() else { return };
2758                let r = e.mutate(&name, args);
2759                r.client.on_settle(move |o| client.settle(o.clone()));
2760                r.server.on_settle(move |o| server.settle(o.clone()));
2761            });
2762            return result;
2763        }
2764        // JS throws this before it looks at the connection; here both
2765        // outcomes carry it, and nothing is written or pushed.
2766        let Some(mutator) = self.mutators.get(name).cloned() else {
2767            let message = format!(
2768                "Mutator \"{name}\" is not registered. Mutators must be registered with the Zero \
2769                 constructor before use."
2770            );
2771            self.log.error(|| message.clone());
2772            client.resolve(MutationOutcome::ZeroError {
2773                message: message.clone(),
2774            });
2775            server.resolve(MutationOutcome::ZeroError { message });
2776            return result;
2777        };
2778        let rejection = self.mutation_rejection.borrow().clone();
2779        if let Some(rejection) = rejection {
2780            self.log.warn(|| {
2781                format!(
2782                    "Mutation \"{name}\" rejected because Zero is {}. Details: {}.",
2783                    rejection.state_description(),
2784                    rejection.message()
2785                )
2786            });
2787            let message = rejection.message().to_owned();
2788            client.resolve(MutationOutcome::ZeroError {
2789                message: message.clone(),
2790            });
2791            server.resolve(MutationOutcome::ZeroError { message });
2792            return result;
2793        }
2794        let (ephemeral_id, tracked) = self.mutation_tracker.track_mutation();
2795        {
2796            let w = self.me.clone();
2797            let name = name.to_owned();
2798            let server = server.clone();
2799            tracked.on_settle(move |r| match r {
2800                Ok(data) => server.resolve(MutationOutcome::Success { data: data.clone() }),
2801                Err(error) => {
2802                    let outcome = Self::outcome(error);
2803                    if let Some(e) = w.upgrade() {
2804                        e.log_failure(&name, &outcome, "server");
2805                    }
2806                    server.resolve(outcome);
2807                }
2808            });
2809        }
2810        // A mutate from inside a mutator, or while a poke rebases, waits for
2811        // it, as JS's waits for Replicache's write lock.
2812        if self.write_locked.get() || !self.waiting_for_write_lock.borrow().is_empty() {
2813            let w = self.me.clone();
2814            let name = name.to_owned();
2815            self.waiting_for_write_lock
2816                .borrow_mut()
2817                .push_back(Box::new(move || {
2818                    if let Some(e) = w.upgrade() {
2819                        e.with_write_lock(|| {
2820                            e.run_mutation(&mutator, &name, args, ephemeral_id, &client)
2821                        });
2822                    }
2823                }));
2824        } else {
2825            self.with_write_lock(|| self.run_mutation(&mutator, name, args, ephemeral_id, &client));
2826        }
2827        result
2828    }
2829
2830    fn with_write_lock<T>(&self, body: impl FnOnce() -> T) -> T {
2831        self.write_locked.set(true);
2832        let result = body();
2833        self.write_locked.set(false);
2834        if !self.waiting_for_write_lock.borrow().is_empty() {
2835            self.micro(|e| {
2836                if e.write_locked.get() {
2837                    return;
2838                }
2839                let next = e.waiting_for_write_lock.borrow_mut().pop_front();
2840                if let Some(next) = next {
2841                    next();
2842                }
2843            });
2844        }
2845        result
2846    }
2847
2848    fn run_mutation(
2849        &self,
2850        mutator: &MutatorFn,
2851        name: &str,
2852        args: Option<Value>,
2853        ephemeral_id: u64,
2854        client: &Deferred<MutationOutcome>,
2855    ) {
2856        let mutation_id = self.store.borrow().next_mutation_id(&self.client_id);
2857        // Last mutation IDs are checked when they arrive, so this holds.
2858        if !number::is_safe_integer(mutation_id) {
2859            let error: Error = ConnectionError::client(
2860                ClientErrorKind::Internal,
2861                format!("Invalid mutation ID {}", number::format(mutation_id)),
2862            )
2863            .into();
2864            self.mutation_tracker.reject_mutation(ephemeral_id, &error);
2865            client.resolve(Self::outcome(&error));
2866            return;
2867        }
2868        let write = KvWrite::new(self.store.borrow().main.clone());
2869        let branch = self.ivm_main.fork();
2870        self.mutation_tracker
2871            .mutation_id_assigned(ephemeral_id, mutation_id);
2872        let tx = Transaction::new(
2873            self.client_id.clone(),
2874            mutation_id,
2875            Reason::Optimistic,
2876            self.token(),
2877            write,
2878            branch,
2879            self.schema.clone(),
2880        );
2881        // JS forks the views' branch for the transaction, which throws once
2882        // that branch has failed.
2883        let ran = match self.context.branch_failure() {
2884            Some(f) => Err(f.into()),
2885            None => mutator(&tx, args.as_ref()),
2886        };
2887        let data = match ran {
2888            Ok(data) => data,
2889            Err(error) => {
2890                self.mutation_tracker.reject_mutation(ephemeral_id, &error);
2891                let outcome = Self::outcome(&error);
2892                self.log_failure(name, &outcome, "client");
2893                client.resolve(outcome);
2894                return;
2895            }
2896        };
2897        let (write, rows) = tx.finish();
2898        let pending = PendingMutation {
2899            id: mutation_id,
2900            client_id: self.client_id.clone(),
2901            name: name.to_owned(),
2902            args: args.clone().unwrap_or(Value::Null),
2903            timestamp: self.scheduler.monotonic_now(),
2904            writes: write.writes.clone(),
2905        };
2906        self.mutation_tracker.mutation_ran(mutation_id, name, args);
2907        let stored = StoredMutation {
2908            client_id: self.client_id.clone(),
2909            id: mutation_id,
2910            name: name.to_owned(),
2911            args: pending.args.clone(),
2912            timestamp: pending.timestamp,
2913            aggregate_writes: aggregate_writes(&write),
2914        };
2915        let diff = self.store.borrow_mut().commit_mutation(pending, &write);
2916        self.database.apply(vec![StorageOp::AddMutation {
2917            group: self.client_group_id.clone(),
2918            mutation: stored,
2919        }]);
2920        if let Err(error) = self.context.process_changes(&diff, &rows) {
2921            // As in JS, the mutation stays committed and pending, but isn't
2922            // pushed now, and its client outcome is the error.
2923            self.fire_watchers(&diff, false);
2924            let outcome = Self::outcome(&error.into());
2925            self.log_failure(name, &outcome, "client");
2926            client.resolve(outcome);
2927            return;
2928        }
2929        self.request_push(false);
2930        self.fire_watchers(&diff, false);
2931        self.schedule_persist();
2932        client.resolve(MutationOutcome::Success { data });
2933    }
2934
2935    /// The store watches: the mutation tracker on `m/<clientID>/`, then the
2936    /// query manager on `g/`. A watch hears only about diffs that touch its
2937    /// prefix, except for its first run.
2938    fn fire_watchers(&self, diff: &KvDiff, initial: bool) {
2939        let mutations = with_prefix(diff, &format!("{MUTATIONS_PREFIX}{}/", self.client_id));
2940        if initial || !mutations.is_empty() {
2941            // A fatal error disconnects from inside the tracker.
2942            let _ = self.mutation_tracker.process_mutation_responses(&mutations);
2943        }
2944        let got = with_prefix(diff, GOT_QUERIES_PREFIX);
2945        if initial || !got.is_empty() {
2946            self.query_manager.process_got_queries(&got);
2947        }
2948    }
2949
2950    // MARK: Pushing
2951
2952    /// `pushConnectionLoop.send(now)`.
2953    fn request_push(&self, now: bool) {
2954        if self.rep_closed.get() {
2955            return;
2956        }
2957        self.push_requested.set(true);
2958        if self.push_in_flight.get() {
2959            return;
2960        }
2961        if now {
2962            if let Some(t) = self.push_timer.borrow_mut().take() {
2963                t.cancel();
2964            }
2965            self.micro(|e| e.invoke_push());
2966        } else if self.push_timer.borrow().is_none() {
2967            let t = self.later(0.0, |e| {
2968                e.push_timer.borrow_mut().take();
2969                e.invoke_push();
2970            });
2971            *self.push_timer.borrow_mut() = Some(t);
2972        }
2973    }
2974
2975    /// `#invokePush`: pushes the pending mutations once connected.
2976    fn invoke_push(&self) {
2977        if self.push_in_flight.get()
2978            || !self.push_requested.get()
2979            || self.rep_closed.get()
2980            || self.client_group_disabled.get()
2981        {
2982            return;
2983        }
2984        self.push_requested.set(false);
2985        let pending = self.store.borrow().pending.clone();
2986        if pending.is_empty() {
2987            return;
2988        }
2989        self.push_in_flight.set(true);
2990        let w = self.me.clone();
2991        let ready = self.connection_ready.borrow().clone();
2992        ready.on_settle(&self.event_loop, move |connected| {
2993            let Some(e) = w.upgrade() else { return };
2994            if connected {
2995                e.pusher(&pending);
2996            }
2997            e.push_in_flight.set(false);
2998            if !connected {
2999                e.push_requested.set(true);
3000            }
3001            if e.push_requested.get() {
3002                e.request_push(false);
3003            }
3004        });
3005    }
3006
3007    /// `#pusher`: one `push` message per mutation not yet sent on this
3008    /// connection.
3009    fn pusher(&self, mutations: &[PendingMutation]) {
3010        let now = self.scheduler.now();
3011        let counter = self.push_request_counter.get() + 1;
3012        self.push_request_counter.set(counter);
3013        let request_id = format!("{}-{counter}", self.client_id);
3014        for m in mutations {
3015            let sent = self
3016                .last_mutation_ids_sent
3017                .borrow()
3018                .get(&m.client_id)
3019                .copied()
3020                .unwrap_or(0.0);
3021            if m.id <= sent {
3022                continue;
3023            }
3024            let timestamp = now - (self.scheduler.monotonic_now() - m.timestamp).round();
3025            self.send(Upstream::Push(PushBody {
3026                client_group_id: self.client_group_id.clone(),
3027                mutations: vec![Mutation::Custom {
3028                    id: m.id,
3029                    client_id: m.client_id.clone(),
3030                    name: m.name.clone(),
3031                    args: vec![m.args.clone()],
3032                    timestamp,
3033                }],
3034                push_version: 1.0,
3035                schema_version: None,
3036                timestamp: now,
3037                request_id: request_id.clone(),
3038                auth: None,
3039                traceparent: self.traceparent(),
3040            }));
3041            self.last_mutation_ids_sent
3042                .borrow_mut()
3043                .insert(m.client_id.clone(), m.id);
3044        }
3045    }
3046
3047    // MARK: Pokes
3048
3049    /// `rep.poke`: applies a merged poke to the snapshot and rebases the
3050    /// pending mutations on it (`handlePullResponseV1` and
3051    /// `maybeEndPull`). Errors are JS's `String(error)`.
3052    fn apply_poke(&self, poke: &MergedPoke) -> Result<(), String> {
3053        let socket_at_start = self.socket.borrow().clone();
3054        let base = self.store.borrow().snapshot.clone();
3055        if poke.base_cookie != base.cookie {
3056            return Err(format!(
3057                "Error: unexpected base cookie for poke: {}",
3058                json::stringify(&poke.to_json())
3059            ));
3060        }
3061        let mut changes: Vec<(&String, &f64)> = poke.last_mutation_id_changes.iter().collect();
3062        changes.sort_by(|a, b| strings::utf16(a.0, b.0));
3063        for (cid, lmid) in changes {
3064            if let Some(last) = base.last_mutation_ids.get(cid)
3065                && lmid < last
3066            {
3067                return Err(format!(
3068                    "Error: {}",
3069                    bad_order_message(
3070                        &format!("{cid} lastMutationID"),
3071                        &number::format(*lmid),
3072                        &number::format(*last)
3073                    )
3074                ));
3075            }
3076        }
3077        if compare_cookies(&poke.cookie, &base.cookie) < 0.0 {
3078            return Err(format!(
3079                "Error: {}",
3080                bad_order_message(
3081                    "cookie",
3082                    &json::stringify(&poke.cookie),
3083                    &json::stringify(&base.cookie)
3084                )
3085            ));
3086        }
3087        if poke.cookie == base.cookie {
3088            // `zero.ts` trusts the got queries once `rep.poke` returns, even
3089            // for a poke that changes nothing.
3090            self.mark_got_queries_authoritative(socket_at_start.as_ref());
3091            return Ok(());
3092        }
3093
3094        let mut snapshot = base;
3095        for (cid, lmid) in &poke.last_mutation_id_changes {
3096            snapshot.last_mutation_ids.insert(cid.clone(), *lmid);
3097        }
3098        snapshot.cookie = poke.cookie.clone();
3099        let mut patch_write = KvWrite::new(snapshot.map.clone());
3100        apply_patch(&poke.patch, &mut patch_write).map_err(|e| format!("Error: {}", e.0))?;
3101        self.check_updated_rows(&poke.patch, &patch_write)?;
3102        snapshot.map = patch_write.map.clone();
3103
3104        let (store_main, pending_writes, pending) = {
3105            let store = self.store.borrow();
3106            (
3107                store.main.clone(),
3108                store.pending_writes(),
3109                store.pending.clone(),
3110            )
3111        };
3112        let mut touched: HashSet<String> =
3113            patch_write.writes.union(&pending_writes).cloned().collect();
3114        let mut main = snapshot.map.clone();
3115        let mut replayed = Vec::new();
3116        let pending: Vec<PendingMutation> = pending
3117            .into_iter()
3118            .filter(|m| {
3119                m.id > snapshot
3120                    .last_mutation_ids
3121                    .get(&m.client_id)
3122                    .copied()
3123                    .unwrap_or(0.0)
3124            })
3125            .collect();
3126        if !pending.is_empty() {
3127            // The rebase runs over a fork of the views' branch moved to the
3128            // new snapshot. JS's fork throws once that branch has failed.
3129            if let Some(f) = self.context.branch_failure() {
3130                return Err(format!("Error: {f}"));
3131            }
3132            let branch = self.ivm_main.fork();
3133            let moved = diff_keys(&store_main, &snapshot.map, &touched);
3134            capture(|| branch.apply(&moved, &HashMap::new())).map_err(|e| format!("Error: {e}"))?;
3135            let (m, r, t, _) =
3136                self.replay(&pending, &main, branch, &HashMap::new(), &mut |_, _| {});
3137            main = m;
3138            replayed = r;
3139            touched.extend(t);
3140        }
3141        let cookie = snapshot.cookie.clone();
3142        let last_mutation_ids = snapshot.last_mutation_ids.clone();
3143        let changes: Vec<(String, Option<Value>)> = {
3144            let mut keys: Vec<&String> = patch_write.writes.iter().collect();
3145            keys.sort();
3146            keys.into_iter()
3147                .filter_map(|k| {
3148                    let v = snapshot.map.get(k).cloned();
3149                    // After a clear, only what the poke put back.
3150                    (!(patch_write.cleared && v.is_none())).then(|| (k.clone(), v))
3151                })
3152                .collect()
3153        };
3154        let diff = self
3155            .store
3156            .borrow_mut()
3157            .advance(snapshot, main, replayed, Some(&touched));
3158        self.database.apply(vec![StorageOp::UpdateSnapshot {
3159            group: self.client_group_id.clone(),
3160            changes,
3161            cleared: patch_write.cleared,
3162            cookie,
3163            last_mutation_ids,
3164        }]);
3165        // A push that fails is a poke error: the client disconnects.
3166        self.context
3167            .process_changes(&diff, &HashMap::new())
3168            .map_err(|e| format!("Error: {e}"))?;
3169        self.fire_watchers(&diff, false);
3170        self.schedule_persist();
3171        self.mark_got_queries_authoritative(socket_at_start.as_ref());
3172        Ok(())
3173    }
3174
3175    /// The server has caught the client up, unless the connection the poke
3176    /// came on went away while it was applied: the disconnect already
3177    /// re-gated trust, and the next connection's first poke restores it.
3178    fn mark_got_queries_authoritative(&self, socket_at_start: Option<&Rc<SocketHandle>>) {
3179        if let Some(s) = socket_at_start
3180            && self.is_socket(s)
3181        {
3182            self.query_manager.mark_got_queries_authoritative();
3183        }
3184    }
3185
3186    /// Runs `pending` again, in order, over `map` and on forks of `branch`
3187    /// (the rebase of `maybeEndPull`). A mutation that fails stays pending
3188    /// without writes. Returns the new main map, the mutations with what
3189    /// they wrote, every key they wrote, and the branch they ran on.
3190    ///
3191    /// At launch, no query has registered the synced aggregates a mutation
3192    /// moves yet, so each run puts back the aggregate rows `restoring` has
3193    /// for the mutation, as JS's stored commit holds them. `ran` hears each
3194    /// mutation with its write, `None` when it failed.
3195    #[allow(clippy::type_complexity)]
3196    fn replay(
3197        &self,
3198        pending: &[PendingMutation],
3199        map: &KvMap,
3200        branch: Rc<IvmBranch>,
3201        restoring: &HashMap<MutationKey, AggregateWrites>,
3202        ran: &mut dyn FnMut(&PendingMutation, Option<&KvWrite>),
3203    ) -> (KvMap, Vec<PendingMutation>, HashSet<String>, Rc<IvmBranch>) {
3204        let mut main = map.clone();
3205        let mut branch = branch;
3206        let mut touched = HashSet::new();
3207        let mut replayed = Vec::new();
3208        for m in pending {
3209            let mut m = m.clone();
3210            let fork = branch.fork();
3211            let tx = Transaction::new(
3212                m.client_id.clone(),
3213                m.id,
3214                Reason::Rebase,
3215                self.token(),
3216                KvWrite::new(main.clone()),
3217                fork.clone(),
3218                self.schema.clone(),
3219            );
3220            let result = match self.mutators.get(&m.name) {
3221                Some(mutator) => mutator(&tx, Some(&m.args).filter(|a| !a.is_null())),
3222                None => Err(ApplicationError::new(format!("Unknown mutator: {}", m.name)).into()),
3223            };
3224            match result {
3225                Ok(_) => {
3226                    let (mut write, _) = tx.finish();
3227                    if let Some(writes) = restoring.get(&mutation_key(&m.client_id, m.id)) {
3228                        for (key, value) in writes {
3229                            match value {
3230                                Some(v) => write.set(key, v.clone()),
3231                                None => {
3232                                    write.delete(key);
3233                                }
3234                            }
3235                        }
3236                    }
3237                    main = write.map.clone();
3238                    branch = fork;
3239                    m.writes = write.writes.clone();
3240                    touched.extend(write.writes.iter().cloned());
3241                    ran(&m, Some(&write));
3242                }
3243                Err(_) => {
3244                    // The mutation stays pending; its writes are dropped.
3245                    m.writes = HashSet::new();
3246                    ran(&m, None);
3247                }
3248            }
3249            replayed.push(m);
3250        }
3251        (main, replayed, touched, branch)
3252    }
3253
3254    /// Replicache's persist scheduler (`ProcessScheduler`): after a
3255    /// mutation or a poke, a persist runs once the last one is
3256    /// `PERSIST_THROTTLE_MS` old, then after `PERSIST_IDLE_TIMEOUT_MS`;
3257    /// requests while one waits join it.
3258    fn schedule_persist(&self) {
3259        if self.persist_timer.borrow().is_some() || self.rep_closed.get() {
3260            return;
3261        }
3262        let now = self.scheduler.monotonic_now();
3263        let throttled = self
3264            .last_persist_at
3265            .get()
3266            .map_or(0.0, |at| (at + PERSIST_THROTTLE_MS - now).max(0.0));
3267        let t = self.later(throttled + PERSIST_IDLE_TIMEOUT_MS, |e| e.persist());
3268        *self.persist_timer.borrow_mut() = Some(t);
3269    }
3270
3271    /// `persistDD31`, for the synced aggregate rows the store keeps of each
3272    /// pending mutation. The snapshot and the mutations are written through
3273    /// as they come, and the mutations replayed at launch; JS persists its
3274    /// commits now and then, running the mutations again for it, so the
3275    /// aggregates they move are the ones registered at that persist, not
3276    /// when they ran. A persist runs the mutations it hasn't persisted over
3277    /// what it persisted before, or, when the snapshot is newer, all of
3278    /// them over it, and stores the aggregate rows each wrote.
3279    fn persist(&self) {
3280        self.persist_timer.borrow_mut().take();
3281        self.last_persist_at
3282            .set(Some(self.scheduler.monotonic_now()));
3283        let (pending, snapshot_cookie, snapshot_map, store_main, pending_writes) = {
3284            let store = self.store.borrow();
3285            (
3286                store.pending.clone(),
3287                store.snapshot.cookie.clone(),
3288                store.snapshot.map.clone(),
3289                store.main.clone(),
3290                store.pending_writes(),
3291            )
3292        };
3293        let to_run: Vec<PendingMutation> = {
3294            let mut persisted = self.persisted.borrow_mut();
3295            let to_run = if persisted.cookie != snapshot_cookie {
3296                persisted.cookie = snapshot_cookie;
3297                persisted.main = snapshot_map;
3298                persisted.touched.clear();
3299                pending.clone()
3300            } else {
3301                pending
3302                    .iter()
3303                    .filter(|m| {
3304                        !persisted
3305                            .mutations
3306                            .contains(&mutation_key(&m.client_id, m.id))
3307                    })
3308                    .cloned()
3309                    .collect()
3310            };
3311            persisted.mutations = pending
3312                .iter()
3313                .map(|m| mutation_key(&m.client_id, m.id))
3314                .collect();
3315            to_run
3316        };
3317        // The persist runs over a fork of the views' branch moved to what
3318        // it persisted. JS's fork throws once that branch has failed, and
3319        // the persist with it; the branch's sources no longer follow the
3320        // replica.
3321        if to_run.is_empty() || self.context.branch_failure().is_some() {
3322            return;
3323        }
3324        let (persisted_main, persisted_touched) = {
3325            let p = self.persisted.borrow();
3326            (p.main.clone(), p.touched.clone())
3327        };
3328        let branch = self.ivm_main.fork();
3329        let keys: HashSet<String> = pending_writes.union(&persisted_touched).cloned().collect();
3330        let moved = diff_keys(&store_main, &persisted_main, &keys);
3331        if capture(|| branch.apply(&moved, &HashMap::new())).is_err() {
3332            return;
3333        }
3334        let mut ops = Vec::new();
3335        let group = self.client_group_id.clone();
3336        let (main, _, touched, _) = self.replay(
3337            &to_run,
3338            &persisted_main,
3339            branch,
3340            &HashMap::new(),
3341            &mut |m, write| {
3342                ops.push(StorageOp::SetAggregateWrites {
3343                    group: group.clone(),
3344                    client_id: m.client_id.clone(),
3345                    id: m.id,
3346                    writes: write.map(aggregate_writes).unwrap_or_default(),
3347                });
3348            },
3349        );
3350        {
3351            let mut p = self.persisted.borrow_mut();
3352            p.main = main;
3353            p.touched.extend(touched);
3354        }
3355        self.database.apply(ops);
3356    }
3357
3358    /// Rejects a poke whose `update` ops leave a row under a key that isn't
3359    /// its primary key: one whose merge drops or changes the key's columns.
3360    /// The table's source would then hold rows that disagree with the
3361    /// replica, which JS's `MemorySource` reports by throwing ("Row already
3362    /// exists", "Row not found") and the poke handler by disconnecting.
3363    fn check_updated_rows(&self, patch: &[KvPatchOp], write: &KvWrite) -> Result<(), String> {
3364        let mut keyless: HashMap<String, Vec<Row>> = HashMap::new();
3365        for op in patch {
3366            let KvPatchOp::Update { key, .. } = op else {
3367                continue;
3368            };
3369            let Some(value) = write.get(key) else {
3370                continue;
3371            };
3372            let table = keys::source_name(key);
3373            let Some(t) = self.schema.tables.get(&table) else {
3374                continue;
3375            };
3376            let row = value.as_object().cloned().unwrap_or_default();
3377            if let Ok(row_key) = keys::primary_key_string(&table, &t.primary_key, &row) {
3378                if row_key != *key {
3379                    return Err(format!(
3380                        "Error: update of {key} changes its primary key to {row_key}"
3381                    ));
3382                }
3383                continue;
3384            }
3385            // JS keeps a row whose merge leaves it without a usable primary
3386            // key, and fails once a second one collides with it in the
3387            // source.
3388            let Some(source) = self.ivm_main.source(&table) else {
3389                continue;
3390            };
3391            let row: Row = Rc::new(row);
3392            let previous = self
3393                .store
3394                .borrow()
3395                .main
3396                .get(key)
3397                .map(|v| Rc::new(v.as_object().cloned().unwrap_or_default()));
3398            let replaces_itself = previous.is_some_and(|p| source.same_key(&p, &row));
3399            let rows = keyless.entry(table).or_default();
3400            if (!replaces_itself && source.contains(&row))
3401                || rows.iter().any(|r| source.same_key(r, &row))
3402            {
3403                return Err(format!("Error: Row already exists: update of {key}"));
3404            }
3405            rows.push(row);
3406        }
3407        Ok(())
3408    }
3409}
3410
3411impl Drop for Engine {
3412    fn drop(&mut self) {
3413        self.event_loop.stop();
3414        for t in self.lifecycle_timers.get_mut().drain(..) {
3415            t.cancel();
3416        }
3417        if !self.rep_closed.get() {
3418            self.database.client_stopped(&self.client_id);
3419            self.database.remove_failure_listener(&self.client_id);
3420            self.database.release();
3421        }
3422    }
3423}
3424
3425#[cfg(test)]
3426mod tests {
3427    use super::*;
3428
3429    #[test]
3430    fn iso_dates() {
3431        assert_eq!(
3432            iso_date_string(1_700_000_000_000.0),
3433            "2023-11-14T22:13:20.000Z"
3434        );
3435        assert_eq!(iso_date_string(0.0), "1970-01-01T00:00:00.000Z");
3436        assert_eq!(iso_date_string(-1.0), "1969-12-31T23:59:59.999Z");
3437    }
3438
3439    #[test]
3440    fn backoff() {
3441        assert_eq!(reconnect_backoff_ms(1, 5000.0, 0.0), 250.0);
3442        assert_eq!(reconnect_backoff_ms(1, 5000.0, 1.0), 500.0);
3443        assert_eq!(reconnect_backoff_ms(4, 5000.0, 0.5), 3000.0);
3444        assert_eq!(reconnect_backoff_ms(5, 5000.0, 0.5), 5000.0);
3445    }
3446}