Skip to main content

zero_client/client/
engine.rs

1//! The client engine (`Zero` in `packages/zero-client/src/client/zero.ts`,
2//! with the parts of Replicache it uses): the run loop that keeps a
3//! connection, the replica and its pending mutations, and the query and
4//! mutation bookkeeping. Everything runs synchronously on one thread; see
5//! `event_loop`.
6
7use super::connection_manager::{ConnectionManager, State};
8use super::database::{AggregateWrites, ReplicaDatabase, StorageOp, StoredMutation, base36};
9use super::event_loop::{EventLoop, LoopScheduler};
10use super::ivm_branch::IvmBranch;
11use super::log::Log;
12use super::mutation_tracker::MutationTracker;
13use super::poke_handler::{MergedPoke, PokeHandler};
14use super::query_context::QueryContext;
15use super::query_manager::QueryManager;
16use super::replica_store::{
17    KvDiff, KvMap, KvPatchOp, KvWrite, PendingMutation, ReplicaStore, Snapshot, apply_patch,
18    compare_cookies, diff_all, diff_keys, with_prefix,
19};
20use super::socket::{SocketConnection, SocketEvents, SocketFactory, SocketMessage};
21use super::storage::{LoadError, StorageFailure, ZeroStorage};
22use super::transaction::{Reason, Transaction};
23use super::{
24    ApplicationError, ClientErrorKind, ConnectionError, ConnectionStatus, ConnectionTransition,
25    Deferred, Error, LogLevel, LogSink, Scheduler, StderrLogSink, Timer,
26};
27use crate::decode::{DecodeError, JsonCodable};
28use crate::hash::xxhash32;
29use crate::ivm::{Row, capture};
30use crate::json::{self, Object, Value, number};
31use crate::keys::{self, AGGREGATES_PREFIX, GOT_QUERIES_PREFIX, MUTATIONS_PREFIX};
32use crate::map::OrderedMap;
33use crate::protocol::{
34    ChangeDesiredQueriesBody, ConnectedBody, ConnectionUrl, DeleteClientsBody, Downstream,
35    ErrorBody, ErrorKind, InitConnectionBody, Mutation, PushBody, PushResponseBody,
36    UpQueriesPatchOp, UpdateAuthBody, Upstream, poke_chunk, sec_protocols,
37};
38use crate::schema::Schema;
39use crate::strings;
40use std::cell::{Cell, RefCell};
41use std::collections::{HashMap, HashSet, VecDeque};
42use std::hash::{BuildHasher, Hasher};
43use std::rc::{Rc, Weak};
44use std::sync::Arc;
45
46/// `RUN_LOOP_INTERVAL_MS`: the backoff before retrying a failed connection.
47pub(crate) const RUN_LOOP_INTERVAL_MS: f64 = 5_000.0;
48/// `FAST_RECONNECT_INITIAL_MS`: the longest wait before the first retry
49/// after a transport failure. Each further consecutive failure doubles it,
50/// up to `RUN_LOOP_INTERVAL_MS`.
51const FAST_RECONNECT_INITIAL_MS: f64 = 500.0;
52/// `STABLE_CONNECTION_MS`: how long a connection must stay up before a
53/// later failure starts the fast schedule from the beginning again.
54const STABLE_CONNECTION_MS: f64 = 5_000.0;
55/// `CONNECT_TIMEOUT_MS`: each phase of a connection attempt gets this long.
56const CONNECT_TIMEOUT_MS: f64 = 10_000.0;
57/// `HEARTBEAT_INTERVAL`.
58const HEARTBEAT_INTERVAL_MS: f64 = 60_000.0;
59/// `GC_INTERVAL`, of clients and of client groups.
60const GC_INTERVAL_MS: f64 = 5.0 * 60_000.0;
61/// `CLIENT_MAX_INACTIVE_TIME`.
62const CLIENT_MAX_INACTIVE_MS: f64 = 24.0 * 60.0 * 60_000.0;
63/// `INITIAL_COLLECT_IDB_DELAY`.
64const INITIAL_COLLECT_DATABASES_DELAY_MS: f64 = 5.0 * 60_000.0;
65/// `COLLECT_IDB_INTERVAL`.
66const COLLECT_DATABASES_INTERVAL_MS: f64 = 12.0 * 60.0 * 60_000.0;
67/// `PERSIST_IDLE_TIMEOUT_MS` and `PERSIST_THROTTLE_MS`.
68const PERSIST_IDLE_TIMEOUT_MS: f64 = 1000.0;
69const PERSIST_THROTTLE_MS: f64 = 500.0;
70
71/// `reconnectBackoffMs`: the wait before retrying after `failed_attempts`
72/// consecutive transport failures (1 for the first). The ceiling starts at
73/// `FAST_RECONNECT_INITIAL_MS` and doubles with each failure, and the wait
74/// is between half the ceiling and the ceiling, so that clients that lost
75/// the same server don't all retry at once. Once the ceiling reaches
76/// `max_ms`, the wait is `max_ms`.
77pub(crate) fn reconnect_backoff_ms(failed_attempts: u32, max_ms: f64, random: f64) -> f64 {
78    let ceiling = FAST_RECONNECT_INITIAL_MS * 2f64.powi(failed_attempts as i32 - 1);
79    if ceiling >= max_ms {
80        return max_ms;
81    }
82    (ceiling / 2.0 + random * ceiling / 2.0).round()
83}
84
85thread_local! {
86    static RNG: Cell<u64> = Cell::new({
87        let mut h = std::collections::hash_map::RandomState::new().build_hasher();
88        h.write_u128(
89            std::time::SystemTime::now()
90                .duration_since(std::time::UNIX_EPOCH)
91                .map(|d| d.as_nanos())
92                .unwrap_or(0),
93        );
94        h.finish() | 1
95    });
96}
97
98/// A random 64-bit number (xorshift64*, seeded per thread from the OS).
99fn random_u64() -> u64 {
100    RNG.with(|r| {
101        let mut x = r.get();
102        x ^= x >> 12;
103        x ^= x << 25;
104        x ^= x >> 27;
105        r.set(x);
106        x.wrapping_mul(0x2545_F491_4F6C_DD1D)
107    })
108}
109
110/// `Math.random()`.
111fn random_f64() -> f64 {
112    (random_u64() >> 11) as f64 / (1u64 << 53) as f64
113}
114
115/// A random ID like the JS client's `nanoid`.
116pub(crate) fn random_id() -> String {
117    const ALPHABET: &[u8; 64] = b"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-";
118    (0..21)
119        .map(|_| ALPHABET[(random_u64() % 64) as usize] as char)
120        .collect()
121}
122
123/// `Date.prototype.toISOString` of a time in milliseconds since 1970.
124pub(crate) fn iso_date_string(ms: f64) -> String {
125    let t = ms.floor() as i64;
126    let mut days = t.div_euclid(86_400_000);
127    let ms_of_day = t.rem_euclid(86_400_000);
128    // Howard Hinnant's civil_from_days.
129    days += 719_468;
130    let era = days.div_euclid(146_097);
131    let doe = days - era * 146_097;
132    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
133    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
134    let mp = (5 * doy + 2) / 153;
135    let d = doy - (153 * mp + 2) / 5 + 1;
136    let m = if mp < 10 { mp + 3 } else { mp - 9 };
137    let y = yoe + era * 400 + i64::from(m <= 2);
138    format!(
139        "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}.{:03}Z",
140        ms_of_day / 3_600_000,
141        ms_of_day / 60_000 % 60,
142        ms_of_day / 1000 % 60,
143        ms_of_day % 1000
144    )
145}
146
147/// The message valita gives for a message that doesn't parse (`getMessage`
148/// in `packages/shared/src/valita.ts`), for the cases the client can tell
149/// apart.
150fn valita_message(e: &DecodeError, text: &str) -> String {
151    if e.message.starts_with("Unknown message type") || e.message == "Expected [tag, body] tuple" {
152        return format!("Invalid union value: {text}");
153    }
154    e.valita_message()
155}
156
157/// The options of a Zero client (`ZeroOptions` in
158/// `packages/zero-client/src/client/options.ts`).
159#[derive(Clone)]
160pub struct ZeroOptions {
161    /// The zero-cache URL, `http` or `https`. Without one the client runs
162    /// locally and never connects.
163    pub cache_url: Option<String>,
164    /// The user the client syncs for. `None` for a logged-out client.
165    pub user_id: Option<String>,
166    /// The auth token zero-cache and the API server see.
167    pub auth: Option<String>,
168    /// Distinguishes local stores of the same user and URLs.
169    pub storage_key: Option<String>,
170    /// How long the client waits for a pong before it reconnects.
171    pub ping_timeout_ms: f64,
172    /// How long the client tries to reconnect before the state is
173    /// `Disconnected`.
174    pub disconnect_timeout_ms: f64,
175    /// How long query changes are batched before they are sent.
176    pub query_change_throttle_ms: f64,
177    /// The longest `Sec-WebSocket-Protocol` header the client sends.
178    /// `initConnection` is sent as a message instead when it doesn't fit.
179    pub max_header_length: usize,
180    /// The API server's push URL, when it differs from the one zero-cache is
181    /// configured with.
182    pub mutate_url: Option<String>,
183    /// Headers zero-cache sends to `mutate_url`.
184    pub mutate_headers: Option<Vec<(String, String)>>,
185    /// The API server's query URL, when it differs from the one zero-cache is
186    /// configured with.
187    pub query_url: Option<String>,
188    /// Headers zero-cache sends to `query_url`.
189    pub query_headers: Option<Vec<(String, String)>>,
190    /// The server needs a newer client. Defaults to doing nothing.
191    pub on_update_needed: Option<UpdateNeededHandler>,
192    /// The server no longer has the client's state, or the stored replica
193    /// didn't decode and the local store was reset. Without it, a client
194    /// the server lost replaces itself by a new one on the same store, as
195    /// JS reloads the page (see [`Zero::new`]).
196    ///
197    /// [`Zero::new`]: crate::client::Zero::new
198    pub on_client_state_not_found: Option<Rc<dyn Fn()>>,
199    /// Where the client keeps its replica. Defaults to
200    /// [`ZeroStorage::Memory`], which keeps nothing across restarts.
201    pub storage: ZeroStorage,
202    /// The local store failed, so the client runs on what it has in memory
203    /// and persists nothing more (`onStorageFailure`). Called once.
204    pub on_storage_failure: Option<StorageFailureHandler>,
205    /// The least severe messages logged.
206    pub log_level: LogLevel,
207    /// Where messages are logged. Defaults to [`StderrLogSink`].
208    pub log_sink: Option<Rc<dyn LogSink>>,
209    /// How long the app is hidden before the client disconnects
210    /// (`hiddenTabDisconnectDelay`); see [`Zero::set_visible`].
211    ///
212    /// [`Zero::set_visible`]: crate::client::Zero::set_visible
213    pub hidden_disconnect_delay_ms: f64,
214}
215
216impl Default for ZeroOptions {
217    fn default() -> Self {
218        ZeroOptions {
219            cache_url: None,
220            user_id: None,
221            auth: None,
222            storage_key: None,
223            ping_timeout_ms: 5_000.0,
224            disconnect_timeout_ms: 60_000.0,
225            query_change_throttle_ms: 10.0,
226            max_header_length: 8 * 1024,
227            mutate_url: None,
228            mutate_headers: None,
229            query_url: None,
230            query_headers: None,
231            on_update_needed: None,
232            on_client_state_not_found: None,
233            storage: ZeroStorage::Memory,
234            on_storage_failure: None,
235            log_level: LogLevel::Warn,
236            log_sink: None,
237            hidden_disconnect_delay_ms: 0.0,
238        }
239    }
240}
241
242/// Called when the local store fails (`onStorageFailure`).
243pub type StorageFailureHandler = Rc<dyn Fn(&StorageFailure)>;
244
245/// Called when the server needs a newer client (`onUpdateNeeded`).
246pub type UpdateNeededHandler = Rc<dyn Fn(&UpdateNeededReason)>;
247
248/// Why the client must be updated (`UpdateNeededReason`).
249#[derive(Clone, Debug, PartialEq, Eq)]
250pub struct UpdateNeededReason {
251    /// `VersionNotSupported`, `SchemaVersionNotSupported`, or
252    /// `NewClientGroup`.
253    pub type_: String,
254    /// The server's explanation, if it gave one.
255    pub message: Option<String>,
256}
257
258impl UpdateNeededReason {
259    /// The JS reason object.
260    pub fn to_json(&self) -> Value {
261        let mut o = Object::new();
262        o.insert("type", Value::String(self.type_.clone()));
263        if let Some(m) = &self.message {
264            o.insert("message", Value::String(m.clone()));
265        }
266        Value::Object(o)
267    }
268}
269
270/// How a mutation ended, on the client or on the server
271/// (`MutatorResultDetails` in `packages/zero-client/src/client/custom.ts`).
272#[derive(Clone, Debug, PartialEq)]
273pub enum MutationOutcome {
274    /// The mutation ran.
275    Success {
276        /// What the mutator returned (client) or the server's result.
277        data: Option<Value>,
278    },
279    /// The mutator or the API server raised an [`ApplicationError`].
280    AppError {
281        /// The error's message.
282        message: String,
283        /// The error's details.
284        details: Option<Value>,
285    },
286    /// Zero couldn't run the mutation or learn its result, such as while
287    /// offline or when the connection closed. On the server side this
288    /// doesn't mean the mutation wasn't applied: the client sends pending
289    /// mutations again when it reconnects.
290    ZeroError {
291        /// Why.
292        message: String,
293    },
294}
295
296impl MutationOutcome {
297    /// The JS result object.
298    pub fn to_json(&self) -> Value {
299        let mut o = Object::new();
300        match self {
301            MutationOutcome::Success { data } => {
302                o.insert("type", Value::from("success"));
303                if let Some(d) = data {
304                    o.insert("data", d.clone());
305                }
306            }
307            MutationOutcome::AppError { message, details } => {
308                let mut e = Object::new();
309                e.insert("type", Value::from("app"));
310                e.insert("message", Value::String(message.clone()));
311                if let Some(d) = details {
312                    e.insert("details", d.clone());
313                }
314                o.insert("type", Value::from("error"));
315                o.insert("error", Value::Object(e));
316            }
317            MutationOutcome::ZeroError { message } => {
318                let mut e = Object::new();
319                e.insert("type", Value::from("zero"));
320                e.insert("message", Value::String(message.clone()));
321                o.insert("type", Value::from("error"));
322                o.insert("error", Value::Object(e));
323            }
324        }
325        Value::Object(o)
326    }
327}
328
329/// What `mutate` returns: the outcome of the optimistic run on the client,
330/// and the server's.
331#[derive(Clone)]
332pub struct MutatorResult {
333    /// Settles when the mutator has run on the client: what it returned, or
334    /// what it raised.
335    pub client: Deferred<MutationOutcome>,
336    /// Settles when the server has run the mutation, or when the client can't
337    /// learn its result.
338    pub server: Deferred<MutationOutcome>,
339}
340
341/// A mutator bound to the app's context: runs it in a transaction.
342pub(crate) type MutatorFn =
343    Rc<dyn Fn(&Transaction, Option<&Value>) -> Result<Option<Value>, Error>>;
344
345/// A connection's readiness, as the pusher waits on it
346/// (`#connectionReadyResolver`).
347#[derive(Clone, Default)]
348struct ReadyPromise(Rc<RefCell<ReadyState>>);
349
350#[derive(Default)]
351struct ReadyState {
352    settled: Option<bool>,
353    waiters: Vec<Box<dyn FnOnce(bool)>>,
354}
355
356impl ReadyPromise {
357    fn is(&self, other: &ReadyPromise) -> bool {
358        Rc::ptr_eq(&self.0, &other.0)
359    }
360
361    fn settle(&self, value: bool, event_loop: &EventLoop) {
362        let waiters = {
363            let mut s = self.0.borrow_mut();
364            if s.settled.is_some() {
365                return;
366            }
367            s.settled = Some(value);
368            std::mem::take(&mut s.waiters)
369        };
370        for w in waiters {
371            event_loop.microtask(move || w(value));
372        }
373    }
374
375    /// Calls `waiter` in a microtask once settled.
376    fn on_settle(&self, event_loop: &EventLoop, waiter: impl FnOnce(bool) + 'static) {
377        let settled = self.0.borrow().settled;
378        match settled {
379            Some(v) => event_loop.microtask(move || waiter(v)),
380            None => self.0.borrow_mut().waiters.push(Box::new(waiter)),
381        }
382    }
383}
384
385#[derive(Default)]
386struct ConnectAttempt {
387    events: RefCell<Vec<(f64, String)>>,
388    aborted: Cell<bool>,
389    abort_reason: RefCell<Option<ConnectionError>>,
390    timeout: RefCell<Option<Box<dyn Timer>>>,
391    finished: Cell<bool>,
392    on_connected: RefCell<Option<Box<dyn FnOnce()>>>,
393    /// When the socket was opened, for the open and `connected` logs.
394    start: Cell<f64>,
395}
396
397impl ConnectAttempt {
398    fn cancel_timeout(&self) {
399        if let Some(t) = self.timeout.borrow_mut().take() {
400            t.cancel();
401        }
402    }
403}
404
405/// A pending mutation's identity.
406type MutationKey = (String, u64);
407
408fn mutation_key(client_id: &str, id: f64) -> MutationKey {
409    (client_id.to_owned(), id.to_bits())
410}
411
412/// The replica as JS's last persist left it in the store: the snapshot it
413/// persisted (by cookie), its main map over that snapshot, the keys where
414/// the two may differ, and the pending mutations it ran.
415#[derive(Default)]
416struct PersistedState {
417    cookie: Value,
418    main: KvMap,
419    touched: HashSet<String>,
420    mutations: HashSet<MutationKey>,
421}
422
423/// The engine's end of one socket.
424pub(crate) struct SocketHandle {
425    me: Weak<SocketHandle>,
426    engine: Weak<Engine>,
427    connection: RefCell<Option<Box<dyn SocketConnection>>>,
428    closed: Cell<bool>,
429}
430
431impl SocketHandle {
432    fn new(engine: Weak<Engine>) -> Rc<Self> {
433        Rc::new_cyclic(|me| SocketHandle {
434            me: me.clone(),
435            engine,
436            connection: RefCell::new(None),
437            closed: Cell::new(false),
438        })
439    }
440
441    fn send(&self, text: &str) {
442        if self.closed.get() {
443            return;
444        }
445        if let Some(c) = &*self.connection.borrow() {
446            c.send(text);
447        }
448    }
449
450    fn close(&self, code: Option<u16>) {
451        if self.closed.replace(true) {
452            return;
453        }
454        let connection = self.connection.borrow_mut().take();
455        if let Some(c) = connection {
456            c.close(code);
457        }
458    }
459}
460
461impl SocketEvents for SocketHandle {
462    fn did_open(&self) {
463        if self.closed.get() {
464            return;
465        }
466        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
467            e.socket_did_open(&me);
468        }
469    }
470
471    fn did_receive(&self, message: SocketMessage) {
472        if self.closed.get() {
473            return;
474        }
475        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
476            e.socket_did_receive(&me, message);
477        }
478    }
479
480    fn did_close(&self, code: u16, reason: &str, was_clean: bool) {
481        if self.closed.get() {
482            return;
483        }
484        if let (Some(e), Some(me)) = (self.engine.upgrade(), self.me.upgrade()) {
485            e.socket_did_close(&me, code, reason, was_clean);
486        }
487    }
488}
489
490type VisibilityWaiter = (bool, u64, Box<dyn FnOnce()>);
491
492pub(crate) struct Engine {
493    me: Weak<Engine>,
494    pub(crate) schema: Arc<Schema>,
495    pub(crate) options: ZeroOptions,
496    pub(crate) client_id: String,
497    pub(crate) client_group_id: String,
498    profile_id: String,
499    pub(crate) user_id: Option<String>,
500    pub(crate) event_loop: Rc<EventLoop>,
501    scheduler: Rc<dyn Scheduler>,
502    pub(crate) log: Log,
503    socket_factory: Option<Rc<dyn SocketFactory>>,
504    socket_origin: Option<String>,
505    mutators: Rc<HashMap<String, MutatorFn>>,
506    client_schema: crate::protocol::ClientSchema,
507
508    store: RefCell<ReplicaStore>,
509    pub(crate) database: Rc<ReplicaDatabase>,
510    lifecycle_timers: RefCell<Vec<Box<dyn Timer>>>,
511    ivm_main: Rc<IvmBranch>,
512    pub(crate) context: Rc<QueryContext>,
513    pub(crate) connection_manager: Rc<ConnectionManager>,
514    mutation_tracker: Rc<MutationTracker>,
515    query_manager: Rc<QueryManager>,
516    poke_handler: RefCell<PokeHandler>,
517
518    /// The current token: `options.auth`, then the last one `connect`
519    /// set; empty for none.
520    pub(crate) auth: RefCell<String>,
521    rep_closed: Cell<bool>,
522    storage_failure_reported: Cell<bool>,
523    socket: RefCell<Option<Rc<SocketHandle>>>,
524    connect_cookie: RefCell<Value>,
525    init_connection_queries: RefCell<Option<OrderedMap<UpQueriesPatchOp>>>,
526    /// Deleted clients still to tell the server about on this connection.
527    deleted_clients: RefCell<Option<DeleteClientsBody>>,
528    last_mutation_id_received: Cell<f64>,
529    last_mutation_ids_sent: RefCell<HashMap<String, f64>>,
530    persisted: RefCell<PersistedState>,
531    persist_timer: RefCell<Option<Box<dyn Timer>>>,
532    last_persist_at: Cell<Option<f64>>,
533    additional_connect_params: RefCell<Option<Vec<(String, String)>>>,
534    /// When the current connection got the server's `connected`.
535    connected_at: Cell<Option<f64>>,
536    /// Consecutive transport failures, which set the reconnect backoff.
537    transport_failures: Cell<u32>,
538    current_attempt: RefCell<Option<Rc<ConnectAttempt>>>,
539    connection_ready: RefCell<ReadyPromise>,
540    recent_sent_messages: RefCell<VecDeque<Value>>,
541    mutation_rejection: RefCell<Option<ConnectionError>>,
542    client_group_disabled: Cell<bool>,
543
544    // The run loop's current wait. Each wait gets a new token, so a wait
545    // that was superseded does nothing when it ends.
546    loop_token: Cell<u64>,
547    abort_ping_timeout: RefCell<Rc<dyn Fn()>>,
548    on_pong: RefCell<Rc<dyn Fn()>>,
549    /// Pings the server now, when the run loop waits while connected.
550    ping_now: RefCell<Rc<dyn Fn()>>,
551    /// The run loop's backoff before it reconnects.
552    backoff: RefCell<Option<(u64, Box<dyn Timer>)>>,
553
554    // The push loop (Replicache's push `ConnectionLoop`).
555    push_timer: RefCell<Option<Box<dyn Timer>>>,
556    push_in_flight: Cell<bool>,
557    push_requested: Cell<bool>,
558    push_request_counter: Cell<u64>,
559
560    /// Trails the app's visibility by `hidden_disconnect_delay_ms` when it
561    /// is hidden (`DocumentVisibilityWatcher`).
562    pub(crate) visible: Cell<bool>,
563    hidden_timer: RefCell<Option<Box<dyn Timer>>>,
564    visibility_waiters: RefCell<Vec<VisibilityWaiter>>,
565    pending_waiters: RefCell<Vec<Box<dyn FnOnce()>>>,
566
567    /// Set while a mutator runs, for a mutation or a rebase: Replicache's
568    /// write lock. Mutations that start meanwhile wait, in order.
569    write_locked: Cell<bool>,
570    waiting_for_write_lock: RefCell<VecDeque<Box<dyn FnOnce()>>>,
571
572    /// What `Zero` does when the client can't sync any longer: by default
573    /// `ZeroOptions::on_client_state_not_found`.
574    pub(crate) on_client_state_not_found: RefCell<Option<Rc<dyn Fn()>>>,
575    /// What `Zero` does when the app must be updated to sync, besides
576    /// calling `ZeroOptions::on_update_needed`.
577    pub(crate) on_update_needed: RefCell<Option<UpdateNeededHandler>>,
578}
579
580fn noop() -> Rc<dyn Fn()> {
581    Rc::new(|| {})
582}
583
584/// `JSON.stringify(recentSentMessages, null, 2)`.
585fn pretty_recent_messages(messages: &VecDeque<Value>) -> String {
586    let entries: Vec<String> = messages
587        .iter()
588        .map(|m| {
589            let fields: Vec<String> = ["type", "size", "snippet"]
590                .iter()
591                .filter_map(|k| {
592                    m.get(k).map(|v| {
593                        format!(
594                            "    {}: {}",
595                            json::stringify(&Value::from(*k)),
596                            json::stringify(v)
597                        )
598                    })
599                })
600                .collect();
601            format!("  {{\n{}\n  }}", fields.join(",\n"))
602        })
603        .collect();
604    format!("[\n{}\n]", entries.join(",\n"))
605}
606
607fn bad_order_message(name: &str, received: &str, last: &str) -> String {
608    format!(
609        "Received {name} {received} is < than last snapshot {name} {last}; ignoring client view"
610    )
611}
612
613/// The synced aggregate rows `write` wrote, with their values now.
614fn aggregate_writes(write: &KvWrite) -> AggregateWrites {
615    let mut writes: AggregateWrites = write
616        .writes
617        .iter()
618        .filter(|k| k.starts_with(AGGREGATES_PREFIX))
619        .map(|k| (k.clone(), write.get(k).cloned()))
620        .collect();
621    writes.sort_by(|a, b| a.0.cmp(&b.0));
622    writes
623}
624
625impl Engine {
626    pub(crate) fn new(
627        schema: Arc<Schema>,
628        options: ZeroOptions,
629        mutators: Rc<HashMap<String, MutatorFn>>,
630        base: Rc<dyn Scheduler>,
631        socket_factory: Option<Rc<dyn SocketFactory>>,
632        visible: bool,
633    ) -> Result<Rc<Engine>, Error> {
634        if options.user_id.as_deref() == Some("") {
635            return Err(ConnectionError::client(
636                ClientErrorKind::Internal,
637                "ZeroOptions.userID should not be empty. Omit it entirely for logged-out clients.",
638            )
639            .into());
640        }
641        Self::check_auth_valid(options.user_id.as_deref(), options.auth.as_deref())?;
642        let socket_origin = options
643            .cache_url
644            .as_deref()
645            .map(Self::socket_origin)
646            .transpose()?;
647        let event_loop = EventLoop::new(base.clone());
648        let scheduler: Rc<dyn Scheduler> = Rc::new(LoopScheduler {
649            base: base.clone(),
650            event_loop: Rc::downgrade(&event_loop),
651        });
652        let client_id = random_id();
653        let sink = options
654            .log_sink
655            .clone()
656            .unwrap_or_else(|| Rc::new(StderrLogSink));
657        let log = Log::new(options.log_level, sink).with("clientID", &client_id);
658        let client_schema = schema.client_schema();
659        let schema_hash = base36(xxhash32::h64(&json::stringify(
660            &client_schema.normalized().to_json(),
661        )));
662
663        // Open the database and join a client group (`initClientV6`).
664        let name = ReplicaDatabase::name(
665            options.user_id.as_deref(),
666            options.storage_key.as_deref().unwrap_or(""),
667            options.mutate_url.as_deref(),
668            options.query_url.as_deref(),
669            &schema_hash,
670        );
671        let mut mutator_names: Vec<String> = mutators.keys().cloned().collect();
672        mutator_names.sort();
673        let (mut database, mut open_failure, mut corrupt) =
674            ReplicaDatabase::open(&options.storage, &name);
675        let joined = match database.init_client(&client_id, &mutator_names, base.now()) {
676            // What the client group holds doesn't decode, so the client
677            // state is lost: drop the database and start a new one
678            // (`#handleInvalidRefCount`).
679            Err(LoadError::Corrupt(_)) => {
680                corrupt = true;
681                database.drop_database();
682                database.release();
683                (database, open_failure, _) = ReplicaDatabase::open(&options.storage, &name);
684                database.init_client(&client_id, &mutator_names, base.now())
685            }
686            r => r,
687        };
688        let (client_group_id, group) = match joined {
689            Ok(j) => j,
690            // The client group can't be read: run on memory, as when the
691            // open fails.
692            Err(e) => {
693                database.release();
694                database = ReplicaDatabase::fallback(&name, &options.storage);
695                open_failure = Some(match e {
696                    LoadError::Failure(f) => f,
697                    LoadError::Corrupt(m) => {
698                        StorageFailure::new(super::storage::StorageFailureKind::IoError, m)
699                    }
700                });
701                database
702                    .init_client(&client_id, &mutator_names, base.now())
703                    .expect("a memory database reads")
704            }
705        };
706        let profile_id = database.profile_id();
707        // Drop stored mutation IDs that aren't mutation IDs (a corrupt
708        // store) rather than trust them.
709        let is_mutation_id = |id: f64| id >= 0.0 && number::is_safe_integer(id);
710        let last_mutation_ids = group
711            .last_mutation_ids
712            .into_iter()
713            .filter(|(cid, lmid)| {
714                let ok = is_mutation_id(*lmid);
715                if !ok {
716                    log.error(|| {
717                        format!(
718                            "Dropping invalid stored last mutation ID of {cid}: {}",
719                            number::format(*lmid)
720                        )
721                    });
722                }
723                ok
724            })
725            .collect();
726        let mutations: Vec<StoredMutation> = group
727            .mutations
728            .into_iter()
729            .filter(|m| {
730                let ok = is_mutation_id(m.id);
731                if !ok {
732                    log.error(|| {
733                        format!(
734                            "Dropping stored mutation \"{}\" of {} with invalid ID {}",
735                            m.name,
736                            m.client_id,
737                            number::format(m.id)
738                        )
739                    });
740                }
741                ok
742            })
743            .collect();
744        let store = ReplicaStore::new(
745            Snapshot {
746                map: group.entries,
747                cookie: group.cookie,
748                last_mutation_ids,
749            },
750            mutations
751                .iter()
752                .map(|m| PendingMutation {
753                    id: m.id,
754                    client_id: m.client_id.clone(),
755                    name: m.name.clone(),
756                    args: m.args.clone(),
757                    timestamp: m.timestamp,
758                    writes: HashSet::new(),
759                })
760                .collect(),
761        );
762        let stored_aggregate_writes: HashMap<MutationKey, AggregateWrites> = mutations
763            .into_iter()
764            .map(|m| (mutation_key(&m.client_id, m.id), m.aggregate_writes))
765            .collect();
766        let ivm_main = IvmBranch::new(schema.clone());
767        let connection_manager = ConnectionManager::new(
768            options.disconnect_timeout_ms,
769            scheduler.clone(),
770            Rc::downgrade(&event_loop),
771        );
772
773        let engine = Rc::new_cyclic(|me: &Weak<Engine>| {
774            let mutation_tracker = {
775                let (a, f, q) = (me.clone(), me.clone(), me.clone());
776                Rc::new(MutationTracker::new(
777                    client_id.clone(),
778                    Box::new(move |id| {
779                        if let Some(e) = a.upgrade() {
780                            e.send_if_connected(Upstream::AckMutationResponses(id));
781                        }
782                    }),
783                    Box::new(move |error| {
784                        if let Some(e) = f.upgrade() {
785                            e.disconnect(error, None);
786                        }
787                    }),
788                    Box::new(move || {
789                        if let Some(e) = q.upgrade() {
790                            e.query_manager.on_all_mutations_applied();
791                        }
792                    }),
793                ))
794            };
795            let context = {
796                let (a, u, c) = (me.clone(), me.clone(), me.clone());
797                let l = event_loop.clone();
798                QueryContext::new(
799                    ivm_main.clone(),
800                    Box::new(move |id, ttl, got| match a.upgrade() {
801                        Some(e) => e
802                            .query_manager
803                            .add_custom(&id.name, &id.args, ttl, Some(got)),
804                        None => Box::new(|| {}),
805                    }),
806                    Rc::new(move |id, ttl| {
807                        if let Some(e) = u.upgrade() {
808                            e.query_manager.update_custom(&id.name, &id.args, ttl);
809                        }
810                    }),
811                    Rc::new(move |f| l.microtask(f)),
812                    Some(log.clone()),
813                    Box::new(move || match c.upgrade()?.connection_manager.state() {
814                        State::Closed(r) => Some(r),
815                        _ => None,
816                    }),
817                    true,
818                )
819            };
820            let query_manager = {
821                let (s, p) = (me.clone(), me.clone());
822                QueryManager::new(
823                    client_id.clone(),
824                    scheduler.clone(),
825                    options.query_change_throttle_ms,
826                    Box::new(move |body| {
827                        if let Some(e) = s.upgrade() {
828                            e.send_if_connected(Upstream::ChangeDesiredQueries(body));
829                        }
830                    }),
831                    Box::new(move || p.upgrade().map_or(0, |e| e.mutation_tracker.size())),
832                )
833            };
834            Engine {
835                me: me.clone(),
836                poke_handler: RefCell::new(PokeHandler::new(client_id.clone(), schema.clone())),
837                schema: schema.clone(),
838                user_id: options.user_id.clone(),
839                // Set now, as JS's constructor does, so a token from
840                // `connect` right after init isn't replaced when the run
841                // loop starts.
842                auth: RefCell::new(options.auth.clone().unwrap_or_default()),
843                options: options.clone(),
844                client_id: client_id.clone(),
845                client_group_id: client_group_id.clone(),
846                profile_id,
847                event_loop: event_loop.clone(),
848                scheduler: scheduler.clone(),
849                log: log.clone(),
850                socket_factory,
851                socket_origin,
852                mutators,
853                client_schema,
854                store: RefCell::new(store),
855                database: database.clone(),
856                lifecycle_timers: RefCell::default(),
857                ivm_main: ivm_main.clone(),
858                context,
859                connection_manager: connection_manager.clone(),
860                mutation_tracker,
861                query_manager,
862                rep_closed: Cell::new(false),
863                storage_failure_reported: Cell::new(false),
864                socket: RefCell::new(None),
865                connect_cookie: RefCell::new(Value::Null),
866                init_connection_queries: RefCell::new(None),
867                deleted_clients: RefCell::new(None),
868                last_mutation_id_received: Cell::new(0.0),
869                last_mutation_ids_sent: RefCell::default(),
870                persisted: RefCell::default(),
871                persist_timer: RefCell::new(None),
872                last_persist_at: Cell::new(None),
873                additional_connect_params: RefCell::new(None),
874                connected_at: Cell::new(None),
875                transport_failures: Cell::new(0),
876                current_attempt: RefCell::new(None),
877                connection_ready: RefCell::default(),
878                recent_sent_messages: RefCell::default(),
879                mutation_rejection: RefCell::new(None),
880                client_group_disabled: Cell::new(false),
881                loop_token: Cell::new(0),
882                abort_ping_timeout: RefCell::new(noop()),
883                on_pong: RefCell::new(noop()),
884                ping_now: RefCell::new(noop()),
885                backoff: RefCell::new(None),
886                push_timer: RefCell::new(None),
887                push_in_flight: Cell::new(false),
888                push_requested: Cell::new(false),
889                push_request_counter: Cell::new(0),
890                visible: Cell::new(visible),
891                hidden_timer: RefCell::new(None),
892                visibility_waiters: RefCell::default(),
893                pending_waiters: RefCell::default(),
894                write_locked: Cell::new(false),
895                waiting_for_write_lock: RefCell::default(),
896                on_client_state_not_found: RefCell::new(None),
897                on_update_needed: RefCell::new(None),
898            }
899        });
900
901        let e = &engine;
902        {
903            let w = Rc::downgrade(e);
904            // The closure is dropped with the manager; no need to keep the
905            // unsubscribe.
906            let _ = connection_manager.subscribe(move |state| {
907                if let (Some(e), State::Closed(reason)) = (w.upgrade(), state) {
908                    e.query_manager.handle_closed(reason);
909                }
910            });
911            let w = Rc::downgrade(e);
912            let _ = connection_manager.subscribe(move |state| {
913                if let Some(e) = w.upgrade() {
914                    e.on_connection_state_change(state);
915                }
916            });
917        }
918        e.on_connection_state_change(&connection_manager.state());
919
920        // Load the snapshot into the sources and replay the pending
921        // mutations on it, with the synced aggregate rows they wrote when
922        // they were last persisted, as JS's stored commits have them.
923        let initial = diff_all(&KvMap::default(), &e.store.borrow().snapshot.map);
924        e.ivm_main.apply(&initial, &HashMap::new());
925        let pending = e.store.borrow().pending.clone();
926        if !pending.is_empty() {
927            let snapshot_map = e.store.borrow().snapshot.map.clone();
928            let (main, replayed, touched, _) = e.replay(
929                &pending,
930                &snapshot_map,
931                e.ivm_main.fork(),
932                &stored_aggregate_writes,
933                &mut |_, _| {},
934            );
935            let diff = {
936                let mut store = e.store.borrow_mut();
937                let snapshot = store.snapshot.clone();
938                store.advance(snapshot, main, replayed, Some(&touched))
939            };
940            e.ivm_main.apply(&diff, &HashMap::new());
941        }
942        {
943            let store = e.store.borrow();
944            *e.persisted.borrow_mut() = PersistedState {
945                cookie: store.snapshot.cookie.clone(),
946                main: store.main.clone(),
947                touched: store.pending_writes(),
948                mutations: store
949                    .pending
950                    .iter()
951                    .map(|m| mutation_key(&m.client_id, m.id))
952                    .collect(),
953            };
954        }
955        {
956            let w = Rc::downgrade(e);
957            database.add_failure_listener(
958                &e.client_id,
959                Rc::new(move |f| {
960                    if let Some(e) = w.upgrade() {
961                        e.storage_failed(f, false);
962                    }
963                }),
964            );
965        }
966        // Another client of the group stopped (the `onDelete` of
967        // `makeActiveClientsManager`): tell the server, if connected.
968        {
969            let w = Rc::downgrade(e);
970            database.client_started(
971                &e.client_id,
972                &e.client_group_id,
973                Rc::new(move |id| {
974                    if let Some(e) = w.upgrade() {
975                        let id = id.to_owned();
976                        e.event_loop.clone().dispatch(|| {
977                            e.send_if_connected(Upstream::DeleteClients(DeleteClientsBody {
978                                client_ids: Some(vec![id]),
979                                client_group_ids: None,
980                            }))
981                        });
982                    }
983                }),
984            );
985        }
986        if corrupt {
987            // This client already starts on the new database, which is what
988            // replacing it would do; the app's callback still hears it, as
989            // in JS.
990            e.log.error(|| {
991                format!("Client state is corrupt; dropped database {name} and started a new one")
992            });
993            // Only the app's callback: this client is already on a new
994            // database, so it isn't replaced (as in Swift).
995            e.micro(|e| {
996                if let Some(f) = &e.options.on_client_state_not_found {
997                    f();
998                }
999            });
1000        }
1001        if let Some(f) = open_failure {
1002            e.log.warn(|| {
1003                format!(
1004                    "Local store storage failed ({}) opening {name}; running this instance in memory: {}",
1005                    f.kind.as_str(),
1006                    f.message
1007                )
1008            });
1009            e.micro(move |e| e.storage_failed(&f, true));
1010        }
1011        e.start_lifecycle();
1012
1013        // The replica is loaded: report its got queries and mutation
1014        // results, as the first runs of the store watches do, then build
1015        // the pipelines.
1016        let main = e.store.borrow().main.clone();
1017        e.fire_watchers(&diff_all(&KvMap::default(), &main), true);
1018        e.context.mark_pipelines_ready();
1019        e.micro(|e| e.start_run_loop());
1020        Ok(engine)
1021    }
1022
1023    fn check_auth_valid(user_id: Option<&str>, auth: Option<&str>) -> Result<(), Error> {
1024        if user_id.is_none() && auth.is_some_and(|a| !a.is_empty()) {
1025            return Err(ConnectionError::client(
1026                ClientErrorKind::Internal,
1027                "ZeroOptions.userID is required when auth is set.",
1028            )
1029            .into());
1030        }
1031        Ok(())
1032    }
1033
1034    /// The WebSocket origin of the zero-cache URL (`getServer`,
1035    /// `toWSString`).
1036    fn socket_origin(url: &str) -> Result<String, Error> {
1037        let (scheme, rest) = if let Some(rest) = url.strip_prefix("http://") {
1038            ("ws://", rest)
1039        } else if let Some(rest) = url.strip_prefix("https://") {
1040            ("wss://", rest)
1041        } else {
1042            return Err(ConnectionError::client(
1043                ClientErrorKind::Internal,
1044                "ZeroOptions.cacheURL must use the \"http\" or \"https\" scheme.",
1045            )
1046            .into());
1047        };
1048        let mut rest = rest.to_owned();
1049        if !rest.contains('/') {
1050            rest.push('/');
1051        }
1052        Ok(format!("{scheme}{rest}"))
1053    }
1054
1055    fn micro(&self, f: impl FnOnce(&Engine) + 'static) {
1056        let w = self.me.clone();
1057        self.event_loop.microtask(move || {
1058            if let Some(e) = w.upgrade() {
1059                f(&e);
1060            }
1061        });
1062    }
1063
1064    fn later(&self, ms: f64, f: impl FnOnce(&Engine) + 'static) -> Box<dyn Timer> {
1065        let w = self.me.clone();
1066        self.scheduler.schedule(
1067            ms,
1068            Box::new(move || {
1069                if let Some(e) = w.upgrade() {
1070                    f(&e);
1071                }
1072            }),
1073        )
1074    }
1075
1076    fn status_is(&self, status: ConnectionStatus) -> bool {
1077        self.connection_manager.is(status)
1078    }
1079
1080    // MARK: Sending
1081
1082    fn send_if_connected(&self, message: Upstream) {
1083        if self.status_is(ConnectionStatus::Connected) {
1084            self.send(message);
1085        }
1086    }
1087
1088    fn send(&self, message: Upstream) {
1089        let Some(socket) = self.socket.borrow().clone() else {
1090            return;
1091        };
1092        let text = message.to_text();
1093        if message.tag() != "ping" {
1094            let units: Vec<u16> = text.encode_utf16().collect();
1095            let snippet = if units.len() > 200 {
1096                String::from_utf16(&units[..200]).unwrap_or_else(|_| text.clone())
1097            } else {
1098                text.clone()
1099            };
1100            let mut o = Object::new();
1101            o.insert("type", Value::from(message.tag()));
1102            o.insert("size", Value::Number(units.len() as f64));
1103            o.insert("snippet", Value::String(snippet));
1104            let mut recent = self.recent_sent_messages.borrow_mut();
1105            if recent.len() >= 5 {
1106                recent.pop_front();
1107            }
1108            recent.push_back(Value::Object(o));
1109        }
1110        socket.send(&text);
1111    }
1112
1113    // MARK: The run loop
1114
1115    fn start_run_loop(&self) {
1116        if self.socket_origin.is_none() {
1117            self.connection_manager
1118                .disconnected(ConnectionError::client(
1119                    ClientErrorKind::NoSocketOrigin,
1120                    "No server socket origin provided",
1121                ));
1122            return;
1123        }
1124        self.connection_manager.initialized();
1125        self.iterate();
1126    }
1127
1128    fn next_loop_token(&self) -> u64 {
1129        let t = self.loop_token.get() + 1;
1130        self.loop_token.set(t);
1131        t
1132    }
1133
1134    fn is_token(&self, token: u64) -> bool {
1135        self.loop_token.get() == token
1136    }
1137
1138    /// One turn of the JS run loop: acts on the connection status.
1139    fn iterate(&self) {
1140        if !self.connection_manager.should_continue_run_loop() {
1141            return;
1142        }
1143        let token = self.next_loop_token();
1144        match self.connection_manager.status() {
1145            ConnectionStatus::Connecting | ConnectionStatus::Disconnected => {
1146                if !self.visible.get() {
1147                    // Wait for the app to come back, or for a state change.
1148                    self.on_visibility(true, token, |e| e.begin_attempt());
1149                    self.on_state_change(token, |e, state| e.resume_after_state_change(state));
1150                    return;
1151                }
1152                // The app is visible, which wins the race with a state
1153                // change.
1154                self.micro(move |e| {
1155                    if e.is_token(token) {
1156                        e.begin_attempt();
1157                    }
1158                });
1159            }
1160            ConnectionStatus::Connected => self.wait_while_connected(token),
1161            ConnectionStatus::NeedsAuth | ConnectionStatus::Error => {
1162                let status = self.connection_manager.status();
1163                self.log.info(|| {
1164                    format!(
1165                        "Run loop paused in {} state. Call zero.connection.connect() to resume.",
1166                        status.as_str()
1167                    )
1168                });
1169                self.wait_for_connect_request(token);
1170            }
1171            ConnectionStatus::Closed => {}
1172            ConnectionStatus::Initializing => unreachable!("The run loop runs once initialized"),
1173        }
1174    }
1175
1176    /// Ends the current wait of the run loop with `error` (the `catch` of
1177    /// the JS run loop).
1178    fn loop_failed(&self, error: ConnectionError) {
1179        let (transition, reason) = ConnectionTransition::of(&error);
1180        if !self.status_is(ConnectionStatus::Connected)
1181            && error.client_kind() != Some(ClientErrorKind::ClientClosed)
1182        {
1183            // Only errors that stop the run loop are errors. Reconnects and
1184            // auth failures are routine.
1185            let mut level = match transition {
1186                ConnectionTransition::Error => LogLevel::Error,
1187                ConnectionTransition::Disconnected | ConnectionTransition::Closed => LogLevel::Info,
1188                ConnectionTransition::None | ConnectionTransition::NeedsAuth => LogLevel::Warn,
1189            };
1190            if error.server_kind() == Some(ErrorKind::ClientNotFound) {
1191                level = LogLevel::Warn;
1192            }
1193            self.log
1194                .log(level, || format!("Failed to connect: {}", error.message()));
1195        }
1196        let mut backoff_ms = RUN_LOOP_INTERVAL_MS;
1197        let mut sleep_ms = None;
1198        match transition {
1199            ConnectionTransition::None => {
1200                // A transport failure is retried soon, then less often.
1201                // Errors the server sent keep the full interval, adjusted by
1202                // any backoff they carry.
1203                if reason.is_transport_failure() {
1204                    let failures = self.transport_failures.get() + 1;
1205                    self.transport_failures.set(failures);
1206                    backoff_ms = reconnect_backoff_ms(failures, RUN_LOOP_INTERVAL_MS, random_f64());
1207                }
1208                if let Some(ErrorBody::Backoff {
1209                    min_backoff_ms,
1210                    max_backoff_ms,
1211                    reconnect_params,
1212                    ..
1213                }) = reason.backoff()
1214                {
1215                    if let Some(min) = min_backoff_ms {
1216                        backoff_ms = backoff_ms.max(*min);
1217                    }
1218                    if let Some(max) = max_backoff_ms {
1219                        backoff_ms = backoff_ms.min(*max);
1220                    }
1221                    *self.additional_connect_params.borrow_mut() =
1222                        reconnect_params.as_ref().map(|params| {
1223                            let mut p = params.clone();
1224                            p.sort_by(|a, b| a.0.cmp(&b.0));
1225                            p
1226                        });
1227                }
1228                sleep_ms = Some(backoff_ms);
1229            }
1230            ConnectionTransition::NeedsAuth => self.connection_manager.needs_auth(reason),
1231            ConnectionTransition::Error => self.connection_manager.error(reason),
1232            ConnectionTransition::Disconnected => self.connection_manager.disconnected(reason),
1233            ConnectionTransition::Closed => {}
1234        }
1235        let token = self.next_loop_token();
1236        match sleep_ms {
1237            Some(ms) if ms > 0.0 => {
1238                let timer = self.later(ms, move |e| {
1239                    if e.is_token(token) {
1240                        e.backoff.borrow_mut().take();
1241                        e.iterate();
1242                    }
1243                });
1244                *self.backoff.borrow_mut() = Some((token, timer));
1245            }
1246            _ => self.micro(move |e| {
1247                if e.is_token(token) {
1248                    e.iterate();
1249                }
1250            }),
1251        }
1252    }
1253
1254    /// `throwIfConnectionError`: whether a state the run loop wakes up to
1255    /// stops it. Dropped sockets and timeouts just reconnect.
1256    fn connection_error(state: &State) -> Option<ConnectionError> {
1257        match state {
1258            State::NeedsAuth(r) | State::Error(r) | State::Closed(r) => Some(r.clone()),
1259            State::Connecting {
1260                reason: Some(r), ..
1261            }
1262            | State::Disconnected(r) => match r.client_kind() {
1263                Some(
1264                    ClientErrorKind::ConnectTimeout
1265                    | ClientErrorKind::AbruptClose
1266                    | ClientErrorKind::CleanClose
1267                    | ClientErrorKind::Hidden,
1268                ) => None,
1269                _ => Some(r.clone()),
1270            },
1271            _ => None,
1272        }
1273    }
1274
1275    /// Wakes the loop on the next state change, unless the wait is over.
1276    fn on_state_change(&self, token: u64, body: impl FnOnce(&Engine, State) + 'static) {
1277        let (a, b) = (self.me.clone(), self.me.clone());
1278        self.connection_manager.on_next_state_change(
1279            move || a.upgrade().is_some_and(|e| e.is_token(token)),
1280            move |state| {
1281                if let Some(e) = b.upgrade()
1282                    && e.is_token(token)
1283                {
1284                    body(&e, state);
1285                }
1286            },
1287        );
1288    }
1289
1290    fn resume_after_state_change(&self, state: State) {
1291        match Self::connection_error(&state) {
1292            Some(error) => self.loop_failed(error),
1293            None => self.iterate(),
1294        }
1295    }
1296
1297    /// Connected: ping after `ping_timeout_ms` without poke traffic.
1298    fn wait_while_connected(&self, token: u64) {
1299        let ready = self.connection_ready.borrow().clone();
1300        let timer: Rc<RefCell<Option<Box<dyn Timer>>>> = Rc::default();
1301        let cancel = {
1302            let timer = timer.clone();
1303            move || {
1304                if let Some(t) = timer.borrow_mut().take() {
1305                    t.cancel();
1306                }
1307            }
1308        };
1309        let end: Rc<dyn Fn()> = {
1310            let w = self.me.clone();
1311            Rc::new(move || {
1312                let Some(e) = w.upgrade() else { return };
1313                // The `finally` of the connected case.
1314                if !e.status_is(ConnectionStatus::Connected)
1315                    && e.connection_ready.borrow().is(&ready)
1316                {
1317                    *e.connection_ready.borrow_mut() = ReadyPromise::default();
1318                }
1319            })
1320        };
1321        {
1322            let w = self.me.clone();
1323            let cancel = cancel.clone();
1324            let end = end.clone();
1325            *self.abort_ping_timeout.borrow_mut() = Rc::new(move || {
1326                cancel();
1327                let end = end.clone();
1328                if let Some(e) = w.upgrade() {
1329                    e.micro(move |e| {
1330                        if e.is_token(token) {
1331                            end();
1332                            e.iterate();
1333                        }
1334                    });
1335                }
1336            });
1337        }
1338        {
1339            let end = end.clone();
1340            let t = self.later(self.options.ping_timeout_ms, move |e| {
1341                if e.is_token(token) {
1342                    e.ping(end);
1343                }
1344            });
1345            *timer.borrow_mut() = Some(t);
1346        }
1347        {
1348            let w = self.me.clone();
1349            let cancel = cancel.clone();
1350            let end = end.clone();
1351            *self.ping_now.borrow_mut() = Rc::new(move || {
1352                if let Some(e) = w.upgrade()
1353                    && e.is_token(token)
1354                {
1355                    cancel();
1356                    e.ping(end.clone());
1357                }
1358            });
1359        }
1360        {
1361            let cancel = cancel.clone();
1362            let end = end.clone();
1363            self.on_visibility(false, token, move |e| {
1364                cancel();
1365                e.disconnect(
1366                    ConnectionError::client(
1367                        ClientErrorKind::Hidden,
1368                        "Connection closed because tab was hidden",
1369                    ),
1370                    None,
1371                );
1372                end();
1373                e.iterate();
1374            });
1375        }
1376        self.on_state_change(token, move |e, state| {
1377            end();
1378            e.resume_after_state_change(state);
1379        });
1380    }
1381
1382    fn ping(&self, end: Rc<dyn Fn()>) {
1383        let token = self.next_loop_token();
1384        let t0 = self.scheduler.monotonic_now();
1385        // Before the send, as JS sets `#onPong` first: a transport may
1386        // deliver the pong from inside `send`.
1387        {
1388            let w = self.me.clone();
1389            let end = end.clone();
1390            *self.on_pong.borrow_mut() = Rc::new(move || {
1391                let end = end.clone();
1392                if let Some(e) = w.upgrade() {
1393                    e.micro(move |e| {
1394                        if !e.is_token(token) {
1395                            return;
1396                        }
1397                        let ms = e.scheduler.monotonic_now() - t0;
1398                        e.log
1399                            .debug(|| format!("ping succeeded in {} ms", number::format(ms)));
1400                        end();
1401                        e.iterate();
1402                    });
1403                }
1404            });
1405        }
1406        self.send(Upstream::Ping);
1407        {
1408            let end = end.clone();
1409            // The timer ends with the token; it needn't be cancelled.
1410            let _ = self.later(self.options.ping_timeout_ms, move |e| {
1411                if !e.is_token(token) {
1412                    return;
1413                }
1414                let ms = e.scheduler.monotonic_now() - t0;
1415                e.log
1416                    .info(|| format!("ping failed in {} ms - disconnecting", number::format(ms)));
1417                let error = ConnectionError::client(
1418                    ClientErrorKind::PingTimeout,
1419                    "Server ping request failed",
1420                );
1421                e.disconnect(error.clone(), None);
1422                end();
1423                e.loop_failed(error);
1424            });
1425        }
1426        self.on_state_change(token, move |e, state| {
1427            end();
1428            e.resume_after_state_change(state);
1429        });
1430    }
1431
1432    /// needs-auth and error: wait for `connect()`.
1433    fn wait_for_connect_request(&self, token: u64) {
1434        let w = self.me.clone();
1435        self.connection_manager.on_connect_request(move || {
1436            if let Some(e) = w.upgrade()
1437                && e.is_token(token)
1438            {
1439                e.connection_manager.resume_from_connect_request();
1440                e.iterate();
1441            }
1442        });
1443        self.on_state_change(token, |e, _| e.iterate());
1444    }
1445
1446    // MARK: The app's lifecycle
1447
1448    /// The app was shown, or hidden. Hidden, the client disconnects and
1449    /// waits to be shown again.
1450    pub(crate) fn set_visible(&self, visible: bool) {
1451        if let Some(t) = self.hidden_timer.borrow_mut().take() {
1452            t.cancel();
1453        }
1454        if visible {
1455            self.set_visibility_state(true);
1456        } else {
1457            let t = self.later(self.options.hidden_disconnect_delay_ms, |e| {
1458                e.hidden_timer.borrow_mut().take();
1459                e.set_visibility_state(false);
1460            });
1461            *self.hidden_timer.borrow_mut() = Some(t);
1462        }
1463    }
1464
1465    fn set_visibility_state(&self, visible: bool) {
1466        if visible == self.visible.get() {
1467            return;
1468        }
1469        self.visible.set(visible);
1470        let ready: Vec<VisibilityWaiter> = {
1471            let mut waiters = self.visibility_waiters.borrow_mut();
1472            let (ready, rest) = std::mem::take(&mut *waiters)
1473                .into_iter()
1474                .partition(|(v, _, _)| *v == visible);
1475            *waiters = rest;
1476            ready
1477        };
1478        for (_, _, w) in ready {
1479            w();
1480        }
1481    }
1482
1483    /// Calls `body` in a microtask once the app's visibility is `visible`,
1484    /// unless the run loop's wait `token` is over by then.
1485    fn on_visibility(&self, visible: bool, token: u64, body: impl FnOnce(&Engine) + 'static) {
1486        let w = self.me.clone();
1487        let waiter: Box<dyn FnOnce()> = Box::new(move || {
1488            if let Some(e) = w.upgrade() {
1489                e.micro(move |e| {
1490                    if !e.is_token(token) {
1491                        return;
1492                    }
1493                    // The wait is over: a state change no longer ends it.
1494                    e.next_loop_token();
1495                    body(e);
1496                });
1497            }
1498        });
1499        if self.visible.get() == visible {
1500            waiter();
1501        } else {
1502            let current = self.loop_token.get();
1503            let mut waiters = self.visibility_waiters.borrow_mut();
1504            waiters.retain(|(_, t, _)| *t == current);
1505            waiters.push((visible, token, waiter));
1506        }
1507    }
1508
1509    /// The network may be back: retry now instead of after the backoff,
1510    /// and check that an open connection still works.
1511    pub(crate) fn reconnect_now(&self) {
1512        let backoff = {
1513            let mut b = self.backoff.borrow_mut();
1514            match &*b {
1515                Some((token, _)) if *token == self.loop_token.get() => b.take(),
1516                _ => None,
1517            }
1518        };
1519        if let Some((_, timer)) = backoff {
1520            timer.cancel();
1521            self.iterate();
1522        } else if self.status_is(ConnectionStatus::Connected) {
1523            let ping_now = self.ping_now.borrow().clone();
1524            ping_now();
1525        }
1526    }
1527
1528    fn pending_settled(&self) -> bool {
1529        self.store.borrow().pending.is_empty()
1530            || !self.status_is(ConnectionStatus::Connected)
1531            || self.client_group_disabled.get()
1532            || self.rep_closed.get()
1533    }
1534
1535    /// Calls `done` once the server has confirmed every pending mutation,
1536    /// or right away when it can't: the client isn't connected, or the
1537    /// group is disabled.
1538    pub(crate) fn when_pending_settled(&self, done: impl FnOnce() + 'static) {
1539        if self.pending_settled() {
1540            done();
1541        } else {
1542            self.pending_waiters.borrow_mut().push(Box::new(done));
1543        }
1544    }
1545
1546    fn settle_pending_waiters(&self) {
1547        if self.pending_waiters.borrow().is_empty() || !self.pending_settled() {
1548            return;
1549        }
1550        let waiters = std::mem::take(&mut *self.pending_waiters.borrow_mut());
1551        for w in waiters {
1552            w();
1553        }
1554    }
1555
1556    // MARK: Connecting
1557
1558    fn add_connect_event(&self, attempt: &ConnectAttempt, event: &str) {
1559        attempt
1560            .events
1561            .borrow_mut()
1562            .push((self.scheduler.now(), event.to_owned()));
1563    }
1564
1565    fn arm_connect_timeout(&self, attempt: &Rc<ConnectAttempt>, ack: bool) {
1566        attempt.cancel_timeout();
1567        if attempt.aborted.get() {
1568            return;
1569        }
1570        let weak = Rc::downgrade(attempt);
1571        let t = self.later(CONNECT_TIMEOUT_MS, move |e| {
1572            let Some(attempt) = weak.upgrade() else {
1573                return;
1574            };
1575            let events = Value::Array(
1576                attempt
1577                    .events
1578                    .borrow()
1579                    .iter()
1580                    .map(|(t, ev)| {
1581                        Value::Array(vec![
1582                            Value::String(iso_date_string(*t)),
1583                            Value::String(ev.clone()),
1584                        ])
1585                    })
1586                    .collect(),
1587            );
1588            e.log.debug(|| "Connection attempt timed out".into());
1589            let seconds = number::format(CONNECT_TIMEOUT_MS / 1000.0);
1590            let message = format!(
1591                "{}Connect events: {}",
1592                if ack {
1593                    format!("Server did not acknowledge the connection within {seconds} seconds. ")
1594                } else {
1595                    format!("Connection setup timed out after {seconds} seconds. ")
1596                },
1597                json::stringify(&events)
1598            );
1599            e.disconnect(
1600                ConnectionError::client(ClientErrorKind::ConnectTimeout, message),
1601                None,
1602            );
1603        });
1604        *attempt.timeout.borrow_mut() = Some(t);
1605    }
1606
1607    fn begin_attempt(&self) {
1608        assert!(
1609            self.current_attempt.borrow().is_none(),
1610            "Connection attempt already active"
1611        );
1612        let attempt = Rc::new(ConnectAttempt::default());
1613        *self.current_attempt.borrow_mut() = Some(attempt.clone());
1614        self.add_connect_event(&attempt, "starting the connection");
1615        self.arm_connect_timeout(&attempt, false);
1616        self.connect_attempt(&attempt);
1617    }
1618
1619    fn token(&self) -> Option<String> {
1620        let auth = self.auth.borrow();
1621        (!auth.is_empty()).then(|| auth.clone())
1622    }
1623
1624    /// `#connectAttempt`: opens a socket and waits for `connected`.
1625    fn connect_attempt(&self, attempt: &Rc<ConnectAttempt>) {
1626        if self.rep_closed.get() {
1627            self.finish_attempt(attempt, None);
1628            return;
1629        }
1630        let (Some(socket_origin), Some(socket_factory)) =
1631            (self.socket_origin.clone(), self.socket_factory.clone())
1632        else {
1633            self.finish_attempt(
1634                attempt,
1635                Some(ConnectionError::client(
1636                    ClientErrorKind::Internal,
1637                    "No server provided",
1638                )),
1639            );
1640            return;
1641        };
1642        assert!(
1643            self.status_is(ConnectionStatus::Disconnected)
1644                || self.status_is(ConnectionStatus::Connecting),
1645            "connectAttempt() called from invalid state"
1646        );
1647        let wsid = random_id();
1648        self.log.info(|| "Connecting...".into());
1649        self.connection_manager.connecting(None);
1650        attempt.start.set(self.scheduler.monotonic_now());
1651        self.add_connect_event(attempt, "reading the cookie");
1652        let connect_cookie = self.store.borrow().cookie().clone();
1653        self.add_connect_event(attempt, "reading the client group ID");
1654        self.add_connect_event(attempt, "initializing active clients");
1655        if attempt.aborted.get() {
1656            let reason = attempt.abort_reason.borrow().clone();
1657            self.finish_attempt(attempt, reason);
1658            return;
1659        }
1660
1661        // `createSocket`.
1662        self.add_connect_event(attempt, "reading the profile ID");
1663        let url = ConnectionUrl {
1664            socket_origin,
1665            client_id: self.client_id.clone(),
1666            client_group_id: self.client_group_id.clone(),
1667            user_id: self.user_id.clone(),
1668            base_cookie: match &connect_cookie {
1669                Value::String(s) => Some(s.clone()),
1670                Value::Number(n) => Some(number::format(*n)),
1671                _ => None,
1672            },
1673            timestamp: self.scheduler.monotonic_now(),
1674            last_mutation_id: self.last_mutation_id_received.get(),
1675            wsid,
1676            profile_id: self.profile_id.clone(),
1677            debug_perf: false,
1678            additional_params: self
1679                .additional_connect_params
1680                .borrow()
1681                .clone()
1682                .unwrap_or_default(),
1683        }
1684        .to_string();
1685        self.add_connect_event(attempt, "reading deleted clients");
1686        let mut deleted = self.deleted_clients_body();
1687        self.add_connect_event(attempt, "reading desired queries");
1688        let mut queries_patch = Some(
1689            self.query_manager
1690                .queries_patch(&self.store.borrow().main, None),
1691        );
1692        let mut active_clients = vec![self.client_id.clone()];
1693        active_clients.extend(
1694            self.database
1695                .active_clients(&self.client_group_id)
1696                .into_iter()
1697                .filter(|c| *c != self.client_id),
1698        );
1699        let init_connection = InitConnectionBody {
1700            desired_queries_patch: queries_patch
1701                .as_ref()
1702                .map(|p| p.values().cloned().collect())
1703                .unwrap_or_default(),
1704            client_schema: connect_cookie.is_null().then(|| self.client_schema.clone()),
1705            deleted: deleted.clone(),
1706            user_push_url: self.options.mutate_url.clone(),
1707            user_push_headers: self.options.mutate_headers.clone(),
1708            user_query_url: self.options.query_url.clone(),
1709            user_query_headers: self.options.query_headers.clone(),
1710            active_clients: Some(active_clients),
1711            traceparent: None,
1712        };
1713        let auth_token = self.token();
1714        let mut sec_protocol = sec_protocols::encode(Some(&init_connection), auth_token.as_deref());
1715        if sec_protocol.len() > self.options.max_header_length {
1716            sec_protocol = sec_protocols::encode(None, auth_token.as_deref());
1717            queries_patch = None;
1718            if sec_protocol.len() > self.options.max_header_length {
1719                let (len, max) = (sec_protocol.len(), self.options.max_header_length);
1720                self.log.warn(|| {
1721                    format!(
1722                        "Encoded auth token length ({len}) exceeds ZeroOptions.maxHeaderLength \
1723                         ({max}). This may cause connection failures."
1724                    )
1725                });
1726            }
1727        } else {
1728            deleted = None;
1729        }
1730        self.log.info(|| format!("Connecting to {url}"));
1731        self.add_connect_event(attempt, "creating the WebSocket");
1732        let handle = SocketHandle::new(self.me.clone());
1733        let connection = socket_factory.open(&url, Some(&sec_protocol), handle.clone());
1734        *handle.connection.borrow_mut() = Some(connection);
1735
1736        if attempt.aborted.get() {
1737            handle.close(None);
1738            let reason = attempt.abort_reason.borrow().clone();
1739            self.finish_attempt(attempt, reason);
1740            return;
1741        }
1742        if self.rep_closed.get() {
1743            handle.close(None);
1744            self.finish_attempt(attempt, None);
1745            return;
1746        }
1747        *self.connect_cookie.borrow_mut() = connect_cookie;
1748        *self.init_connection_queries.borrow_mut() = queries_patch;
1749        *self.deleted_clients.borrow_mut() = deleted;
1750        *self.socket.borrow_mut() = Some(handle);
1751        self.add_connect_event(attempt, "waiting for the server acknowledgement");
1752        self.log
1753            .debug(|| "Waiting for connection to be acknowledged".into());
1754        self.arm_connect_timeout(attempt, true);
1755        let (w, weak_attempt) = (self.me.clone(), Rc::downgrade(attempt));
1756        *attempt.on_connected.borrow_mut() = Some(Box::new(move || {
1757            let (Some(e), Some(attempt)) = (w.upgrade(), weak_attempt.upgrade()) else {
1758                return;
1759            };
1760            if attempt.aborted.get() {
1761                let reason = attempt.abort_reason.borrow().clone();
1762                e.finish_attempt(&attempt, reason);
1763                return;
1764            }
1765            e.mutation_tracker
1766                .on_connected(e.last_mutation_id_received.get());
1767            e.request_push(true);
1768            e.finish_attempt(&attempt, None);
1769        }));
1770    }
1771
1772    /// The run loop's handling of a finished attempt; `error` `None` for
1773    /// success.
1774    fn finish_attempt(&self, attempt: &Rc<ConnectAttempt>, error: Option<ConnectionError>) {
1775        if attempt.finished.replace(true) {
1776            return;
1777        }
1778        attempt.cancel_timeout();
1779        {
1780            let mut current = self.current_attempt.borrow_mut();
1781            if current.as_ref().is_some_and(|c| Rc::ptr_eq(c, attempt)) {
1782                *current = None;
1783            }
1784        }
1785        let ready = self.connection_ready.borrow().clone();
1786        if let Some(error) = error {
1787            if !attempt.aborted.get() {
1788                self.disconnect(error.clone(), None);
1789            }
1790            ready.settle(false, &self.event_loop);
1791            if self.connection_ready.borrow().is(&ready) {
1792                *self.connection_ready.borrow_mut() = ReadyPromise::default();
1793            }
1794            self.loop_failed(error);
1795            return;
1796        }
1797        ready.settle(true, &self.event_loop);
1798        *self.additional_connect_params.borrow_mut() = None;
1799        match Self::connection_error(&self.connection_manager.state()) {
1800            Some(error) => self.loop_failed(error),
1801            None => self.iterate(),
1802        }
1803    }
1804
1805    /// `#disconnect`: closes the socket and moves the connection status.
1806    pub(crate) fn disconnect(&self, reason: ConnectionError, close_code: Option<u16>) {
1807        let attempt = self.current_attempt.borrow().clone();
1808        if let Some(attempt) = attempt
1809            && !attempt.aborted.get()
1810        {
1811            *attempt.abort_reason.borrow_mut() = Some(reason.clone());
1812            attempt.aborted.set(true);
1813            attempt.cancel_timeout();
1814            let reason = reason.clone();
1815            self.micro(move |e| e.finish_attempt(&attempt, Some(reason)));
1816        }
1817        let status = self.connection_manager.status();
1818        self.log.info(|| {
1819            format!(
1820                "disconnecting: reason={} connectionState={}",
1821                reason.kind_name(),
1822                status.as_str()
1823            )
1824        });
1825        if status == ConnectionStatus::Closed {
1826            self.log.debug(|| "disconnect() called while closed".into());
1827            return;
1828        }
1829        if let Some(at) = self.connected_at.get()
1830            && self.scheduler.monotonic_now() - at >= STABLE_CONNECTION_MS
1831        {
1832            self.transport_failures.set(0);
1833        }
1834        self.connected_at.set(None);
1835        let socket = self.socket.borrow_mut().take();
1836        if let Some(s) = socket {
1837            s.close(close_code);
1838        }
1839        self.last_mutation_ids_sent.borrow_mut().clear();
1840        self.poke_handler.borrow_mut().clear();
1841        self.query_manager.clear_got_queries_authoritative();
1842        let (transition, r) = ConnectionTransition::of(&reason);
1843        match transition {
1844            ConnectionTransition::NeedsAuth => self.connection_manager.needs_auth(r),
1845            ConnectionTransition::Error => self.connection_manager.error(r),
1846            ConnectionTransition::Disconnected => self.connection_manager.disconnected(r),
1847            ConnectionTransition::Closed => self.connection_manager.closed(),
1848            ConnectionTransition::None => {
1849                if !self.connection_manager.is_in_terminal_state() {
1850                    self.connection_manager.connecting(Some(r));
1851                }
1852            }
1853        }
1854    }
1855
1856    // MARK: Messages
1857
1858    fn is_socket(&self, handle: &Rc<SocketHandle>) -> bool {
1859        self.socket
1860            .borrow()
1861            .as_ref()
1862            .is_some_and(|s| Rc::ptr_eq(s, handle))
1863    }
1864
1865    fn socket_did_receive(&self, handle: &Rc<SocketHandle>, message: SocketMessage) {
1866        if !self.is_socket(handle) {
1867            return;
1868        }
1869        let event_loop = self.event_loop.clone();
1870        event_loop.dispatch(|| self.on_message(message));
1871    }
1872
1873    fn socket_did_open(&self, handle: &Rc<SocketHandle>) {
1874        if !self.is_socket(handle) {
1875            return;
1876        }
1877        let Some(attempt) = self.current_attempt.borrow().clone() else {
1878            return;
1879        };
1880        let ms = self.scheduler.monotonic_now() - attempt.start.get();
1881        self.log
1882            .info(|| format!("Got socket open event: timeToOpenMs={}", number::format(ms)));
1883    }
1884
1885    fn socket_did_close(
1886        &self,
1887        handle: &Rc<SocketHandle>,
1888        code: u16,
1889        reason: &str,
1890        was_clean: bool,
1891    ) {
1892        if !self.is_socket(handle) {
1893            return;
1894        }
1895        let event_loop = self.event_loop.clone();
1896        event_loop.dispatch(|| {
1897            // 1005 and 1006 are how platforms report a close without a
1898            // frame.
1899            if code <= 1001 || code == 1005 || code == 1006 {
1900                self.log.info(|| {
1901                    format!("Got socket close event: code={code} reason={reason} wasClean={was_clean}")
1902                });
1903            } else {
1904                self.log.error(|| {
1905                    format!(
1906                        "Got unexpected socket close event: code={code} reason={reason} wasClean={was_clean}"
1907                    )
1908                });
1909            }
1910            if code == 1009 {
1911                let mut message = "A WebSocket message exceeded the server message size limit. \
1912                    This is usually caused by a mutation's args containing a large value such as \
1913                    a base64-encoded image. Consider uploading large files to object storage and \
1914                    storing only the URL in Zero."
1915                    .to_owned();
1916                let recent = self.recent_sent_messages.borrow();
1917                if !recent.is_empty() {
1918                    message.push_str("\nRecent sent messages:\n");
1919                    message.push_str(&pretty_recent_messages(&recent));
1920                }
1921                drop(recent);
1922                self.disconnect(
1923                    ConnectionError::client(ClientErrorKind::InvalidMessage, message),
1924                    None,
1925                );
1926                self.disable_client_group();
1927                self.micro(|e| e.client_state_not_found());
1928            } else if was_clean {
1929                self.disconnect(
1930                    ConnectionError::client(
1931                        ClientErrorKind::CleanClose,
1932                        "WebSocket connection closed cleanly",
1933                    ),
1934                    None,
1935                );
1936            } else {
1937                self.disconnect(
1938                    ConnectionError::client(
1939                        ClientErrorKind::AbruptClose,
1940                        "WebSocket connection closed abruptly",
1941                    ),
1942                    None,
1943                );
1944            }
1945        });
1946    }
1947
1948    fn abort_ping_timeout(&self) {
1949        let f = self.abort_ping_timeout.borrow().clone();
1950        f();
1951    }
1952
1953    /// Disconnects with `error` for a message that doesn't make sense.
1954    fn fail_on_message(&self, error: ConnectionError) {
1955        self.disconnect(error, None);
1956    }
1957
1958    fn on_message(&self, message: SocketMessage) {
1959        if self.rep_closed.get() {
1960            return;
1961        }
1962        let text = match message {
1963            SocketMessage::Binary(bytes) => {
1964                if bytes.first() != Some(&poke_chunk::MESSAGE_TYPE) {
1965                    let t = bytes
1966                        .first()
1967                        .map_or_else(|| "undefined".to_owned(), |b| b.to_string());
1968                    self.fail_on_message(ConnectionError::client(
1969                        ClientErrorKind::InvalidMessage,
1970                        format!("Unknown binary message type: {t}"),
1971                    ));
1972                    return;
1973                }
1974                self.abort_ping_timeout();
1975                let result = self
1976                    .poke_handler
1977                    .borrow_mut()
1978                    .handle_poke_chunk(&bytes[1..]);
1979                if let Err(m) = result {
1980                    self.poke_message_error(m);
1981                }
1982                return;
1983            }
1984            SocketMessage::Text(t) => t,
1985        };
1986        let downstream = match Downstream::from_text(&text) {
1987            Ok(d) => d,
1988            Err(e) => {
1989                self.fail_on_message(ConnectionError::client(
1990                    ClientErrorKind::InvalidMessage,
1991                    format!(
1992                        "Invalid message received from server: {}{text}",
1993                        valita_message(&e, &text)
1994                    ),
1995                ));
1996                return;
1997            }
1998        };
1999        match downstream {
2000            Downstream::Connected(body) => self.micro(move |e| e.handle_connected(body)),
2001            Downstream::Error(body) => self.handle_error_message(body),
2002            Downstream::Pong => {
2003                let f = self.on_pong.borrow().clone();
2004                f();
2005            }
2006            Downstream::PokeStart(body) => {
2007                self.abort_ping_timeout();
2008                let result = self.poke_handler.borrow_mut().handle_poke_start(body);
2009                if let Err(m) = result {
2010                    self.poke_message_error(m);
2011                }
2012            }
2013            Downstream::PokePart(body) => {
2014                self.abort_ping_timeout();
2015                let result = self.poke_handler.borrow_mut().handle_poke_part(body);
2016                match result {
2017                    Ok(Some(lmid)) => self.last_mutation_id_received.set(lmid),
2018                    Ok(None) => {}
2019                    Err(m) => self.poke_message_error(m),
2020                }
2021            }
2022            Downstream::PokeEnd(body) => {
2023                self.abort_ping_timeout();
2024                let result = self.poke_handler.borrow_mut().handle_poke_end(body);
2025                match result {
2026                    Ok(Some(r)) => {
2027                        if let Some(lmid) = r.last_mutation_id_change_for_self {
2028                            self.last_mutation_id_received.set(lmid);
2029                        }
2030                        if !self.poke_handler.borrow().playback_running {
2031                            self.start_playback();
2032                        }
2033                    }
2034                    Ok(None) => {}
2035                    Err(m) => self.poke_message_error(m),
2036                }
2037            }
2038            Downstream::Pull(_) => self.abort_ping_timeout(),
2039            Downstream::DeleteClients(body) => self.database.confirm_deleted_clients(
2040                body.client_ids.as_deref().unwrap_or_default(),
2041                body.client_group_ids.as_deref().unwrap_or_default(),
2042            ),
2043            Downstream::PushResponse(body) => {
2044                if matches!(body, PushResponseBody::Error(_)) {
2045                    self.log.error(|| {
2046                        format!(
2047                            "Received an error response when pushing mutations: {}",
2048                            json::stringify(&body.to_json())
2049                        )
2050                    });
2051                }
2052                if let Err(e) = self.mutation_tracker.process_push_response(&body) {
2053                    self.disconnect(
2054                        ConnectionError::client(ClientErrorKind::Internal, e.message()),
2055                        None,
2056                    );
2057                }
2058            }
2059            Downstream::TransformError(errors) => {
2060                self.query_manager.handle_transform_errors(&errors)
2061            }
2062            Downstream::Inspect(_) => {}
2063        }
2064    }
2065
2066    fn handle_connected(&self, _body: ConnectedBody) {
2067        let Some(attempt) = self.current_attempt.borrow().clone() else {
2068            return;
2069        };
2070        let ms = self.scheduler.monotonic_now() - attempt.start.get();
2071        self.log
2072            .info(|| format!("Connected: timeToConnectMs={}", number::format(ms)));
2073        self.add_connect_event(&attempt, "processing the server acknowledgement");
2074        self.connected_at.set(Some(self.scheduler.monotonic_now()));
2075        self.last_mutation_ids_sent.borrow_mut().clear();
2076        let init_queries = self.init_connection_queries.borrow_mut().take();
2077        let patch = self
2078            .query_manager
2079            .queries_patch(&self.store.borrow().main, init_queries.as_ref());
2080        let patch_ops: Vec<UpQueriesPatchOp> = patch.values().cloned().collect();
2081        match init_queries {
2082            Some(_) if !patch.is_empty() => {
2083                self.send(Upstream::ChangeDesiredQueries(ChangeDesiredQueriesBody {
2084                    desired_queries_patch: patch_ops,
2085                    traceparent: None,
2086                }));
2087            }
2088            Some(_) => {}
2089            None => {
2090                let deleted = self.deleted_clients.borrow_mut().take();
2091                let client_schema = self
2092                    .connect_cookie
2093                    .borrow()
2094                    .is_null()
2095                    .then(|| self.client_schema.clone());
2096                self.send(Upstream::InitConnection(InitConnectionBody {
2097                    desired_queries_patch: patch_ops,
2098                    client_schema,
2099                    deleted,
2100                    user_push_url: self.options.mutate_url.clone(),
2101                    user_push_headers: self.options.mutate_headers.clone(),
2102                    user_query_url: self.options.query_url.clone(),
2103                    user_query_headers: self.options.query_headers.clone(),
2104                    active_clients: None,
2105                    traceparent: None,
2106                }));
2107            }
2108        }
2109        let deleted = self.deleted_clients.borrow_mut().take();
2110        if let Some(deleted) = deleted {
2111            self.send(Upstream::DeleteClients(deleted));
2112        }
2113        self.connection_manager.connected();
2114        let on_connected = attempt.on_connected.borrow_mut().take();
2115        if let Some(f) = on_connected {
2116            self.event_loop.microtask(f);
2117        }
2118    }
2119
2120    fn handle_error_message(&self, body: ErrorBody) {
2121        let kind = body.kind();
2122        if kind == ErrorKind::MutationRateLimited {
2123            self.last_mutation_ids_sent.borrow_mut().clear();
2124            self.log.error(|| {
2125                format!(
2126                    "{} Mutation rate limited: {}",
2127                    kind.as_str(),
2128                    body.message()
2129                )
2130            });
2131            return;
2132        }
2133        // Being sent elsewhere or later, a lost client group (which
2134        // recovers), and auth failures are routine; every other kind ends
2135        // the connection.
2136        let message = body.message().to_owned();
2137        let error = ConnectionError::Server(Box::new(body));
2138        let routine =
2139            error.backoff().is_some() || kind == ErrorKind::ClientNotFound || error.is_auth_error();
2140        self.log.log(
2141            if routine {
2142                LogLevel::Warn
2143            } else {
2144                LogLevel::Error
2145            },
2146            || format!("{}:\n\n{message}", kind.as_str()),
2147        );
2148        self.disconnect(error, None);
2149        match kind {
2150            ErrorKind::VersionNotSupported => self.update_needed(UpdateNeededReason {
2151                type_: kind.as_str().to_owned(),
2152                message: Some(message),
2153            }),
2154            ErrorKind::SchemaVersionNotSupported => {
2155                self.disable_client_group();
2156                self.micro(move |e| {
2157                    e.update_needed(UpdateNeededReason {
2158                        type_: "SchemaVersionNotSupported".into(),
2159                        message: Some(message),
2160                    })
2161                });
2162            }
2163            ErrorKind::ClientNotFound => {
2164                self.disable_client_group();
2165                self.micro(|e| e.client_state_not_found());
2166            }
2167            ErrorKind::InvalidConnectionRequestLastMutationID
2168            | ErrorKind::InvalidConnectionRequestBaseCookie => {
2169                // The server is behind the replica: drop it.
2170                self.database.drop_database();
2171                self.micro(|e| e.client_state_not_found());
2172            }
2173            _ => {}
2174        }
2175    }
2176
2177    /// A poke message that doesn't fit the poke being received.
2178    fn poke_message_error(&self, message: String) {
2179        self.poke_handler.borrow_mut().clear();
2180        self.on_poke_error(&message);
2181    }
2182
2183    fn on_poke_error(&self, message: &str) {
2184        if self.status_is(ConnectionStatus::Disconnected) {
2185            return;
2186        }
2187        if message.contains("unexpected base cookie") {
2188            self.log.debug(|| format!("clearing due to {message}"));
2189            self.disconnect(
2190                ConnectionError::client(
2191                    ClientErrorKind::UnexpectedBaseCookie,
2192                    "Server returned unexpected base cookie during sync",
2193                ),
2194                None,
2195            );
2196        } else {
2197            self.log
2198                .error(|| format!("clearing due to unexpected poke error: {message}"));
2199            self.disconnect(
2200                ConnectionError::client(
2201                    ClientErrorKind::Internal,
2202                    format!("Poke processing error: {message}"),
2203                ),
2204                None,
2205            );
2206        }
2207    }
2208
2209    // MARK: Poke playback
2210
2211    fn start_playback(&self) {
2212        self.poke_handler.borrow_mut().playback_running = true;
2213        let _ = self.later(0.0, |e| e.playback_frame());
2214    }
2215
2216    /// Applies the pokes that arrived since the last frame, at most once
2217    /// per scheduler turn, merged (`#rafCallback`).
2218    fn playback_frame(&self) {
2219        if !self.poke_handler.borrow().has_buffered() {
2220            self.poke_handler.borrow_mut().playback_running = false;
2221            return;
2222        }
2223        let _ = self.later(0.0, |e| e.playback_frame());
2224        let merged = self.poke_handler.borrow_mut().take_merged();
2225        let merged = match merged {
2226            Ok(Some(m)) => m,
2227            Ok(None) => return,
2228            Err(e) => {
2229                self.poke_handler.borrow_mut().clear();
2230                self.on_poke_error(&e.0);
2231                return;
2232            }
2233        };
2234        let applied = self.with_write_lock(|| self.apply_poke(&merged));
2235        let result = applied.and_then(|()| {
2236            let lmid = merged
2237                .last_mutation_id_changes
2238                .get(&self.client_id)
2239                .copied();
2240            let advanced = match lmid {
2241                Some(lmid) => self
2242                    .mutation_tracker
2243                    .lmid_advanced(lmid)
2244                    .map_err(|m| format!("Error: {m}")),
2245                None => Ok(()),
2246            };
2247            self.settle_pending_waiters();
2248            advanced
2249        });
2250        if let Err(message) = result {
2251            self.poke_handler.borrow_mut().clear();
2252            self.on_poke_error(&message);
2253        }
2254    }
2255
2256    // MARK: Auth
2257
2258    pub(crate) fn set_auth(&self, auth: Option<String>) {
2259        // A token without a user was rejected at init.
2260        *self.auth.borrow_mut() = auth.clone().unwrap_or_default();
2261        if let Some(auth) = auth
2262            && !auth.is_empty()
2263        {
2264            self.send_if_connected(Upstream::UpdateAuth(UpdateAuthBody { auth }));
2265        }
2266    }
2267
2268    /// `connection.connect()`: resumes from needs-auth or error. `auth`,
2269    /// when set, replaces the token first (`Some(None)` clears it).
2270    pub(crate) fn connect(&self, auth: Option<Option<String>>) {
2271        if let Some(auth) = auth {
2272            // JS's `#setAuth` throws this, so `connect` rejects before it
2273            // does anything.
2274            if let Err(e) = Self::check_auth_valid(self.user_id.as_deref(), auth.as_deref()) {
2275                self.log.error(|| format!("connect(auth:) ignored: {e}"));
2276                return;
2277            }
2278            self.set_auth(auth);
2279        }
2280        if let State::Disconnected(r) = self.connection_manager.state()
2281            && r.client_kind() == Some(ClientErrorKind::NoSocketOrigin)
2282        {
2283            self.log.error(|| {
2284                "connect() called but the connection is disconnected due to a missing cacheURL. \
2285                 No reconnect will be attempted."
2286                    .into()
2287            });
2288            return;
2289        }
2290        if !self.connection_manager.is_in_terminal_state() {
2291            return;
2292        }
2293        self.connection_manager.request_connect();
2294    }
2295
2296    fn client_state_not_found(&self) {
2297        let f = self.on_client_state_not_found.borrow().clone();
2298        match f {
2299            Some(f) => f(),
2300            None => {
2301                if let Some(f) = &self.options.on_client_state_not_found {
2302                    f();
2303                }
2304            }
2305        }
2306    }
2307
2308    fn update_needed(&self, reason: UpdateNeededReason) {
2309        let f = self.on_update_needed.borrow().clone();
2310        if let Some(f) = f {
2311            f(&reason);
2312        }
2313        if let Some(f) = &self.options.on_update_needed {
2314            f(&reason);
2315        }
2316    }
2317
2318    // MARK: Close
2319
2320    pub(crate) fn closed(&self) -> bool {
2321        self.rep_closed.get()
2322    }
2323
2324    pub(crate) fn close(&self) {
2325        if self.rep_closed.get() {
2326            self.log
2327                .debug(|| "close() called on already closed instance".into());
2328            return;
2329        }
2330        self.log.debug(|| "Closing Zero instance".into());
2331        if !self.status_is(ConnectionStatus::Disconnected) {
2332            self.disconnect(
2333                ConnectionError::client(
2334                    ClientErrorKind::ClientClosed,
2335                    "Zero instance closed by user",
2336                ),
2337                Some(1000),
2338            );
2339        }
2340        self.rep_closed.set(true);
2341        if let Some(t) = self.push_timer.borrow_mut().take() {
2342            t.cancel();
2343        }
2344        // JS's close aborts a persist that hasn't run.
2345        if let Some(t) = self.persist_timer.borrow_mut().take() {
2346            t.cancel();
2347        }
2348        for t in std::mem::take(&mut *self.lifecycle_timers.borrow_mut()) {
2349            t.cancel();
2350        }
2351        self.database.client_stopped(&self.client_id);
2352        // Releasing writes what the store had queued; a failure to is
2353        // still this client's to report.
2354        self.database.release();
2355        self.database.remove_failure_listener(&self.client_id);
2356        self.connection_manager.closed();
2357    }
2358
2359    // MARK: Storage and the client lifecycle
2360
2361    /// Starts the heartbeat and the garbage collection of clients and
2362    /// client groups.
2363    fn start_lifecycle(&self) {
2364        let (h, c, g, d) = (
2365            self.me.clone(),
2366            self.me.clone(),
2367            self.me.clone(),
2368            self.me.clone(),
2369        );
2370        let timers = vec![
2371            self.scheduler.schedule_repeating(
2372                HEARTBEAT_INTERVAL_MS,
2373                Box::new(move || {
2374                    if let Some(e) = h.upgrade()
2375                        && !e.database.heartbeat(&e.client_id, e.scheduler.now())
2376                    {
2377                        e.client_state_not_found();
2378                    }
2379                }),
2380            ),
2381            self.scheduler.schedule_repeating(
2382                GC_INTERVAL_MS,
2383                Box::new(move || {
2384                    let Some(e) = c.upgrade() else { return };
2385                    if let Some(deleted) = e.database.collect_clients(
2386                        &e.client_id,
2387                        e.scheduler.now(),
2388                        CLIENT_MAX_INACTIVE_MS,
2389                    ) {
2390                        // `DeleteClientsManager.onClientsDeleted`.
2391                        let ids = deleted
2392                            .into_iter()
2393                            .filter(|p| p.client_group_id == e.client_group_id)
2394                            .map(|p| p.client_id)
2395                            .collect();
2396                        e.send_if_connected(Upstream::DeleteClients(DeleteClientsBody {
2397                            client_ids: Some(ids),
2398                            client_group_ids: None,
2399                        }));
2400                    }
2401                }),
2402            ),
2403            self.scheduler.schedule_repeating(
2404                GC_INTERVAL_MS,
2405                Box::new(move || {
2406                    if let Some(e) = g.upgrade() {
2407                        e.database.collect_client_groups();
2408                    }
2409                }),
2410            ),
2411            self.scheduler.schedule(
2412                INITIAL_COLLECT_DATABASES_DELAY_MS,
2413                Box::new(move || {
2414                    let Some(e) = d.upgrade() else { return };
2415                    e.collect_databases();
2416                    let w = e.me.clone();
2417                    let t = e.scheduler.schedule_repeating(
2418                        COLLECT_DATABASES_INTERVAL_MS,
2419                        Box::new(move || {
2420                            if let Some(e) = w.upgrade() {
2421                                e.collect_databases();
2422                            }
2423                        }),
2424                    );
2425                    e.lifecycle_timers.borrow_mut().push(t);
2426                }),
2427            ),
2428        ];
2429        *self.lifecycle_timers.borrow_mut() = timers;
2430    }
2431
2432    /// Deletes the other databases of the store that no client has used
2433    /// for twice the time a client may be inactive (`collectIDBDatabases`),
2434    /// such as those of an older schema.
2435    fn collect_databases(&self) {
2436        self.database
2437            .collect_databases(self.scheduler.now(), 2.0 * CLIENT_MAX_INACTIVE_MS);
2438    }
2439
2440    /// The store failed: the client keeps running on what it has in
2441    /// memory, and tells the app once.
2442    fn storage_failed(&self, f: &StorageFailure, opening: bool) {
2443        if self.storage_failure_reported.replace(true) {
2444            return;
2445        }
2446        if !opening {
2447            // A full or failing disk is the device's condition, not a
2448            // defect.
2449            self.log.warn(|| {
2450                format!(
2451                    "Local store storage failed ({}); persistence is stopped for this instance: {}",
2452                    f.kind.as_str(),
2453                    f.message
2454                )
2455            });
2456        }
2457        if let Some(cb) = &self.options.on_storage_failure {
2458            cb(f);
2459        }
2460    }
2461
2462    /// The deleted clients of this client group, for `initConnection`
2463    /// (`convertDeletedClientsToBody`).
2464    fn deleted_clients_body(&self) -> Option<DeleteClientsBody> {
2465        let ids: Vec<String> = self
2466            .database
2467            .deleted_clients()
2468            .into_iter()
2469            .filter(|p| p.client_group_id == self.client_group_id)
2470            .map(|p| p.client_id)
2471            .collect();
2472        (!ids.is_empty()).then_some(DeleteClientsBody {
2473            client_ids: Some(ids),
2474            client_group_ids: None,
2475        })
2476    }
2477
2478    /// The server lost the client group: stop pushing, and keep later
2479    /// clients out of it.
2480    fn disable_client_group(&self) {
2481        self.client_group_disabled.set(true);
2482        self.database.disable_client_group(&self.client_group_id);
2483    }
2484
2485    // MARK: Mutations
2486
2487    /// `MutatorProxy#onConnectionStateChange`.
2488    fn on_connection_state_change(&self, state: &State) {
2489        match state {
2490            State::Disconnected(r) if r.client_kind() == Some(ClientErrorKind::NoSocketOrigin) => {
2491                *self.mutation_rejection.borrow_mut() = None;
2492            }
2493            State::Disconnected(r) | State::Error(r) | State::Closed(r) => {
2494                *self.mutation_rejection.borrow_mut() = Some(r.clone());
2495                self.mutation_tracker.reject_all_outstanding_mutations(r);
2496            }
2497            _ => *self.mutation_rejection.borrow_mut() = None,
2498        }
2499        self.settle_pending_waiters();
2500    }
2501
2502    /// What a mutation's caller hears for an error (`wrapErrorFor`).
2503    fn outcome(error: &Error) -> MutationOutcome {
2504        if let Error::Connection(e) = error
2505            && e.is_zero_error()
2506        {
2507            return MutationOutcome::ZeroError {
2508                message: e.message().to_owned(),
2509            };
2510        }
2511        let app = ApplicationError::wrapping(error);
2512        MutationOutcome::AppError {
2513            message: app.message,
2514            details: app.details,
2515        }
2516    }
2517
2518    fn log_failure(&self, name: &str, outcome: &MutationOutcome, origin: &str) {
2519        match outcome {
2520            MutationOutcome::ZeroError { message } => self
2521                .log
2522                .error(|| format!("Mutator \"{name}\" error on {origin}: {message}")),
2523            MutationOutcome::AppError { message, .. } => self
2524                .log
2525                .error(|| format!("Mutator \"{name}\" app error on {origin}: {message}")),
2526            MutationOutcome::Success { .. } => {}
2527        }
2528    }
2529
2530    /// `zero.mutate`, through the `MutatorProxy` and Replicache's `mutate`.
2531    ///
2532    /// A mutation made while listeners are notified, from a listener, runs
2533    /// once the current task is done, as an async JS mutator commits after
2534    /// the notification: every listener sees the change that notified it
2535    /// before the mutation's, and every view sees both.
2536    pub(crate) fn mutate(&self, name: &str, args: Option<Value>) -> MutatorResult {
2537        let client = Deferred::new();
2538        let server = Deferred::new();
2539        let result = MutatorResult {
2540            client: client.clone(),
2541            server: server.clone(),
2542        };
2543        if self.context.is_notifying() {
2544            let w = self.me.clone();
2545            let name = name.to_owned();
2546            self.event_loop.after_current_task(move || {
2547                let Some(e) = w.upgrade() else { return };
2548                let r = e.mutate(&name, args);
2549                r.client.on_settle(move |o| client.settle(o.clone()));
2550                r.server.on_settle(move |o| server.settle(o.clone()));
2551            });
2552            return result;
2553        }
2554        // JS throws this before it looks at the connection; here both
2555        // outcomes carry it, and nothing is written or pushed.
2556        let Some(mutator) = self.mutators.get(name).cloned() else {
2557            let message = format!(
2558                "Mutator \"{name}\" is not registered. Mutators must be registered with the Zero \
2559                 constructor before use."
2560            );
2561            self.log.error(|| message.clone());
2562            client.resolve(MutationOutcome::ZeroError {
2563                message: message.clone(),
2564            });
2565            server.resolve(MutationOutcome::ZeroError { message });
2566            return result;
2567        };
2568        let rejection = self.mutation_rejection.borrow().clone();
2569        if let Some(rejection) = rejection {
2570            self.log.warn(|| {
2571                format!(
2572                    "Mutation \"{name}\" rejected because Zero is {}. Details: {}.",
2573                    rejection.state_description(),
2574                    rejection.message()
2575                )
2576            });
2577            let message = rejection.message().to_owned();
2578            client.resolve(MutationOutcome::ZeroError {
2579                message: message.clone(),
2580            });
2581            server.resolve(MutationOutcome::ZeroError { message });
2582            return result;
2583        }
2584        let (ephemeral_id, tracked) = self.mutation_tracker.track_mutation();
2585        {
2586            let w = self.me.clone();
2587            let name = name.to_owned();
2588            let server = server.clone();
2589            tracked.on_settle(move |r| match r {
2590                Ok(data) => server.resolve(MutationOutcome::Success { data: data.clone() }),
2591                Err(error) => {
2592                    let outcome = Self::outcome(error);
2593                    if let Some(e) = w.upgrade() {
2594                        e.log_failure(&name, &outcome, "server");
2595                    }
2596                    server.resolve(outcome);
2597                }
2598            });
2599        }
2600        // A mutate from inside a mutator, or while a poke rebases, waits for
2601        // it, as JS's waits for Replicache's write lock.
2602        if self.write_locked.get() || !self.waiting_for_write_lock.borrow().is_empty() {
2603            let w = self.me.clone();
2604            let name = name.to_owned();
2605            self.waiting_for_write_lock
2606                .borrow_mut()
2607                .push_back(Box::new(move || {
2608                    if let Some(e) = w.upgrade() {
2609                        e.with_write_lock(|| {
2610                            e.run_mutation(&mutator, &name, args, ephemeral_id, &client)
2611                        });
2612                    }
2613                }));
2614        } else {
2615            self.with_write_lock(|| self.run_mutation(&mutator, name, args, ephemeral_id, &client));
2616        }
2617        result
2618    }
2619
2620    fn with_write_lock<T>(&self, body: impl FnOnce() -> T) -> T {
2621        self.write_locked.set(true);
2622        let result = body();
2623        self.write_locked.set(false);
2624        if !self.waiting_for_write_lock.borrow().is_empty() {
2625            self.micro(|e| {
2626                if e.write_locked.get() {
2627                    return;
2628                }
2629                let next = e.waiting_for_write_lock.borrow_mut().pop_front();
2630                if let Some(next) = next {
2631                    next();
2632                }
2633            });
2634        }
2635        result
2636    }
2637
2638    fn run_mutation(
2639        &self,
2640        mutator: &MutatorFn,
2641        name: &str,
2642        args: Option<Value>,
2643        ephemeral_id: u64,
2644        client: &Deferred<MutationOutcome>,
2645    ) {
2646        let mutation_id = self.store.borrow().next_mutation_id(&self.client_id);
2647        // Last mutation IDs are checked when they arrive, so this holds.
2648        if !number::is_safe_integer(mutation_id) {
2649            let error: Error = ConnectionError::client(
2650                ClientErrorKind::Internal,
2651                format!("Invalid mutation ID {}", number::format(mutation_id)),
2652            )
2653            .into();
2654            self.mutation_tracker.reject_mutation(ephemeral_id, &error);
2655            client.resolve(Self::outcome(&error));
2656            return;
2657        }
2658        let write = KvWrite::new(self.store.borrow().main.clone());
2659        let branch = self.ivm_main.fork();
2660        self.mutation_tracker
2661            .mutation_id_assigned(ephemeral_id, mutation_id);
2662        let tx = Transaction::new(
2663            self.client_id.clone(),
2664            mutation_id,
2665            Reason::Optimistic,
2666            self.token(),
2667            write,
2668            branch,
2669            self.schema.clone(),
2670        );
2671        // JS forks the views' branch for the transaction, which throws once
2672        // that branch has failed.
2673        let ran = match self.context.branch_failure() {
2674            Some(f) => Err(f.into()),
2675            None => mutator(&tx, args.as_ref()),
2676        };
2677        let data = match ran {
2678            Ok(data) => data,
2679            Err(error) => {
2680                self.mutation_tracker.reject_mutation(ephemeral_id, &error);
2681                let outcome = Self::outcome(&error);
2682                self.log_failure(name, &outcome, "client");
2683                client.resolve(outcome);
2684                return;
2685            }
2686        };
2687        let (write, rows) = tx.finish();
2688        let pending = PendingMutation {
2689            id: mutation_id,
2690            client_id: self.client_id.clone(),
2691            name: name.to_owned(),
2692            args: args.clone().unwrap_or(Value::Null),
2693            timestamp: self.scheduler.monotonic_now(),
2694            writes: write.writes.clone(),
2695        };
2696        let stored = StoredMutation {
2697            client_id: self.client_id.clone(),
2698            id: mutation_id,
2699            name: name.to_owned(),
2700            args: pending.args.clone(),
2701            timestamp: pending.timestamp,
2702            aggregate_writes: aggregate_writes(&write),
2703        };
2704        let diff = self.store.borrow_mut().commit_mutation(pending, &write);
2705        self.database.apply(vec![StorageOp::AddMutation {
2706            group: self.client_group_id.clone(),
2707            mutation: stored,
2708        }]);
2709        if let Err(error) = self.context.process_changes(&diff, &rows) {
2710            // As in JS, the mutation stays committed and pending, but isn't
2711            // pushed now, and its client outcome is the error.
2712            self.fire_watchers(&diff, false);
2713            let outcome = Self::outcome(&error.into());
2714            self.log_failure(name, &outcome, "client");
2715            client.resolve(outcome);
2716            return;
2717        }
2718        self.request_push(false);
2719        self.fire_watchers(&diff, false);
2720        self.schedule_persist();
2721        client.resolve(MutationOutcome::Success { data });
2722    }
2723
2724    /// The store watches: the mutation tracker on `m/<clientID>/`, then the
2725    /// query manager on `g/`. A watch hears only about diffs that touch its
2726    /// prefix, except for its first run.
2727    fn fire_watchers(&self, diff: &KvDiff, initial: bool) {
2728        let mutations = with_prefix(diff, &format!("{MUTATIONS_PREFIX}{}/", self.client_id));
2729        if initial || !mutations.is_empty() {
2730            // A fatal error disconnects from inside the tracker.
2731            let _ = self.mutation_tracker.process_mutation_responses(&mutations);
2732        }
2733        let got = with_prefix(diff, GOT_QUERIES_PREFIX);
2734        if initial || !got.is_empty() {
2735            self.query_manager.process_got_queries(&got);
2736        }
2737    }
2738
2739    // MARK: Pushing
2740
2741    /// `pushConnectionLoop.send(now)`.
2742    fn request_push(&self, now: bool) {
2743        if self.rep_closed.get() {
2744            return;
2745        }
2746        self.push_requested.set(true);
2747        if self.push_in_flight.get() {
2748            return;
2749        }
2750        if now {
2751            if let Some(t) = self.push_timer.borrow_mut().take() {
2752                t.cancel();
2753            }
2754            self.micro(|e| e.invoke_push());
2755        } else if self.push_timer.borrow().is_none() {
2756            let t = self.later(0.0, |e| {
2757                e.push_timer.borrow_mut().take();
2758                e.invoke_push();
2759            });
2760            *self.push_timer.borrow_mut() = Some(t);
2761        }
2762    }
2763
2764    /// `#invokePush`: pushes the pending mutations once connected.
2765    fn invoke_push(&self) {
2766        if self.push_in_flight.get()
2767            || !self.push_requested.get()
2768            || self.rep_closed.get()
2769            || self.client_group_disabled.get()
2770        {
2771            return;
2772        }
2773        self.push_requested.set(false);
2774        let pending = self.store.borrow().pending.clone();
2775        if pending.is_empty() {
2776            return;
2777        }
2778        self.push_in_flight.set(true);
2779        let w = self.me.clone();
2780        let ready = self.connection_ready.borrow().clone();
2781        ready.on_settle(&self.event_loop, move |connected| {
2782            let Some(e) = w.upgrade() else { return };
2783            if connected {
2784                e.pusher(&pending);
2785            }
2786            e.push_in_flight.set(false);
2787            if !connected {
2788                e.push_requested.set(true);
2789            }
2790            if e.push_requested.get() {
2791                e.request_push(false);
2792            }
2793        });
2794    }
2795
2796    /// `#pusher`: one `push` message per mutation not yet sent on this
2797    /// connection.
2798    fn pusher(&self, mutations: &[PendingMutation]) {
2799        let now = self.scheduler.now();
2800        let counter = self.push_request_counter.get() + 1;
2801        self.push_request_counter.set(counter);
2802        let request_id = format!("{}-{counter}", self.client_id);
2803        for m in mutations {
2804            let sent = self
2805                .last_mutation_ids_sent
2806                .borrow()
2807                .get(&m.client_id)
2808                .copied()
2809                .unwrap_or(0.0);
2810            if m.id <= sent {
2811                continue;
2812            }
2813            let timestamp = now - (self.scheduler.monotonic_now() - m.timestamp).round();
2814            self.send(Upstream::Push(PushBody {
2815                client_group_id: self.client_group_id.clone(),
2816                mutations: vec![Mutation::Custom {
2817                    id: m.id,
2818                    client_id: m.client_id.clone(),
2819                    name: m.name.clone(),
2820                    args: vec![m.args.clone()],
2821                    timestamp,
2822                }],
2823                push_version: 1.0,
2824                schema_version: None,
2825                timestamp: now,
2826                request_id: request_id.clone(),
2827                auth: None,
2828                traceparent: None,
2829            }));
2830            self.last_mutation_ids_sent
2831                .borrow_mut()
2832                .insert(m.client_id.clone(), m.id);
2833        }
2834    }
2835
2836    // MARK: Pokes
2837
2838    /// `rep.poke`: applies a merged poke to the snapshot and rebases the
2839    /// pending mutations on it (`handlePullResponseV1` and
2840    /// `maybeEndPull`). Errors are JS's `String(error)`.
2841    fn apply_poke(&self, poke: &MergedPoke) -> Result<(), String> {
2842        let socket_at_start = self.socket.borrow().clone();
2843        let base = self.store.borrow().snapshot.clone();
2844        if poke.base_cookie != base.cookie {
2845            return Err(format!(
2846                "Error: unexpected base cookie for poke: {}",
2847                json::stringify(&poke.to_json())
2848            ));
2849        }
2850        let mut changes: Vec<(&String, &f64)> = poke.last_mutation_id_changes.iter().collect();
2851        changes.sort_by(|a, b| strings::utf16(a.0, b.0));
2852        for (cid, lmid) in changes {
2853            if let Some(last) = base.last_mutation_ids.get(cid)
2854                && lmid < last
2855            {
2856                return Err(format!(
2857                    "Error: {}",
2858                    bad_order_message(
2859                        &format!("{cid} lastMutationID"),
2860                        &number::format(*lmid),
2861                        &number::format(*last)
2862                    )
2863                ));
2864            }
2865        }
2866        if compare_cookies(&poke.cookie, &base.cookie) < 0.0 {
2867            return Err(format!(
2868                "Error: {}",
2869                bad_order_message(
2870                    "cookie",
2871                    &json::stringify(&poke.cookie),
2872                    &json::stringify(&base.cookie)
2873                )
2874            ));
2875        }
2876        if poke.cookie == base.cookie {
2877            // `zero.ts` trusts the got queries once `rep.poke` returns, even
2878            // for a poke that changes nothing.
2879            self.mark_got_queries_authoritative(socket_at_start.as_ref());
2880            return Ok(());
2881        }
2882
2883        let mut snapshot = base;
2884        for (cid, lmid) in &poke.last_mutation_id_changes {
2885            snapshot.last_mutation_ids.insert(cid.clone(), *lmid);
2886        }
2887        snapshot.cookie = poke.cookie.clone();
2888        let mut patch_write = KvWrite::new(snapshot.map.clone());
2889        apply_patch(&poke.patch, &mut patch_write).map_err(|e| format!("Error: {}", e.0))?;
2890        self.check_updated_rows(&poke.patch, &patch_write)?;
2891        snapshot.map = patch_write.map.clone();
2892
2893        let (store_main, pending_writes, pending) = {
2894            let store = self.store.borrow();
2895            (
2896                store.main.clone(),
2897                store.pending_writes(),
2898                store.pending.clone(),
2899            )
2900        };
2901        let mut touched: HashSet<String> =
2902            patch_write.writes.union(&pending_writes).cloned().collect();
2903        let mut main = snapshot.map.clone();
2904        let mut replayed = Vec::new();
2905        let pending: Vec<PendingMutation> = pending
2906            .into_iter()
2907            .filter(|m| {
2908                m.id > snapshot
2909                    .last_mutation_ids
2910                    .get(&m.client_id)
2911                    .copied()
2912                    .unwrap_or(0.0)
2913            })
2914            .collect();
2915        if !pending.is_empty() {
2916            // The rebase runs over a fork of the views' branch moved to the
2917            // new snapshot.
2918            let branch = self.ivm_main.fork();
2919            let moved = diff_keys(&store_main, &snapshot.map, &touched);
2920            capture(|| branch.apply(&moved, &HashMap::new())).map_err(|e| format!("Error: {e}"))?;
2921            let (m, r, t, _) =
2922                self.replay(&pending, &main, branch, &HashMap::new(), &mut |_, _| {});
2923            main = m;
2924            replayed = r;
2925            touched.extend(t);
2926        }
2927        let cookie = snapshot.cookie.clone();
2928        let last_mutation_ids = snapshot.last_mutation_ids.clone();
2929        let changes: Vec<(String, Option<Value>)> = {
2930            let mut keys: Vec<&String> = patch_write.writes.iter().collect();
2931            keys.sort();
2932            keys.into_iter()
2933                .filter_map(|k| {
2934                    let v = snapshot.map.get(k).cloned();
2935                    // After a clear, only what the poke put back.
2936                    (!(patch_write.cleared && v.is_none())).then(|| (k.clone(), v))
2937                })
2938                .collect()
2939        };
2940        let diff = self
2941            .store
2942            .borrow_mut()
2943            .advance(snapshot, main, replayed, Some(&touched));
2944        self.database.apply(vec![StorageOp::UpdateSnapshot {
2945            group: self.client_group_id.clone(),
2946            changes,
2947            cleared: patch_write.cleared,
2948            cookie,
2949            last_mutation_ids,
2950        }]);
2951        // A push that fails is a poke error: the client disconnects.
2952        self.context
2953            .process_changes(&diff, &HashMap::new())
2954            .map_err(|e| format!("Error: {e}"))?;
2955        self.fire_watchers(&diff, false);
2956        self.schedule_persist();
2957        self.mark_got_queries_authoritative(socket_at_start.as_ref());
2958        Ok(())
2959    }
2960
2961    /// The server has caught the client up, unless the connection the poke
2962    /// came on went away while it was applied: the disconnect already
2963    /// re-gated trust, and the next connection's first poke restores it.
2964    fn mark_got_queries_authoritative(&self, socket_at_start: Option<&Rc<SocketHandle>>) {
2965        if let Some(s) = socket_at_start
2966            && self.is_socket(s)
2967        {
2968            self.query_manager.mark_got_queries_authoritative();
2969        }
2970    }
2971
2972    /// Runs `pending` again, in order, over `map` and on forks of `branch`
2973    /// (the rebase of `maybeEndPull`). A mutation that fails stays pending
2974    /// without writes. Returns the new main map, the mutations with what
2975    /// they wrote, every key they wrote, and the branch they ran on.
2976    ///
2977    /// At launch, no query has registered the synced aggregates a mutation
2978    /// moves yet, so each run puts back the aggregate rows `restoring` has
2979    /// for the mutation, as JS's stored commit holds them. `ran` hears each
2980    /// mutation with its write, `None` when it failed.
2981    #[allow(clippy::type_complexity)]
2982    fn replay(
2983        &self,
2984        pending: &[PendingMutation],
2985        map: &KvMap,
2986        branch: Rc<IvmBranch>,
2987        restoring: &HashMap<MutationKey, AggregateWrites>,
2988        ran: &mut dyn FnMut(&PendingMutation, Option<&KvWrite>),
2989    ) -> (KvMap, Vec<PendingMutation>, HashSet<String>, Rc<IvmBranch>) {
2990        let mut main = map.clone();
2991        let mut branch = branch;
2992        let mut touched = HashSet::new();
2993        let mut replayed = Vec::new();
2994        for m in pending {
2995            let mut m = m.clone();
2996            let fork = branch.fork();
2997            let tx = Transaction::new(
2998                m.client_id.clone(),
2999                m.id,
3000                Reason::Rebase,
3001                self.token(),
3002                KvWrite::new(main.clone()),
3003                fork.clone(),
3004                self.schema.clone(),
3005            );
3006            let result = match self.mutators.get(&m.name) {
3007                Some(mutator) => mutator(&tx, Some(&m.args).filter(|a| !a.is_null())),
3008                None => Err(ApplicationError::new(format!("Unknown mutator: {}", m.name)).into()),
3009            };
3010            match result {
3011                Ok(_) => {
3012                    let (mut write, _) = tx.finish();
3013                    if let Some(writes) = restoring.get(&mutation_key(&m.client_id, m.id)) {
3014                        for (key, value) in writes {
3015                            match value {
3016                                Some(v) => write.set(key, v.clone()),
3017                                None => {
3018                                    write.delete(key);
3019                                }
3020                            }
3021                        }
3022                    }
3023                    main = write.map.clone();
3024                    branch = fork;
3025                    m.writes = write.writes.clone();
3026                    touched.extend(write.writes.iter().cloned());
3027                    ran(&m, Some(&write));
3028                }
3029                Err(_) => {
3030                    // The mutation stays pending; its writes are dropped.
3031                    m.writes = HashSet::new();
3032                    ran(&m, None);
3033                }
3034            }
3035            replayed.push(m);
3036        }
3037        (main, replayed, touched, branch)
3038    }
3039
3040    /// Replicache's persist scheduler (`ProcessScheduler`): after a
3041    /// mutation or a poke, a persist runs once the last one is
3042    /// `PERSIST_THROTTLE_MS` old, then after `PERSIST_IDLE_TIMEOUT_MS`;
3043    /// requests while one waits join it.
3044    fn schedule_persist(&self) {
3045        if self.persist_timer.borrow().is_some() || self.rep_closed.get() {
3046            return;
3047        }
3048        let now = self.scheduler.monotonic_now();
3049        let throttled = self
3050            .last_persist_at
3051            .get()
3052            .map_or(0.0, |at| (at + PERSIST_THROTTLE_MS - now).max(0.0));
3053        let t = self.later(throttled + PERSIST_IDLE_TIMEOUT_MS, |e| e.persist());
3054        *self.persist_timer.borrow_mut() = Some(t);
3055    }
3056
3057    /// `persistDD31`, for the synced aggregate rows the store keeps of each
3058    /// pending mutation. The snapshot and the mutations are written through
3059    /// as they come, and the mutations replayed at launch; JS persists its
3060    /// commits now and then, running the mutations again for it, so the
3061    /// aggregates they move are the ones registered at that persist, not
3062    /// when they ran. A persist runs the mutations it hasn't persisted over
3063    /// what it persisted before, or, when the snapshot is newer, all of
3064    /// them over it, and stores the aggregate rows each wrote.
3065    fn persist(&self) {
3066        self.persist_timer.borrow_mut().take();
3067        self.last_persist_at
3068            .set(Some(self.scheduler.monotonic_now()));
3069        let (pending, snapshot_cookie, snapshot_map, store_main, pending_writes) = {
3070            let store = self.store.borrow();
3071            (
3072                store.pending.clone(),
3073                store.snapshot.cookie.clone(),
3074                store.snapshot.map.clone(),
3075                store.main.clone(),
3076                store.pending_writes(),
3077            )
3078        };
3079        let to_run: Vec<PendingMutation> = {
3080            let mut persisted = self.persisted.borrow_mut();
3081            let to_run = if persisted.cookie != snapshot_cookie {
3082                persisted.cookie = snapshot_cookie;
3083                persisted.main = snapshot_map;
3084                persisted.touched.clear();
3085                pending.clone()
3086            } else {
3087                pending
3088                    .iter()
3089                    .filter(|m| {
3090                        !persisted
3091                            .mutations
3092                            .contains(&mutation_key(&m.client_id, m.id))
3093                    })
3094                    .cloned()
3095                    .collect()
3096            };
3097            persisted.mutations = pending
3098                .iter()
3099                .map(|m| mutation_key(&m.client_id, m.id))
3100                .collect();
3101            to_run
3102        };
3103        if to_run.is_empty() {
3104            return;
3105        }
3106        // The persist runs over a fork of the views' branch moved to what
3107        // it persisted.
3108        let (persisted_main, persisted_touched) = {
3109            let p = self.persisted.borrow();
3110            (p.main.clone(), p.touched.clone())
3111        };
3112        let branch = self.ivm_main.fork();
3113        let keys: HashSet<String> = pending_writes.union(&persisted_touched).cloned().collect();
3114        let moved = diff_keys(&store_main, &persisted_main, &keys);
3115        if capture(|| branch.apply(&moved, &HashMap::new())).is_err() {
3116            return;
3117        }
3118        let mut ops = Vec::new();
3119        let group = self.client_group_id.clone();
3120        let (main, _, touched, _) = self.replay(
3121            &to_run,
3122            &persisted_main,
3123            branch,
3124            &HashMap::new(),
3125            &mut |m, write| {
3126                ops.push(StorageOp::SetAggregateWrites {
3127                    group: group.clone(),
3128                    client_id: m.client_id.clone(),
3129                    id: m.id,
3130                    writes: write.map(aggregate_writes).unwrap_or_default(),
3131                });
3132            },
3133        );
3134        {
3135            let mut p = self.persisted.borrow_mut();
3136            p.main = main;
3137            p.touched.extend(touched);
3138        }
3139        self.database.apply(ops);
3140    }
3141
3142    /// Rejects a poke whose `update` ops leave a row under a key that isn't
3143    /// its primary key: one whose merge drops or changes the key's columns.
3144    /// The table's source would then hold rows that disagree with the
3145    /// replica, which JS's `MemorySource` reports by throwing ("Row already
3146    /// exists", "Row not found") and the poke handler by disconnecting.
3147    fn check_updated_rows(&self, patch: &[KvPatchOp], write: &KvWrite) -> Result<(), String> {
3148        let mut keyless: HashMap<String, Vec<Row>> = HashMap::new();
3149        for op in patch {
3150            let KvPatchOp::Update { key, .. } = op else {
3151                continue;
3152            };
3153            let Some(value) = write.get(key) else {
3154                continue;
3155            };
3156            let table = keys::source_name(key);
3157            let Some(t) = self.schema.tables.get(&table) else {
3158                continue;
3159            };
3160            let row = value.as_object().cloned().unwrap_or_default();
3161            if let Ok(row_key) = keys::primary_key_string(&table, &t.primary_key, &row) {
3162                if row_key != *key {
3163                    return Err(format!(
3164                        "Error: update of {key} changes its primary key to {row_key}"
3165                    ));
3166                }
3167                continue;
3168            }
3169            // JS keeps a row whose merge leaves it without a usable primary
3170            // key, and fails once a second one collides with it in the
3171            // source.
3172            let Some(source) = self.ivm_main.source(&table) else {
3173                continue;
3174            };
3175            let row: Row = Rc::new(row);
3176            let previous = self
3177                .store
3178                .borrow()
3179                .main
3180                .get(key)
3181                .map(|v| Rc::new(v.as_object().cloned().unwrap_or_default()));
3182            let replaces_itself = previous.is_some_and(|p| source.same_key(&p, &row));
3183            let rows = keyless.entry(table).or_default();
3184            if (!replaces_itself && source.contains(&row))
3185                || rows.iter().any(|r| source.same_key(r, &row))
3186            {
3187                return Err(format!("Error: Row already exists: update of {key}"));
3188            }
3189            rows.push(row);
3190        }
3191        Ok(())
3192    }
3193}
3194
3195impl Drop for Engine {
3196    fn drop(&mut self) {
3197        self.event_loop.stop();
3198        for t in self.lifecycle_timers.get_mut().drain(..) {
3199            t.cancel();
3200        }
3201        if !self.rep_closed.get() {
3202            self.database.client_stopped(&self.client_id);
3203            self.database.remove_failure_listener(&self.client_id);
3204            self.database.release();
3205        }
3206    }
3207}
3208
3209#[cfg(test)]
3210mod tests {
3211    use super::*;
3212
3213    #[test]
3214    fn iso_dates() {
3215        assert_eq!(
3216            iso_date_string(1_700_000_000_000.0),
3217            "2023-11-14T22:13:20.000Z"
3218        );
3219        assert_eq!(iso_date_string(0.0), "1970-01-01T00:00:00.000Z");
3220        assert_eq!(iso_date_string(-1.0), "1969-12-31T23:59:59.999Z");
3221    }
3222
3223    #[test]
3224    fn backoff() {
3225        assert_eq!(reconnect_backoff_ms(1, 5000.0, 0.0), 250.0);
3226        assert_eq!(reconnect_backoff_ms(1, 5000.0, 1.0), 500.0);
3227        assert_eq!(reconnect_backoff_ms(4, 5000.0, 0.5), 3000.0);
3228        assert_eq!(reconnect_backoff_ms(5, 5000.0, 0.5), 5000.0);
3229    }
3230}