Skip to main content

ch_database/spacetimedb/
mod.rs

1//! Shared SpacetimeDB helper utilities.
2//!
3//! App-owned SpacetimeDB tables, reducers, generated bindings, and concrete
4//! clients live in the owning app database crates.
5
6use std::cell::Cell;
7use std::collections::VecDeque;
8use std::fmt;
9use std::mem::ManuallyDrop;
10use std::ops::Deref;
11use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
12use std::sync::{Arc, Mutex, MutexGuard, OnceLock, RwLock, RwLockReadGuard, TryLockError, mpsc};
13use std::thread;
14use std::time::{Duration, Instant};
15
16use spacetimedb_sdk::DbContext;
17
18static RUNTIME_ENDPOINT_URL: OnceLock<String> = OnceLock::new();
19static OPTIONAL_READ_DEGRADED_UNTIL: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
20static SUBSCRIBE_ONCE_QUEUE: OnceLock<ReadProjectionQueue> = OnceLock::new();
21
22thread_local! {
23    static READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE: Cell<Option<Duration>> = const { Cell::new(None) };
24}
25
26const CONNECT_RETRY_DELAYS_MS: [u64; 2] = [150, 500];
27const OPTIONAL_READ_DEGRADE_WINDOW: Duration = Duration::from_secs(15);
28const DEFAULT_READ_PROJECTION_LOCK_TIMEOUT: Duration = Duration::from_secs(5);
29const MAX_PENDING_READ_PROJECTIONS: usize = 256;
30const READ_PROJECTION_PENDING: u8 = 0;
31const READ_PROJECTION_CANCELLED: u8 = 1;
32const READ_PROJECTION_GRANTED: u8 = 2;
33
34#[derive(Debug)]
35struct ReadProjectionWaiter {
36    ticket: u64,
37    deadline: Instant,
38    status: Arc<AtomicU8>,
39    permit: mpsc::Sender<()>,
40}
41
42#[derive(Debug, Default)]
43struct ReadProjectionQueueState {
44    next_ticket: u64,
45    serving_ticket: u64,
46    active: bool,
47    waiters: VecDeque<ReadProjectionWaiter>,
48}
49
50#[derive(Debug, Default)]
51struct ReadProjectionQueue {
52    state: Mutex<ReadProjectionQueueState>,
53}
54
55#[derive(Debug)]
56struct ReadProjectionLease<'a> {
57    queue: &'a ReadProjectionQueue,
58    ticket: u64,
59}
60
61impl Drop for ReadProjectionLease<'_> {
62    fn drop(&mut self) {
63        let Ok(mut state) = self.queue.state.lock() else {
64            return;
65        };
66        if !state.active || state.serving_ticket != self.ticket {
67            return;
68        }
69        state.active = false;
70        state.serving_ticket = self.ticket.saturating_add(1);
71        advance_read_projection_queue(&mut state);
72    }
73}
74
75/// Typed lifecycle failure while entering or executing a serialized read projection.
76#[derive(Debug, thiserror::Error, PartialEq, Eq)]
77#[non_exhaustive]
78pub enum ReadProjectionError {
79    /// The caller did not acquire its FIFO lease before the bounded admission timeout.
80    #[error(
81        "SpacetimeDB read projection is busy; FIFO admission timed out before the read started"
82    )]
83    BusyBeforeStart,
84    /// Shared queue state was poisoned by an unrecoverable synchronization failure.
85    #[error("SpacetimeDB read projection queue was poisoned")]
86    QueuePoisoned,
87    /// The monotonic queue ticket space was exhausted.
88    #[error("SpacetimeDB read projection queue exhausted its ticket space")]
89    TicketSpaceExhausted,
90    /// The caller supplied an admission timeout outside the monotonic clock range.
91    #[error("SpacetimeDB read projection admission timeout exceeded the monotonic clock range")]
92    AdmissionDeadlineOverflow,
93    /// The app-owned projection failed after it acquired the shared lease.
94    #[error("{0}")]
95    Operation(String),
96}
97
98struct ReadProjectionLockTimeoutGuard<'a> {
99    cell: &'a Cell<Option<Duration>>,
100    previous: Option<Duration>,
101}
102
103impl Drop for ReadProjectionLockTimeoutGuard<'_> {
104    fn drop(&mut self) {
105        self.cell.set(self.previous);
106    }
107}
108
109impl ReadProjectionQueue {
110    fn acquire(&self, timeout: Duration) -> Result<ReadProjectionLease<'_>, ReadProjectionError> {
111        let deadline = Instant::now()
112            .checked_add(timeout)
113            .ok_or(ReadProjectionError::AdmissionDeadlineOverflow)?;
114        let mut state = self.lock_state_until(deadline)?;
115        if state.waiters.len() >= MAX_PENDING_READ_PROJECTIONS {
116            return Err(ReadProjectionError::BusyBeforeStart);
117        }
118        let ticket = state.next_ticket;
119        state.next_ticket = state
120            .next_ticket
121            .checked_add(1)
122            .ok_or(ReadProjectionError::TicketSpaceExhausted)?;
123        if !state.active && state.waiters.is_empty() {
124            state.active = true;
125            state.serving_ticket = ticket;
126            return Ok(ReadProjectionLease {
127                queue: self,
128                ticket,
129            });
130        }
131
132        let status = Arc::new(AtomicU8::new(READ_PROJECTION_PENDING));
133        let (permit, granted) = mpsc::channel();
134        state.waiters.push_back(ReadProjectionWaiter {
135            ticket,
136            deadline,
137            status: Arc::clone(&status),
138            permit,
139        });
140        drop(state);
141
142        let remaining = deadline.saturating_duration_since(Instant::now());
143        match granted.recv_timeout(remaining) {
144            Ok(()) => Ok(ReadProjectionLease {
145                queue: self,
146                ticket,
147            }),
148            Err(mpsc::RecvTimeoutError::Timeout) => {
149                if status
150                    .compare_exchange(
151                        READ_PROJECTION_PENDING,
152                        READ_PROJECTION_CANCELLED,
153                        Ordering::AcqRel,
154                        Ordering::Acquire,
155                    )
156                    .is_ok()
157                {
158                    return Err(ReadProjectionError::BusyBeforeStart);
159                }
160                if status.load(Ordering::Acquire) != READ_PROJECTION_GRANTED {
161                    return Err(ReadProjectionError::BusyBeforeStart);
162                }
163                granted
164                    .recv()
165                    .map_err(|_| ReadProjectionError::QueuePoisoned)?;
166                Ok(ReadProjectionLease {
167                    queue: self,
168                    ticket,
169                })
170            }
171            Err(mpsc::RecvTimeoutError::Disconnected) => Err(ReadProjectionError::BusyBeforeStart),
172        }
173    }
174
175    fn lock_state_until(
176        &self,
177        deadline: Instant,
178    ) -> Result<MutexGuard<'_, ReadProjectionQueueState>, ReadProjectionError> {
179        let mut first_attempt = true;
180        loop {
181            if !first_attempt && Instant::now() >= deadline {
182                return Err(ReadProjectionError::BusyBeforeStart);
183            }
184            match self.state.try_lock() {
185                Ok(state) => return Ok(state),
186                Err(TryLockError::Poisoned(_)) => {
187                    return Err(ReadProjectionError::QueuePoisoned);
188                }
189                Err(TryLockError::WouldBlock) => {
190                    first_attempt = false;
191                    let now = Instant::now();
192                    if now >= deadline {
193                        return Err(ReadProjectionError::BusyBeforeStart);
194                    }
195                    thread::park_timeout(
196                        deadline
197                            .saturating_duration_since(now)
198                            .min(Duration::from_millis(1)),
199                    );
200                }
201            }
202        }
203    }
204}
205
206fn advance_read_projection_queue(state: &mut ReadProjectionQueueState) {
207    debug_assert!(state.waiters.len() <= MAX_PENDING_READ_PROJECTIONS);
208    while let Some(waiter) = state.waiters.pop_front() {
209        state.serving_ticket = waiter.ticket;
210        if Instant::now() >= waiter.deadline {
211            let _ = waiter.status.compare_exchange(
212                READ_PROJECTION_PENDING,
213                READ_PROJECTION_CANCELLED,
214                Ordering::AcqRel,
215                Ordering::Acquire,
216            );
217            state.serving_ticket = waiter.ticket.saturating_add(1);
218            continue;
219        }
220        if waiter
221            .status
222            .compare_exchange(
223                READ_PROJECTION_PENDING,
224                READ_PROJECTION_GRANTED,
225                Ordering::AcqRel,
226                Ordering::Acquire,
227            )
228            .is_err()
229        {
230            state.serving_ticket = waiter.ticket.saturating_add(1);
231            continue;
232        }
233        if waiter.permit.send(()).is_ok() {
234            state.active = true;
235            return;
236        }
237        waiter
238            .status
239            .store(READ_PROJECTION_CANCELLED, Ordering::Release);
240        state.serving_ticket = waiter.ticket.saturating_add(1);
241    }
242}
243
244/// Concrete SpacetimeDB database target shared by app-owned clients.
245#[derive(Clone, PartialEq, Eq)]
246pub struct ConnectionTarget {
247    /// HTTP or WebSocket endpoint used by the generated client.
248    pub endpoint_url: String,
249    /// Deployed SpacetimeDB module name.
250    pub module_name: String,
251    /// Optional bearer token used to authenticate the connection.
252    pub auth_token: Option<String>,
253}
254
255impl ConnectionTarget {
256    /// Creates an unauthenticated connection target.
257    #[must_use]
258    pub fn new(endpoint_url: impl Into<String>, module_name: impl Into<String>) -> Self {
259        Self {
260            endpoint_url: endpoint_url.into(),
261            module_name: module_name.into(),
262            auth_token: None,
263        }
264    }
265
266    /// Attaches a bearer token while keeping it redacted from debug output.
267    #[must_use]
268    pub fn with_auth_token(mut self, auth_token: impl Into<String>) -> Self {
269        self.auth_token = Some(auth_token.into());
270        self
271    }
272}
273
274impl fmt::Debug for ConnectionTarget {
275    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
276        formatter
277            .debug_struct("ConnectionTarget")
278            .field("endpoint_url", &self.endpoint_url)
279            .field("module_name", &self.module_name)
280            .field(
281                "auth_token",
282                &self.auth_token.as_ref().map(|_| "[REDACTED]"),
283            )
284            .finish()
285    }
286}
287
288/// Shared connection state reported by an app-owned generated connection builder.
289///
290/// The app builder wires `mark_disconnected` into the generated SDK's
291/// `on_disconnect` callback and calls `mark_connected` only after its required
292/// subscriptions have been applied. The shared client uses that state to rebuild
293/// the complete connection without duplicating reconnect policy in each app.
294#[derive(Clone, Default)]
295pub struct SpacetimeDbConnectionStatus {
296    connected: Arc<AtomicBool>,
297}
298
299impl SpacetimeDbConnectionStatus {
300    /// Marks the generated connection and its required subscriptions ready.
301    pub fn mark_connected(&self) {
302        self.connected.store(true, Ordering::Release);
303    }
304
305    /// Marks the generated connection unavailable so the next access reconnects.
306    pub fn mark_disconnected(&self) {
307        self.connected.store(false, Ordering::Release);
308    }
309
310    /// Returns whether the complete app-owned connection is ready for use.
311    #[must_use]
312    pub fn is_connected(&self) -> bool {
313        self.connected.load(Ordering::Acquire)
314    }
315}
316
317/// One generated connection, its worker, and resources tied to that connection.
318///
319/// App clients may retain generated subscription handles through `keep_alive`.
320/// All retained resources are released before disconnecting the generated client.
321pub struct ManagedSpacetimeDbConnection<Connection>
322where
323    Connection: DbContext + Send + 'static,
324{
325    connection: Option<Connection>,
326    worker: Option<thread::JoinHandle<()>>,
327    keepalive: Vec<Box<dyn Send + Sync>>,
328    cleanup_thread_name: String,
329    status: Option<SpacetimeDbConnectionStatus>,
330}
331
332impl<Connection> ManagedSpacetimeDbConnection<Connection>
333where
334    Connection: DbContext + Send + 'static,
335{
336    /// Wraps a generated connection and its background worker in the shared owner.
337    pub fn new(connection: Connection, worker: thread::JoinHandle<()>) -> Self {
338        Self {
339            connection: Some(connection),
340            worker: Some(worker),
341            keepalive: Vec::new(),
342            cleanup_thread_name: "spacetimedb-cleanup".to_owned(),
343            status: None,
344        }
345    }
346
347    /// Returns the generated connection while an app builder finishes setup.
348    pub fn connection(&self) -> &Connection {
349        self.connection
350            .as_ref()
351            .expect("SpacetimeDB connection is only taken during Drop")
352    }
353
354    /// Keeps a connection-scoped callback or subscription handle alive.
355    pub fn keep_alive(&mut self, resource: impl Send + Sync + 'static) {
356        self.keepalive.push(Box::new(resource));
357    }
358
359    fn set_cleanup_thread_name(&mut self, cleanup_thread_name: String) {
360        self.cleanup_thread_name = cleanup_thread_name;
361    }
362
363    fn set_status(&mut self, status: SpacetimeDbConnectionStatus) {
364        self.status = Some(status);
365    }
366}
367
368impl<Connection> Drop for ManagedSpacetimeDbConnection<Connection>
369where
370    Connection: DbContext + Send + 'static,
371{
372    fn drop(&mut self) {
373        let connection = self.connection.take();
374        let worker = self.worker.take();
375        let keepalive = std::mem::take(&mut self.keepalive);
376        if connection.is_none() && worker.is_none() && keepalive.is_empty() {
377            return;
378        }
379
380        let cleanup_thread_name = self.cleanup_thread_name.clone();
381        let _ = thread::Builder::new()
382            .name(cleanup_thread_name)
383            .spawn(move || {
384                for resource in keepalive {
385                    let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
386                        drop(resource);
387                    }));
388                }
389                if let Some(connection) = connection {
390                    let connection = ManuallyDrop::new(connection);
391                    let disconnected =
392                        std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
393                            let _ = connection.disconnect();
394                        }))
395                        .is_ok();
396                    if disconnected {
397                        let connection = ManuallyDrop::into_inner(connection);
398                        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
399                            drop(connection);
400                        }));
401                    }
402                }
403                if let Some(worker) = worker {
404                    let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
405                        let _ = worker.join();
406                    }));
407                }
408            });
409    }
410}
411
412type ReconnectBuilder<Connection> = dyn Fn(
413        &ConnectionTarget,
414        SpacetimeDbConnectionStatus,
415    ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
416    + Send
417    + Sync;
418
419enum SpacetimeDbClientConnection<Connection>
420where
421    Connection: DbContext + Send + 'static,
422{
423    Fixed(ManagedSpacetimeDbConnection<Connection>),
424    Reconnecting(RwLock<ManagedSpacetimeDbConnection<Connection>>),
425}
426
427/// Shared lifecycle owner for generated app-specific SpacetimeDB connections.
428///
429/// Apps still own their generated `DbConnection` type, table accessors, reducers,
430/// subscriptions, and row projection logic. This shared owner centralizes retry,
431/// reconnect, worker ownership, optional-read degradation, serial read-projection
432/// throttling, and panic-tolerant cleanup.
433pub struct SpacetimeDbClient<Connection>
434where
435    Connection: DbContext + Send + 'static,
436{
437    target: ConnectionTarget,
438    connection_label: String,
439    connection: SpacetimeDbClientConnection<Connection>,
440    reconnect_builder: Option<Arc<ReconnectBuilder<Connection>>>,
441}
442
443/// Process-local registry for sharing one reconnecting client per database target.
444pub struct SpacetimeDbClientRegistry<Connection>
445where
446    Connection: DbContext + Send + 'static,
447{
448    clients: Mutex<Vec<(ConnectionTarget, Arc<SpacetimeDbClient<Connection>>)>>,
449}
450
451/// Failure while resolving a shared SpacetimeDB client.
452#[derive(Debug, thiserror::Error, PartialEq, Eq)]
453#[non_exhaustive]
454pub enum SpacetimeDbClientRegistryError {
455    /// The process-local registry can no longer be accessed safely.
456    #[error("SpacetimeDB client registry lock was poisoned")]
457    LockPoisoned,
458    /// A generated app client could not connect to its configured target.
459    #[error("SpacetimeDB client {label} could not connect: {message}")]
460    Connection {
461        /// Human-readable owner label supplied by the app client.
462        label: String,
463        /// Concrete connection failure returned by the generated client builder.
464        message: String,
465    },
466}
467
468impl<Connection> SpacetimeDbClientRegistry<Connection>
469where
470    Connection: DbContext + Send + 'static,
471{
472    /// Creates an empty shared-client registry.
473    #[must_use]
474    pub const fn new() -> Self {
475        Self {
476            clients: Mutex::new(Vec::new()),
477        }
478    }
479
480    /// Returns the existing client for `target` or builds and retains one.
481    pub fn get_or_connect_reconnecting<Build>(
482        &self,
483        target: ConnectionTarget,
484        connection_label: impl Into<String>,
485        build: Build,
486    ) -> Result<Arc<SpacetimeDbClient<Connection>>, SpacetimeDbClientRegistryError>
487    where
488        Build: Fn(
489                &ConnectionTarget,
490                SpacetimeDbConnectionStatus,
491            ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
492            + Send
493            + Sync
494            + 'static,
495    {
496        let connection_label = connection_label.into();
497        let mut clients = self
498            .clients
499            .lock()
500            .map_err(|_| SpacetimeDbClientRegistryError::LockPoisoned)?;
501        if let Some((_, client)) = clients.iter().find(|(candidate, _)| candidate == &target) {
502            return Ok(Arc::clone(client));
503        }
504
505        let client = Arc::new(
506            SpacetimeDbClient::connect_reconnecting(
507                target.clone(),
508                connection_label.clone(),
509                build,
510            )
511            .map_err(|message| SpacetimeDbClientRegistryError::Connection {
512                label: connection_label,
513                message,
514            })?,
515        );
516        clients.push((target, Arc::clone(&client)));
517        Ok(client)
518    }
519}
520
521impl<Connection> Default for SpacetimeDbClientRegistry<Connection>
522where
523    Connection: DbContext + Send + 'static,
524{
525    fn default() -> Self {
526        Self::new()
527    }
528}
529
530/// Read guard for the generated connection currently owned by the shared client.
531pub struct SpacetimeDbConnectionGuard<'a, Connection>
532where
533    Connection: DbContext + Send + 'static,
534{
535    connection: RwLockReadGuard<'a, ManagedSpacetimeDbConnection<Connection>>,
536}
537
538impl<Connection> Deref for SpacetimeDbConnectionGuard<'_, Connection>
539where
540    Connection: DbContext + Send + 'static,
541{
542    type Target = Connection;
543
544    fn deref(&self) -> &Self::Target {
545        self.connection.connection()
546    }
547}
548
549impl<Connection> SpacetimeDbClient<Connection>
550where
551    Connection: DbContext + Send + 'static,
552{
553    /// Connects a required app-owned client using the shared retry and cleanup policy.
554    pub fn connect<Build>(
555        target: ConnectionTarget,
556        connection_label: impl Into<String>,
557        build: Build,
558    ) -> Result<Self, String>
559    where
560        Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
561            + Send
562            + 'static,
563    {
564        let connection_label = connection_label.into();
565        let connection =
566            connect_with_retries(target.clone(), connection_label.clone(), build, false)?;
567        Ok(Self {
568            target,
569            connection_label,
570            connection: SpacetimeDbClientConnection::Fixed(connection),
571            reconnect_builder: None,
572        })
573    }
574
575    /// Connects an optional read projection unless its shared circuit breaker is degraded.
576    pub fn connect_optional_read<Build>(
577        target: ConnectionTarget,
578        connection_label: impl Into<String>,
579        build: Build,
580    ) -> Result<Self, String>
581    where
582        Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
583            + Send
584            + 'static,
585    {
586        ensure_optional_read_not_degraded()?;
587        let connection_label = connection_label.into();
588        let connection =
589            connect_with_retries(target.clone(), connection_label.clone(), build, true)?;
590        Ok(Self {
591            target,
592            connection_label,
593            connection: SpacetimeDbClientConnection::Fixed(connection),
594            reconnect_builder: None,
595        })
596    }
597
598    /// Connects a required app-owned client and restores it after disconnection.
599    pub fn connect_reconnecting<Build>(
600        target: ConnectionTarget,
601        connection_label: impl Into<String>,
602        build: Build,
603    ) -> Result<Self, String>
604    where
605        Build: Fn(
606                &ConnectionTarget,
607                SpacetimeDbConnectionStatus,
608            ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
609            + Send
610            + Sync
611            + 'static,
612    {
613        let connection_label = connection_label.into();
614        let reconnect_builder: Arc<ReconnectBuilder<Connection>> = Arc::new(build);
615        let connection = connect_reconnecting_with_retries(
616            target.clone(),
617            connection_label.clone(),
618            reconnect_builder.clone(),
619        )?;
620        Ok(Self {
621            target,
622            connection_label,
623            connection: SpacetimeDbClientConnection::Reconnecting(RwLock::new(connection)),
624            reconnect_builder: Some(reconnect_builder),
625        })
626    }
627
628    /// Returns a fixed generated connection owned by this lifecycle guard.
629    ///
630    /// Reconnecting clients use `reconnecting_connection` so access remains locked
631    /// while a disconnected connection is replaced.
632    pub fn connection(&self) -> &Connection {
633        let SpacetimeDbClientConnection::Fixed(connection) = &self.connection else {
634            panic!("reconnecting SpacetimeDB clients use reconnecting_connection")
635        };
636        connection.connection()
637    }
638
639    /// Returns the current reconnecting connection, rebuilding it when required.
640    pub fn reconnecting_connection(
641        &self,
642    ) -> Result<SpacetimeDbConnectionGuard<'_, Connection>, String> {
643        self.ensure_connected()?;
644        let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
645            return Err("fixed SpacetimeDB clients use connection".to_owned());
646        };
647        connection
648            .read()
649            .map(|connection| SpacetimeDbConnectionGuard { connection })
650            .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())
651    }
652
653    /// Returns the last connection status reported by a reconnecting client.
654    #[must_use]
655    pub fn is_connected(&self) -> bool {
656        match &self.connection {
657            SpacetimeDbClientConnection::Fixed(_) => true,
658            SpacetimeDbClientConnection::Reconnecting(connection) => {
659                connection.read().is_ok_and(|connection| {
660                    connection
661                        .status
662                        .as_ref()
663                        .is_some_and(SpacetimeDbConnectionStatus::is_connected)
664                })
665            }
666        }
667    }
668
669    /// Invalidates the current reconnecting connection after an incomplete
670    /// connection-scoped operation.
671    ///
672    /// The next `reconnecting_connection` access rebuilds the app-owned
673    /// connection through its registered builder.
674    pub fn invalidate_reconnecting_connection(&self) -> Result<(), String> {
675        let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
676            return Err("fixed SpacetimeDB clients cannot be invalidated".to_owned());
677        };
678        let connection = connection
679            .read()
680            .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
681        let status = connection
682            .status
683            .as_ref()
684            .ok_or_else(|| "SpacetimeDB reconnecting connection has no status owner".to_owned())?;
685        status.mark_disconnected();
686        Ok(())
687    }
688
689    fn ensure_connected(&self) -> Result<(), String> {
690        let Some(reconnect_builder) = self.reconnect_builder.as_ref() else {
691            return Ok(());
692        };
693        let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
694            return Err("SpacetimeDB reconnect builder has no reconnecting connection".to_owned());
695        };
696        {
697            let connection = connection
698                .read()
699                .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
700            if connection
701                .status
702                .as_ref()
703                .is_some_and(SpacetimeDbConnectionStatus::is_connected)
704            {
705                return Ok(());
706            }
707        }
708
709        let mut connection = connection
710            .write()
711            .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
712        if connection
713            .status
714            .as_ref()
715            .is_some_and(SpacetimeDbConnectionStatus::is_connected)
716        {
717            return Ok(());
718        }
719        *connection = connect_reconnecting_with_retries(
720            self.target.clone(),
721            self.connection_label.clone(),
722            reconnect_builder.clone(),
723        )?;
724        Ok(())
725    }
726}
727
728/// Runs one read projection while holding the shared FIFO subscription lease.
729///
730/// Subscription projections share one generated-client cache, so overlapping
731/// projections are not safe. FIFO admission prevents a burst of newer readers
732/// from starving an earlier request, while the caller-scoped admission timeout
733/// keeps the queue bounded.
734pub fn with_serialized_read_projection<T>(
735    read_projection: impl FnOnce() -> Result<T, String>,
736) -> Result<T, String> {
737    with_serialized_read_projection_typed(read_projection).map_err(|error| match error {
738        ReadProjectionError::Operation(error) => error,
739        error => error.to_string(),
740    })
741}
742
743/// Typed variant of [`with_serialized_read_projection`].
744///
745/// Apps that need to distinguish admission pressure from a projection failure
746/// should use this owner instead of classifying error strings.
747pub fn with_serialized_read_projection_typed<T>(
748    read_projection: impl FnOnce() -> Result<T, String>,
749) -> Result<T, ReadProjectionError> {
750    let lock_timeout = READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE
751        .with(|timeout| timeout.get())
752        .unwrap_or(DEFAULT_READ_PROJECTION_LOCK_TIMEOUT);
753    let _lease = subscribe_once_queue().acquire(lock_timeout)?;
754    let result = read_projection();
755    if let Err(error) = &result {
756        record_optional_read_failure(error.as_str());
757    }
758    result.map_err(ReadProjectionError::Operation)
759}
760
761/// Applies a shorter projection-lock budget to reads that are optional for the
762/// caller, without weakening required product reads on the same client.
763pub fn with_read_projection_lock_timeout<T>(timeout: Duration, operation: impl FnOnce() -> T) -> T {
764    READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE.with(|cell| {
765        let _guard = ReadProjectionLockTimeoutGuard {
766            cell,
767            previous: cell.replace(Some(timeout)),
768        };
769        operation()
770    })
771}
772
773/// Returns the caller-scoped optional projection budget, when one is active.
774/// App-owned clients use the same budget for subscription cleanup so a fast
775/// optional read cannot outlive its outer bootstrap timeout.
776#[must_use]
777pub fn read_projection_timeout_override() -> Option<Duration> {
778    READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE.with(|timeout| timeout.get())
779}
780
781/// Sets the process-wide SpacetimeDB endpoint discovered by the runtime adapter.
782///
783/// The first non-empty endpoint wins for the lifetime of the process.
784pub fn set_runtime_endpoint_url(endpoint_url: impl Into<String>) {
785    let endpoint_url = endpoint_url.into().trim().to_string();
786    if endpoint_url.is_empty() {
787        return;
788    }
789
790    let _ = RUNTIME_ENDPOINT_URL.set(endpoint_url);
791}
792
793/// Returns the process-wide SpacetimeDB endpoint, when one has been configured.
794pub fn runtime_endpoint_url() -> Option<String> {
795    RUNTIME_ENDPOINT_URL.get().cloned()
796}
797
798/// Invalid component in a scoped storage key.
799#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)]
800#[non_exhaustive]
801pub enum ScopedKeyError {
802    /// A component was empty after trimming.
803    #[error("scoped key parts must not be empty")]
804    EmptyPart,
805    /// A component contains the separator and would make the key ambiguous.
806    #[error("scoped key parts must not contain ':'")]
807    ContainsSeparator,
808    /// Leading or trailing whitespace would change the key silently.
809    #[error("scoped key parts must not contain surrounding whitespace")]
810    SurroundingWhitespace,
811}
812
813/// Builds a validated two-part colon-delimited key.
814pub fn scoped_key(left: &str, right: &str) -> Result<String, ScopedKeyError> {
815    validate_scoped_key_part(left)?;
816    validate_scoped_key_part(right)?;
817    Ok(format!("{left}:{right}"))
818}
819
820/// Builds a validated three-part colon-delimited key.
821pub fn scoped_key3(left: &str, middle: &str, right: &str) -> Result<String, ScopedKeyError> {
822    validate_scoped_key_part(left)?;
823    validate_scoped_key_part(middle)?;
824    validate_scoped_key_part(right)?;
825    Ok(format!("{left}:{middle}:{right}"))
826}
827
828fn validate_scoped_key_part(part: &str) -> Result<(), ScopedKeyError> {
829    if part.trim().is_empty() {
830        return Err(ScopedKeyError::EmptyPart);
831    }
832    if part.trim() != part {
833        return Err(ScopedKeyError::SurroundingWhitespace);
834    }
835    if part.contains(':') {
836        return Err(ScopedKeyError::ContainsSeparator);
837    }
838    Ok(())
839}
840
841fn connect_with_retries<Connection, Build>(
842    target: ConnectionTarget,
843    connection_label: String,
844    mut build: Build,
845    optional_read: bool,
846) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
847where
848    Connection: DbContext + Send + 'static,
849    Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
850        + Send
851        + 'static,
852{
853    connect_managed_with_retries(
854        target,
855        connection_label,
856        move |target| {
857            let (connection, worker) = build(target)?;
858            Ok(ManagedSpacetimeDbConnection::new(connection, worker))
859        },
860        optional_read,
861    )
862}
863
864fn connect_reconnecting_with_retries<Connection>(
865    target: ConnectionTarget,
866    connection_label: String,
867    build: Arc<ReconnectBuilder<Connection>>,
868) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
869where
870    Connection: DbContext + Send + 'static,
871{
872    let readiness_label = connection_label.clone();
873    connect_managed_with_retries(
874        target,
875        connection_label,
876        move |target| {
877            let status = SpacetimeDbConnectionStatus::default();
878            let mut connection = build(target, status.clone())?;
879            if !status.is_connected() {
880                return Err(format!(
881                    "SpacetimeDB {readiness_label} connection setup did not mark the connection ready"
882                ));
883            }
884            connection.set_status(status);
885            Ok(connection)
886        },
887        false,
888    )
889}
890
891fn connect_managed_with_retries<Connection, Build>(
892    target: ConnectionTarget,
893    connection_label: String,
894    mut build: Build,
895    optional_read: bool,
896) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
897where
898    Connection: DbContext + Send + 'static,
899    Build: FnMut(&ConnectionTarget) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
900        + Send
901        + 'static,
902{
903    let cleanup_thread_name = cleanup_thread_name(connection_label.as_str());
904    let endpoint_url = target.endpoint_url.clone();
905    let module_name = target.module_name.clone();
906    let thread_endpoint_url = endpoint_url.clone();
907    let thread_module_name = module_name.clone();
908    let thread_label = connection_label.clone();
909    let connection_thread = thread::spawn(move || {
910        let mut last_error = None;
911        for attempt in 0..=CONNECT_RETRY_DELAYS_MS.len() {
912            match build(&target) {
913                Ok(connection) => return Ok(connection),
914                Err(error) => {
915                    last_error = Some(error);
916                    if let Some(delay_ms) = CONNECT_RETRY_DELAYS_MS.get(attempt) {
917                        thread::sleep(Duration::from_millis(*delay_ms));
918                    }
919                }
920            }
921        }
922        Err(last_error.unwrap_or_else(|| {
923            format!(
924                "SpacetimeDB {thread_label} connection to {thread_module_name} at {thread_endpoint_url} failed"
925            )
926        }))
927    });
928
929    let mut connection = match connection_thread.join() {
930        Ok(Ok(connection)) => connection,
931        Ok(Err(error)) => {
932            if optional_read {
933                record_optional_read_failure(error.as_str());
934            }
935            return Err(error);
936        }
937        Err(payload) => {
938            let message = panic_message(
939                &payload,
940                format!(
941                    "SpacetimeDB {connection_label} connection to {module_name} at {endpoint_url} panicked"
942                ),
943            );
944            if optional_read {
945                record_optional_read_failure(message.as_str());
946            }
947            return Err(message);
948        }
949    };
950    connection.set_cleanup_thread_name(cleanup_thread_name);
951    Ok(connection)
952}
953
954fn cleanup_thread_name(connection_label: &str) -> String {
955    let label = connection_label
956        .chars()
957        .map(|value| {
958            if value.is_ascii_alphanumeric() {
959                value.to_ascii_lowercase()
960            } else {
961                '-'
962            }
963        })
964        .collect::<String>()
965        .trim_matches('-')
966        .to_string();
967    match label.as_str() {
968        "" => "spacetimedb-cleanup".to_owned(),
969        _ => format!("{label}-cleanup"),
970    }
971}
972
973fn panic_message(payload: &Box<dyn std::any::Any + Send>, fallback: String) -> String {
974    if let Some(message) = payload.downcast_ref::<&str>() {
975        format!("{fallback}: {message}")
976    } else if let Some(message) = payload.downcast_ref::<String>() {
977        format!("{fallback}: {message}")
978    } else {
979        fallback
980    }
981}
982
983fn optional_read_degraded_until() -> &'static Mutex<Option<Instant>> {
984    OPTIONAL_READ_DEGRADED_UNTIL.get_or_init(|| Mutex::new(None))
985}
986
987fn subscribe_once_queue() -> &'static ReadProjectionQueue {
988    SUBSCRIBE_ONCE_QUEUE.get_or_init(ReadProjectionQueue::default)
989}
990
991fn ensure_optional_read_not_degraded() -> Result<(), String> {
992    let mut degraded_until = optional_read_degraded_until()
993        .lock()
994        .map_err(|_| "SpacetimeDB optional read circuit breaker was poisoned".to_string())?;
995    let Some(until) = *degraded_until else {
996        return Ok(());
997    };
998    if Instant::now() < until {
999        return Err(
1000            "SpacetimeDB optional read projections are temporarily degraded after a recent live-state failure"
1001                .to_string(),
1002        );
1003    }
1004    *degraded_until = None;
1005    Ok(())
1006}
1007
1008/// Records a transport-level optional-read failure in the shared circuit breaker.
1009pub fn record_optional_read_failure(error: &str) {
1010    if !is_optional_read_circuit_breaking_error(error) {
1011        return;
1012    }
1013    if let Ok(mut degraded_until) = optional_read_degraded_until().lock() {
1014        *degraded_until = Some(Instant::now() + OPTIONAL_READ_DEGRADE_WINDOW);
1015    }
1016}
1017
1018/// Reports whether an optional-read error should degrade its caller-facing feature.
1019pub fn is_optional_read_degrading_error(error: &str) -> bool {
1020    let error = error.to_ascii_lowercase();
1021    [
1022        "connection refused",
1023        "subscription did not apply",
1024        "subscription failed before read projection",
1025        "read projection is busy",
1026        "live-state connection",
1027        "panicked",
1028        "buffer should be unique",
1029    ]
1030    .into_iter()
1031    .any(|needle| error.contains(needle))
1032}
1033
1034fn is_optional_read_circuit_breaking_error(error: &str) -> bool {
1035    let error = error.to_ascii_lowercase();
1036    [
1037        "connection refused",
1038        "live-state connection",
1039        "panicked",
1040        "buffer should be unique",
1041    ]
1042    .into_iter()
1043    .any(|needle| error.contains(needle))
1044}
1045
1046#[cfg(test)]
1047mod tests {
1048    use super::{
1049        ConnectionTarget, MAX_PENDING_READ_PROJECTIONS, READ_PROJECTION_CANCELLED,
1050        ReadProjectionError, ReadProjectionQueue, ReadProjectionWaiter,
1051        SpacetimeDbConnectionStatus, is_optional_read_circuit_breaking_error,
1052        is_optional_read_degrading_error, runtime_endpoint_url, scoped_key, scoped_key3,
1053        set_runtime_endpoint_url, with_serialized_read_projection,
1054        with_serialized_read_projection_typed,
1055    };
1056    use std::sync::atomic::AtomicU8;
1057    use std::sync::{Arc, Mutex, mpsc};
1058    use std::thread;
1059    use std::time::{Duration, Instant};
1060
1061    #[test]
1062    fn connection_target_debug_output_redacts_auth_token() {
1063        let target = ConnectionTarget::new("https://database.example.invalid", "module")
1064            .with_auth_token("visible-auth-token");
1065
1066        let debug = format!("{target:?}");
1067
1068        assert!(!debug.contains("visible-auth-token"));
1069        assert!(debug.contains("[REDACTED]"));
1070    }
1071
1072    #[test]
1073    fn reconnect_status_is_shared_without_relaxed_visibility() {
1074        let status = SpacetimeDbConnectionStatus::default();
1075        let callback_status = status.clone();
1076
1077        assert!(!status.is_connected());
1078        status.mark_connected();
1079        assert!(callback_status.is_connected());
1080        callback_status.mark_disconnected();
1081        assert!(!status.is_connected());
1082    }
1083
1084    #[test]
1085    fn scoped_key_rejects_blank_parts() {
1086        assert!(scoped_key("", "session").is_err());
1087        assert!(scoped_key("owner", " ").is_err());
1088        assert_eq!(
1089            scoped_key("owner", "session").expect("valid key"),
1090            "owner:session"
1091        );
1092        assert!(scoped_key("owner:project", "timeline").is_err());
1093        assert!(scoped_key(" owner", "timeline").is_err());
1094        assert_eq!(
1095            scoped_key3("owner", "project", "timeline").expect("valid key"),
1096            "owner:project:timeline"
1097        );
1098    }
1099
1100    #[test]
1101    fn runtime_endpoint_url_ignores_blank_values() {
1102        set_runtime_endpoint_url(" ");
1103
1104        assert_eq!(runtime_endpoint_url(), None);
1105    }
1106
1107    #[test]
1108    fn optional_read_degrade_detection_matches_live_state_failures() {
1109        assert!(is_optional_read_degrading_error(
1110            "SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
1111        ));
1112        assert!(is_optional_read_degrading_error(
1113            "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
1114        ));
1115        assert!(is_optional_read_degrading_error(
1116            "thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
1117        ));
1118    }
1119
1120    #[test]
1121    fn optional_read_degrade_detection_ignores_domain_errors() {
1122        assert!(!is_optional_read_degrading_error(
1123            "requested record does not exist"
1124        ));
1125        assert!(!is_optional_read_degrading_error(
1126            "stored payload could not be deserialized"
1127        ));
1128    }
1129
1130    #[test]
1131    fn optional_read_circuit_breaker_ignores_subscription_apply_timeouts() {
1132        assert!(!is_optional_read_circuit_breaking_error(
1133            "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
1134        ));
1135        assert!(is_optional_read_degrading_error(
1136            "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
1137        ));
1138        assert!(is_optional_read_circuit_breaking_error(
1139            "SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
1140        ));
1141        assert!(is_optional_read_circuit_breaking_error(
1142            "thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
1143        ));
1144    }
1145
1146    #[test]
1147    fn typed_projection_errors_preserve_operation_failures_without_string_classification() {
1148        let typed_error = with_serialized_read_projection_typed::<()>(|| {
1149            Err("domain projection failed".to_string())
1150        })
1151        .expect_err("typed projection failure");
1152        assert_eq!(
1153            typed_error,
1154            ReadProjectionError::Operation("domain projection failed".to_string())
1155        );
1156
1157        let legacy_error =
1158            with_serialized_read_projection::<()>(|| Err("domain projection failed".to_string()))
1159                .expect_err("legacy projection failure");
1160        assert_eq!(legacy_error, "domain projection failed");
1161    }
1162
1163    #[test]
1164    fn serialized_read_projection_reports_typed_bounded_admission_failure() {
1165        let queue = ReadProjectionQueue::default();
1166        let _lease = queue
1167            .acquire(Duration::from_secs(1))
1168            .expect("first projection lease");
1169        let started_at = Instant::now();
1170
1171        let error = queue
1172            .acquire(Duration::from_millis(25))
1173            .expect_err("contended projection must remain bounded");
1174
1175        assert_eq!(error, ReadProjectionError::BusyBeforeStart);
1176        assert!(error.to_string().contains("read projection is busy"));
1177        assert!(started_at.elapsed() < Duration::from_secs(1));
1178        assert!(is_optional_read_degrading_error(error.to_string().as_str()));
1179    }
1180
1181    #[test]
1182    fn projection_admission_deadline_includes_queue_state_lock_contention() {
1183        let queue = Arc::new(ReadProjectionQueue::default());
1184        let state = queue.state.lock().expect("projection queue state");
1185        let contender_queue = Arc::clone(&queue);
1186        let (result_tx, result_rx) = mpsc::channel();
1187        let contender = thread::spawn(move || {
1188            result_tx
1189                .send(
1190                    contender_queue
1191                        .acquire(Duration::from_millis(25))
1192                        .map(|_| ()),
1193                )
1194                .expect("contender result receiver");
1195        });
1196
1197        let started_at = Instant::now();
1198        assert_eq!(
1199            result_rx
1200                .recv_timeout(Duration::from_millis(100))
1201                .expect("queue-state contention must honor the admission deadline"),
1202            Err(ReadProjectionError::BusyBeforeStart)
1203        );
1204        assert!(started_at.elapsed() < Duration::from_millis(100));
1205        assert_eq!(state.next_ticket, 0);
1206        drop(state);
1207        contender.join().expect("projection contender");
1208    }
1209
1210    #[test]
1211    fn projection_waiter_is_not_admitted_after_its_deadline() {
1212        let queue = Arc::new(ReadProjectionQueue::default());
1213        let first_lease = queue
1214            .acquire(Duration::from_secs(1))
1215            .expect("first projection lease");
1216
1217        let waiter_queue = Arc::clone(&queue);
1218        let (waiter_result_tx, waiter_result_rx) = mpsc::channel();
1219        let waiter = thread::spawn(move || {
1220            waiter_result_tx
1221                .send(waiter_queue.acquire(Duration::from_millis(25)).map(|_| ()))
1222                .expect("waiter result receiver");
1223        });
1224        wait_for_projection_ticket_count(queue.as_ref(), 2);
1225
1226        let state = queue.state.lock().expect("projection queue state");
1227        let started_at = Instant::now();
1228        assert_eq!(
1229            waiter_result_rx
1230                .recv_timeout(Duration::from_millis(100))
1231                .expect("waiter must honor its deadline without reacquiring queue state"),
1232            Err(ReadProjectionError::BusyBeforeStart)
1233        );
1234        assert!(started_at.elapsed() < Duration::from_millis(100));
1235        drop(state);
1236        waiter.join().expect("projection waiter");
1237        drop(first_lease);
1238    }
1239
1240    #[test]
1241    fn cancelled_projection_backlog_is_bounded_and_released_in_one_pass() {
1242        let queue = ReadProjectionQueue::default();
1243        let first_lease = queue
1244            .acquire(Duration::from_secs(1))
1245            .expect("first projection lease");
1246        let mut cancelled_receivers = Vec::with_capacity(MAX_PENDING_READ_PROJECTIONS);
1247
1248        {
1249            let mut state = queue.state.lock().expect("projection queue state");
1250            for _ in 0..MAX_PENDING_READ_PROJECTIONS {
1251                let ticket = state.next_ticket;
1252                state.next_ticket = state.next_ticket.checked_add(1).expect("test ticket space");
1253                let (permit, receiver) = mpsc::channel();
1254                cancelled_receivers.push(receiver);
1255                state.waiters.push_back(ReadProjectionWaiter {
1256                    ticket,
1257                    deadline: Instant::now() + Duration::from_secs(1),
1258                    status: Arc::new(AtomicU8::new(READ_PROJECTION_CANCELLED)),
1259                    permit,
1260                });
1261            }
1262        }
1263
1264        assert_eq!(
1265            queue
1266                .acquire(Duration::from_millis(25))
1267                .expect_err("bounded backlog must reject another waiter"),
1268            ReadProjectionError::BusyBeforeStart
1269        );
1270
1271        let started_at = Instant::now();
1272        drop(first_lease);
1273        assert!(started_at.elapsed() < Duration::from_secs(1));
1274
1275        let state = queue.state.lock().expect("projection queue state");
1276        assert!(state.waiters.is_empty());
1277        assert!(!state.active);
1278        drop(cancelled_receivers);
1279    }
1280
1281    #[test]
1282    fn serialized_read_projection_serves_waiters_in_fifo_order() {
1283        let queue = Arc::new(ReadProjectionQueue::default());
1284        let first_lease = queue
1285            .acquire(Duration::from_secs(1))
1286            .expect("first projection lease");
1287        let order = Arc::new(Mutex::new(Vec::new()));
1288
1289        let second_queue = Arc::clone(&queue);
1290        let second_order = Arc::clone(&order);
1291        let second = thread::spawn(move || {
1292            let _lease = second_queue
1293                .acquire(Duration::from_secs(1))
1294                .expect("second projection lease");
1295            second_order.lock().expect("second order").push(2);
1296        });
1297        wait_for_projection_ticket_count(queue.as_ref(), 2);
1298
1299        let third_queue = Arc::clone(&queue);
1300        let third_order = Arc::clone(&order);
1301        let third = thread::spawn(move || {
1302            let _lease = third_queue
1303                .acquire(Duration::from_secs(1))
1304                .expect("third projection lease");
1305            third_order.lock().expect("third order").push(3);
1306        });
1307        wait_for_projection_ticket_count(queue.as_ref(), 3);
1308
1309        drop(first_lease);
1310        second.join().expect("second projection worker");
1311        third.join().expect("third projection worker");
1312
1313        assert_eq!(order.lock().expect("projection order").as_slice(), [2, 3]);
1314    }
1315
1316    #[test]
1317    fn cancelled_projection_ticket_does_not_block_the_next_waiter() {
1318        let queue = Arc::new(ReadProjectionQueue::default());
1319        let first_lease = queue
1320            .acquire(Duration::from_secs(1))
1321            .expect("first projection lease");
1322
1323        let cancelled_queue = Arc::clone(&queue);
1324        let cancelled = thread::spawn(move || {
1325            cancelled_queue
1326                .acquire(Duration::from_millis(25))
1327                .map(|_| ())
1328        });
1329        assert_eq!(
1330            cancelled.join().expect("cancelled projection worker"),
1331            Err(ReadProjectionError::BusyBeforeStart)
1332        );
1333
1334        let next_queue = Arc::clone(&queue);
1335        let next = thread::spawn(move || next_queue.acquire(Duration::from_secs(1)).is_ok());
1336        wait_for_projection_ticket_count(queue.as_ref(), 3);
1337        drop(first_lease);
1338
1339        assert!(next.join().expect("next projection worker"));
1340    }
1341
1342    #[test]
1343    fn panicking_projection_releases_its_fifo_lease() {
1344        let queue = ReadProjectionQueue::default();
1345
1346        let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1347            let _lease = queue
1348                .acquire(Duration::from_secs(1))
1349                .expect("panicking projection lease");
1350            panic!("projection panic fixture");
1351        }));
1352
1353        assert!(panic.is_err());
1354        assert!(queue.acquire(Duration::from_millis(25)).is_ok());
1355    }
1356
1357    fn wait_for_projection_ticket_count(queue: &ReadProjectionQueue, expected: u64) {
1358        let deadline = Instant::now() + Duration::from_secs(1);
1359        loop {
1360            if queue
1361                .state
1362                .lock()
1363                .expect("projection queue state")
1364                .next_ticket
1365                >= expected
1366            {
1367                return;
1368            }
1369            assert!(
1370                Instant::now() < deadline,
1371                "projection waiter did not enter the FIFO queue"
1372            );
1373            thread::yield_now();
1374        }
1375    }
1376}