1use std::cell::Cell;
7use std::collections::VecDeque;
8use std::fmt;
9use std::mem::ManuallyDrop;
10use std::ops::Deref;
11use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
12use std::sync::{Arc, Mutex, MutexGuard, OnceLock, RwLock, RwLockReadGuard, TryLockError, mpsc};
13use std::thread;
14use std::time::{Duration, Instant};
15
16use spacetimedb_sdk::DbContext;
17
18static RUNTIME_ENDPOINT_URL: OnceLock<String> = OnceLock::new();
19static OPTIONAL_READ_DEGRADED_UNTIL: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
20static SUBSCRIBE_ONCE_QUEUE: OnceLock<ReadProjectionQueue> = OnceLock::new();
21
22thread_local! {
23 static READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE: Cell<Option<Duration>> = const { Cell::new(None) };
24}
25
26const CONNECT_RETRY_DELAYS_MS: [u64; 2] = [150, 500];
27const OPTIONAL_READ_DEGRADE_WINDOW: Duration = Duration::from_secs(15);
28const DEFAULT_READ_PROJECTION_LOCK_TIMEOUT: Duration = Duration::from_secs(5);
29const MAX_PENDING_READ_PROJECTIONS: usize = 256;
30const READ_PROJECTION_PENDING: u8 = 0;
31const READ_PROJECTION_CANCELLED: u8 = 1;
32const READ_PROJECTION_GRANTED: u8 = 2;
33
34#[derive(Debug)]
35struct ReadProjectionWaiter {
36 ticket: u64,
37 deadline: Instant,
38 status: Arc<AtomicU8>,
39 permit: mpsc::Sender<()>,
40}
41
42#[derive(Debug, Default)]
43struct ReadProjectionQueueState {
44 next_ticket: u64,
45 serving_ticket: u64,
46 active: bool,
47 waiters: VecDeque<ReadProjectionWaiter>,
48}
49
50#[derive(Debug, Default)]
51struct ReadProjectionQueue {
52 state: Mutex<ReadProjectionQueueState>,
53}
54
55#[derive(Debug)]
56struct ReadProjectionLease<'a> {
57 queue: &'a ReadProjectionQueue,
58 ticket: u64,
59}
60
61impl Drop for ReadProjectionLease<'_> {
62 fn drop(&mut self) {
63 let Ok(mut state) = self.queue.state.lock() else {
64 return;
65 };
66 if !state.active || state.serving_ticket != self.ticket {
67 return;
68 }
69 state.active = false;
70 state.serving_ticket = self.ticket.saturating_add(1);
71 advance_read_projection_queue(&mut state);
72 }
73}
74
75#[derive(Debug, thiserror::Error, PartialEq, Eq)]
77#[non_exhaustive]
78pub enum ReadProjectionError {
79 #[error(
81 "SpacetimeDB read projection is busy; FIFO admission timed out before the read started"
82 )]
83 BusyBeforeStart,
84 #[error("SpacetimeDB read projection queue was poisoned")]
86 QueuePoisoned,
87 #[error("SpacetimeDB read projection queue exhausted its ticket space")]
89 TicketSpaceExhausted,
90 #[error("SpacetimeDB read projection admission timeout exceeded the monotonic clock range")]
92 AdmissionDeadlineOverflow,
93 #[error("{0}")]
95 Operation(String),
96}
97
98struct ReadProjectionLockTimeoutGuard<'a> {
99 cell: &'a Cell<Option<Duration>>,
100 previous: Option<Duration>,
101}
102
103impl Drop for ReadProjectionLockTimeoutGuard<'_> {
104 fn drop(&mut self) {
105 self.cell.set(self.previous);
106 }
107}
108
109impl ReadProjectionQueue {
110 fn acquire(&self, timeout: Duration) -> Result<ReadProjectionLease<'_>, ReadProjectionError> {
111 let deadline = Instant::now()
112 .checked_add(timeout)
113 .ok_or(ReadProjectionError::AdmissionDeadlineOverflow)?;
114 let mut state = self.lock_state_until(deadline)?;
115 if state.waiters.len() >= MAX_PENDING_READ_PROJECTIONS {
116 return Err(ReadProjectionError::BusyBeforeStart);
117 }
118 let ticket = state.next_ticket;
119 state.next_ticket = state
120 .next_ticket
121 .checked_add(1)
122 .ok_or(ReadProjectionError::TicketSpaceExhausted)?;
123 if !state.active && state.waiters.is_empty() {
124 state.active = true;
125 state.serving_ticket = ticket;
126 return Ok(ReadProjectionLease {
127 queue: self,
128 ticket,
129 });
130 }
131
132 let status = Arc::new(AtomicU8::new(READ_PROJECTION_PENDING));
133 let (permit, granted) = mpsc::channel();
134 state.waiters.push_back(ReadProjectionWaiter {
135 ticket,
136 deadline,
137 status: Arc::clone(&status),
138 permit,
139 });
140 drop(state);
141
142 let remaining = deadline.saturating_duration_since(Instant::now());
143 match granted.recv_timeout(remaining) {
144 Ok(()) => Ok(ReadProjectionLease {
145 queue: self,
146 ticket,
147 }),
148 Err(mpsc::RecvTimeoutError::Timeout) => {
149 if status
150 .compare_exchange(
151 READ_PROJECTION_PENDING,
152 READ_PROJECTION_CANCELLED,
153 Ordering::AcqRel,
154 Ordering::Acquire,
155 )
156 .is_ok()
157 {
158 return Err(ReadProjectionError::BusyBeforeStart);
159 }
160 if status.load(Ordering::Acquire) != READ_PROJECTION_GRANTED {
161 return Err(ReadProjectionError::BusyBeforeStart);
162 }
163 granted
164 .recv()
165 .map_err(|_| ReadProjectionError::QueuePoisoned)?;
166 Ok(ReadProjectionLease {
167 queue: self,
168 ticket,
169 })
170 }
171 Err(mpsc::RecvTimeoutError::Disconnected) => Err(ReadProjectionError::BusyBeforeStart),
172 }
173 }
174
175 fn lock_state_until(
176 &self,
177 deadline: Instant,
178 ) -> Result<MutexGuard<'_, ReadProjectionQueueState>, ReadProjectionError> {
179 let mut first_attempt = true;
180 loop {
181 if !first_attempt && Instant::now() >= deadline {
182 return Err(ReadProjectionError::BusyBeforeStart);
183 }
184 match self.state.try_lock() {
185 Ok(state) => return Ok(state),
186 Err(TryLockError::Poisoned(_)) => {
187 return Err(ReadProjectionError::QueuePoisoned);
188 }
189 Err(TryLockError::WouldBlock) => {
190 first_attempt = false;
191 let now = Instant::now();
192 if now >= deadline {
193 return Err(ReadProjectionError::BusyBeforeStart);
194 }
195 thread::park_timeout(
196 deadline
197 .saturating_duration_since(now)
198 .min(Duration::from_millis(1)),
199 );
200 }
201 }
202 }
203 }
204}
205
206fn advance_read_projection_queue(state: &mut ReadProjectionQueueState) {
207 debug_assert!(state.waiters.len() <= MAX_PENDING_READ_PROJECTIONS);
208 while let Some(waiter) = state.waiters.pop_front() {
209 state.serving_ticket = waiter.ticket;
210 if Instant::now() >= waiter.deadline {
211 let _ = waiter.status.compare_exchange(
212 READ_PROJECTION_PENDING,
213 READ_PROJECTION_CANCELLED,
214 Ordering::AcqRel,
215 Ordering::Acquire,
216 );
217 state.serving_ticket = waiter.ticket.saturating_add(1);
218 continue;
219 }
220 if waiter
221 .status
222 .compare_exchange(
223 READ_PROJECTION_PENDING,
224 READ_PROJECTION_GRANTED,
225 Ordering::AcqRel,
226 Ordering::Acquire,
227 )
228 .is_err()
229 {
230 state.serving_ticket = waiter.ticket.saturating_add(1);
231 continue;
232 }
233 if waiter.permit.send(()).is_ok() {
234 state.active = true;
235 return;
236 }
237 waiter
238 .status
239 .store(READ_PROJECTION_CANCELLED, Ordering::Release);
240 state.serving_ticket = waiter.ticket.saturating_add(1);
241 }
242}
243
244#[derive(Clone, PartialEq, Eq)]
246pub struct ConnectionTarget {
247 pub endpoint_url: String,
249 pub module_name: String,
251 pub auth_token: Option<String>,
253}
254
255impl ConnectionTarget {
256 #[must_use]
258 pub fn new(endpoint_url: impl Into<String>, module_name: impl Into<String>) -> Self {
259 Self {
260 endpoint_url: endpoint_url.into(),
261 module_name: module_name.into(),
262 auth_token: None,
263 }
264 }
265
266 #[must_use]
268 pub fn with_auth_token(mut self, auth_token: impl Into<String>) -> Self {
269 self.auth_token = Some(auth_token.into());
270 self
271 }
272}
273
274impl fmt::Debug for ConnectionTarget {
275 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
276 formatter
277 .debug_struct("ConnectionTarget")
278 .field("endpoint_url", &self.endpoint_url)
279 .field("module_name", &self.module_name)
280 .field(
281 "auth_token",
282 &self.auth_token.as_ref().map(|_| "[REDACTED]"),
283 )
284 .finish()
285 }
286}
287
288#[derive(Clone, Default)]
295pub struct SpacetimeDbConnectionStatus {
296 connected: Arc<AtomicBool>,
297}
298
299impl SpacetimeDbConnectionStatus {
300 pub fn mark_connected(&self) {
302 self.connected.store(true, Ordering::Release);
303 }
304
305 pub fn mark_disconnected(&self) {
307 self.connected.store(false, Ordering::Release);
308 }
309
310 #[must_use]
312 pub fn is_connected(&self) -> bool {
313 self.connected.load(Ordering::Acquire)
314 }
315}
316
317pub struct ManagedSpacetimeDbConnection<Connection>
322where
323 Connection: DbContext + Send + 'static,
324{
325 connection: Option<Connection>,
326 worker: Option<thread::JoinHandle<()>>,
327 keepalive: Vec<Box<dyn Send + Sync>>,
328 cleanup_thread_name: String,
329 status: Option<SpacetimeDbConnectionStatus>,
330}
331
332impl<Connection> ManagedSpacetimeDbConnection<Connection>
333where
334 Connection: DbContext + Send + 'static,
335{
336 pub fn new(connection: Connection, worker: thread::JoinHandle<()>) -> Self {
338 Self {
339 connection: Some(connection),
340 worker: Some(worker),
341 keepalive: Vec::new(),
342 cleanup_thread_name: "spacetimedb-cleanup".to_owned(),
343 status: None,
344 }
345 }
346
347 pub fn connection(&self) -> &Connection {
349 self.connection
350 .as_ref()
351 .expect("SpacetimeDB connection is only taken during Drop")
352 }
353
354 pub fn keep_alive(&mut self, resource: impl Send + Sync + 'static) {
356 self.keepalive.push(Box::new(resource));
357 }
358
359 fn set_cleanup_thread_name(&mut self, cleanup_thread_name: String) {
360 self.cleanup_thread_name = cleanup_thread_name;
361 }
362
363 fn set_status(&mut self, status: SpacetimeDbConnectionStatus) {
364 self.status = Some(status);
365 }
366}
367
368impl<Connection> Drop for ManagedSpacetimeDbConnection<Connection>
369where
370 Connection: DbContext + Send + 'static,
371{
372 fn drop(&mut self) {
373 let connection = self.connection.take();
374 let worker = self.worker.take();
375 let keepalive = std::mem::take(&mut self.keepalive);
376 if connection.is_none() && worker.is_none() && keepalive.is_empty() {
377 return;
378 }
379
380 let cleanup_thread_name = self.cleanup_thread_name.clone();
381 let _ = thread::Builder::new()
382 .name(cleanup_thread_name)
383 .spawn(move || {
384 for resource in keepalive {
385 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
386 drop(resource);
387 }));
388 }
389 if let Some(connection) = connection {
390 let connection = ManuallyDrop::new(connection);
391 let disconnected =
392 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
393 let _ = connection.disconnect();
394 }))
395 .is_ok();
396 if disconnected {
397 let connection = ManuallyDrop::into_inner(connection);
398 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
399 drop(connection);
400 }));
401 }
402 }
403 if let Some(worker) = worker {
404 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
405 let _ = worker.join();
406 }));
407 }
408 });
409 }
410}
411
412type ReconnectBuilder<Connection> = dyn Fn(
413 &ConnectionTarget,
414 SpacetimeDbConnectionStatus,
415 ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
416 + Send
417 + Sync;
418
419enum SpacetimeDbClientConnection<Connection>
420where
421 Connection: DbContext + Send + 'static,
422{
423 Fixed(ManagedSpacetimeDbConnection<Connection>),
424 Reconnecting(RwLock<ManagedSpacetimeDbConnection<Connection>>),
425}
426
427pub struct SpacetimeDbClient<Connection>
434where
435 Connection: DbContext + Send + 'static,
436{
437 target: ConnectionTarget,
438 connection_label: String,
439 connection: SpacetimeDbClientConnection<Connection>,
440 reconnect_builder: Option<Arc<ReconnectBuilder<Connection>>>,
441}
442
443pub struct SpacetimeDbClientRegistry<Connection>
445where
446 Connection: DbContext + Send + 'static,
447{
448 clients: Mutex<Vec<(ConnectionTarget, Arc<SpacetimeDbClient<Connection>>)>>,
449}
450
451#[derive(Debug, thiserror::Error, PartialEq, Eq)]
453#[non_exhaustive]
454pub enum SpacetimeDbClientRegistryError {
455 #[error("SpacetimeDB client registry lock was poisoned")]
457 LockPoisoned,
458 #[error("SpacetimeDB client {label} could not connect: {message}")]
460 Connection {
461 label: String,
463 message: String,
465 },
466}
467
468impl<Connection> SpacetimeDbClientRegistry<Connection>
469where
470 Connection: DbContext + Send + 'static,
471{
472 #[must_use]
474 pub const fn new() -> Self {
475 Self {
476 clients: Mutex::new(Vec::new()),
477 }
478 }
479
480 pub fn get_or_connect_reconnecting<Build>(
482 &self,
483 target: ConnectionTarget,
484 connection_label: impl Into<String>,
485 build: Build,
486 ) -> Result<Arc<SpacetimeDbClient<Connection>>, SpacetimeDbClientRegistryError>
487 where
488 Build: Fn(
489 &ConnectionTarget,
490 SpacetimeDbConnectionStatus,
491 ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
492 + Send
493 + Sync
494 + 'static,
495 {
496 let connection_label = connection_label.into();
497 let mut clients = self
498 .clients
499 .lock()
500 .map_err(|_| SpacetimeDbClientRegistryError::LockPoisoned)?;
501 if let Some((_, client)) = clients.iter().find(|(candidate, _)| candidate == &target) {
502 return Ok(Arc::clone(client));
503 }
504
505 let client = Arc::new(
506 SpacetimeDbClient::connect_reconnecting(
507 target.clone(),
508 connection_label.clone(),
509 build,
510 )
511 .map_err(|message| SpacetimeDbClientRegistryError::Connection {
512 label: connection_label,
513 message,
514 })?,
515 );
516 clients.push((target, Arc::clone(&client)));
517 Ok(client)
518 }
519}
520
521impl<Connection> Default for SpacetimeDbClientRegistry<Connection>
522where
523 Connection: DbContext + Send + 'static,
524{
525 fn default() -> Self {
526 Self::new()
527 }
528}
529
530pub struct SpacetimeDbConnectionGuard<'a, Connection>
532where
533 Connection: DbContext + Send + 'static,
534{
535 connection: RwLockReadGuard<'a, ManagedSpacetimeDbConnection<Connection>>,
536}
537
538impl<Connection> Deref for SpacetimeDbConnectionGuard<'_, Connection>
539where
540 Connection: DbContext + Send + 'static,
541{
542 type Target = Connection;
543
544 fn deref(&self) -> &Self::Target {
545 self.connection.connection()
546 }
547}
548
549impl<Connection> SpacetimeDbClient<Connection>
550where
551 Connection: DbContext + Send + 'static,
552{
553 pub fn connect<Build>(
555 target: ConnectionTarget,
556 connection_label: impl Into<String>,
557 build: Build,
558 ) -> Result<Self, String>
559 where
560 Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
561 + Send
562 + 'static,
563 {
564 let connection_label = connection_label.into();
565 let connection =
566 connect_with_retries(target.clone(), connection_label.clone(), build, false)?;
567 Ok(Self {
568 target,
569 connection_label,
570 connection: SpacetimeDbClientConnection::Fixed(connection),
571 reconnect_builder: None,
572 })
573 }
574
575 pub fn connect_optional_read<Build>(
577 target: ConnectionTarget,
578 connection_label: impl Into<String>,
579 build: Build,
580 ) -> Result<Self, String>
581 where
582 Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
583 + Send
584 + 'static,
585 {
586 ensure_optional_read_not_degraded()?;
587 let connection_label = connection_label.into();
588 let connection =
589 connect_with_retries(target.clone(), connection_label.clone(), build, true)?;
590 Ok(Self {
591 target,
592 connection_label,
593 connection: SpacetimeDbClientConnection::Fixed(connection),
594 reconnect_builder: None,
595 })
596 }
597
598 pub fn connect_reconnecting<Build>(
600 target: ConnectionTarget,
601 connection_label: impl Into<String>,
602 build: Build,
603 ) -> Result<Self, String>
604 where
605 Build: Fn(
606 &ConnectionTarget,
607 SpacetimeDbConnectionStatus,
608 ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
609 + Send
610 + Sync
611 + 'static,
612 {
613 let connection_label = connection_label.into();
614 let reconnect_builder: Arc<ReconnectBuilder<Connection>> = Arc::new(build);
615 let connection = connect_reconnecting_with_retries(
616 target.clone(),
617 connection_label.clone(),
618 reconnect_builder.clone(),
619 )?;
620 Ok(Self {
621 target,
622 connection_label,
623 connection: SpacetimeDbClientConnection::Reconnecting(RwLock::new(connection)),
624 reconnect_builder: Some(reconnect_builder),
625 })
626 }
627
628 pub fn connection(&self) -> &Connection {
633 let SpacetimeDbClientConnection::Fixed(connection) = &self.connection else {
634 panic!("reconnecting SpacetimeDB clients use reconnecting_connection")
635 };
636 connection.connection()
637 }
638
639 pub fn reconnecting_connection(
641 &self,
642 ) -> Result<SpacetimeDbConnectionGuard<'_, Connection>, String> {
643 self.ensure_connected()?;
644 let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
645 return Err("fixed SpacetimeDB clients use connection".to_owned());
646 };
647 connection
648 .read()
649 .map(|connection| SpacetimeDbConnectionGuard { connection })
650 .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())
651 }
652
653 #[must_use]
655 pub fn is_connected(&self) -> bool {
656 match &self.connection {
657 SpacetimeDbClientConnection::Fixed(_) => true,
658 SpacetimeDbClientConnection::Reconnecting(connection) => {
659 connection.read().is_ok_and(|connection| {
660 connection
661 .status
662 .as_ref()
663 .is_some_and(SpacetimeDbConnectionStatus::is_connected)
664 })
665 }
666 }
667 }
668
669 pub fn invalidate_reconnecting_connection(&self) -> Result<(), String> {
675 let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
676 return Err("fixed SpacetimeDB clients cannot be invalidated".to_owned());
677 };
678 let connection = connection
679 .read()
680 .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
681 let status = connection
682 .status
683 .as_ref()
684 .ok_or_else(|| "SpacetimeDB reconnecting connection has no status owner".to_owned())?;
685 status.mark_disconnected();
686 Ok(())
687 }
688
689 fn ensure_connected(&self) -> Result<(), String> {
690 let Some(reconnect_builder) = self.reconnect_builder.as_ref() else {
691 return Ok(());
692 };
693 let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
694 return Err("SpacetimeDB reconnect builder has no reconnecting connection".to_owned());
695 };
696 {
697 let connection = connection
698 .read()
699 .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
700 if connection
701 .status
702 .as_ref()
703 .is_some_and(SpacetimeDbConnectionStatus::is_connected)
704 {
705 return Ok(());
706 }
707 }
708
709 let mut connection = connection
710 .write()
711 .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
712 if connection
713 .status
714 .as_ref()
715 .is_some_and(SpacetimeDbConnectionStatus::is_connected)
716 {
717 return Ok(());
718 }
719 *connection = connect_reconnecting_with_retries(
720 self.target.clone(),
721 self.connection_label.clone(),
722 reconnect_builder.clone(),
723 )?;
724 Ok(())
725 }
726}
727
728pub fn with_serialized_read_projection<T>(
735 read_projection: impl FnOnce() -> Result<T, String>,
736) -> Result<T, String> {
737 with_serialized_read_projection_typed(read_projection).map_err(|error| match error {
738 ReadProjectionError::Operation(error) => error,
739 error => error.to_string(),
740 })
741}
742
743pub fn with_serialized_read_projection_typed<T>(
748 read_projection: impl FnOnce() -> Result<T, String>,
749) -> Result<T, ReadProjectionError> {
750 let lock_timeout = READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE
751 .with(|timeout| timeout.get())
752 .unwrap_or(DEFAULT_READ_PROJECTION_LOCK_TIMEOUT);
753 let _lease = subscribe_once_queue().acquire(lock_timeout)?;
754 let result = read_projection();
755 if let Err(error) = &result {
756 record_optional_read_failure(error.as_str());
757 }
758 result.map_err(ReadProjectionError::Operation)
759}
760
761pub fn with_read_projection_lock_timeout<T>(timeout: Duration, operation: impl FnOnce() -> T) -> T {
764 READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE.with(|cell| {
765 let _guard = ReadProjectionLockTimeoutGuard {
766 cell,
767 previous: cell.replace(Some(timeout)),
768 };
769 operation()
770 })
771}
772
773#[must_use]
777pub fn read_projection_timeout_override() -> Option<Duration> {
778 READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE.with(|timeout| timeout.get())
779}
780
781pub fn set_runtime_endpoint_url(endpoint_url: impl Into<String>) {
785 let endpoint_url = endpoint_url.into().trim().to_string();
786 if endpoint_url.is_empty() {
787 return;
788 }
789
790 let _ = RUNTIME_ENDPOINT_URL.set(endpoint_url);
791}
792
793pub fn runtime_endpoint_url() -> Option<String> {
795 RUNTIME_ENDPOINT_URL.get().cloned()
796}
797
798#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)]
800#[non_exhaustive]
801pub enum ScopedKeyError {
802 #[error("scoped key parts must not be empty")]
804 EmptyPart,
805 #[error("scoped key parts must not contain ':'")]
807 ContainsSeparator,
808 #[error("scoped key parts must not contain surrounding whitespace")]
810 SurroundingWhitespace,
811}
812
813pub fn scoped_key(left: &str, right: &str) -> Result<String, ScopedKeyError> {
815 validate_scoped_key_part(left)?;
816 validate_scoped_key_part(right)?;
817 Ok(format!("{left}:{right}"))
818}
819
820pub fn scoped_key3(left: &str, middle: &str, right: &str) -> Result<String, ScopedKeyError> {
822 validate_scoped_key_part(left)?;
823 validate_scoped_key_part(middle)?;
824 validate_scoped_key_part(right)?;
825 Ok(format!("{left}:{middle}:{right}"))
826}
827
828fn validate_scoped_key_part(part: &str) -> Result<(), ScopedKeyError> {
829 if part.trim().is_empty() {
830 return Err(ScopedKeyError::EmptyPart);
831 }
832 if part.trim() != part {
833 return Err(ScopedKeyError::SurroundingWhitespace);
834 }
835 if part.contains(':') {
836 return Err(ScopedKeyError::ContainsSeparator);
837 }
838 Ok(())
839}
840
841fn connect_with_retries<Connection, Build>(
842 target: ConnectionTarget,
843 connection_label: String,
844 mut build: Build,
845 optional_read: bool,
846) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
847where
848 Connection: DbContext + Send + 'static,
849 Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
850 + Send
851 + 'static,
852{
853 connect_managed_with_retries(
854 target,
855 connection_label,
856 move |target| {
857 let (connection, worker) = build(target)?;
858 Ok(ManagedSpacetimeDbConnection::new(connection, worker))
859 },
860 optional_read,
861 )
862}
863
864fn connect_reconnecting_with_retries<Connection>(
865 target: ConnectionTarget,
866 connection_label: String,
867 build: Arc<ReconnectBuilder<Connection>>,
868) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
869where
870 Connection: DbContext + Send + 'static,
871{
872 let readiness_label = connection_label.clone();
873 connect_managed_with_retries(
874 target,
875 connection_label,
876 move |target| {
877 let status = SpacetimeDbConnectionStatus::default();
878 let mut connection = build(target, status.clone())?;
879 if !status.is_connected() {
880 return Err(format!(
881 "SpacetimeDB {readiness_label} connection setup did not mark the connection ready"
882 ));
883 }
884 connection.set_status(status);
885 Ok(connection)
886 },
887 false,
888 )
889}
890
891fn connect_managed_with_retries<Connection, Build>(
892 target: ConnectionTarget,
893 connection_label: String,
894 mut build: Build,
895 optional_read: bool,
896) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
897where
898 Connection: DbContext + Send + 'static,
899 Build: FnMut(&ConnectionTarget) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
900 + Send
901 + 'static,
902{
903 let cleanup_thread_name = cleanup_thread_name(connection_label.as_str());
904 let endpoint_url = target.endpoint_url.clone();
905 let module_name = target.module_name.clone();
906 let thread_endpoint_url = endpoint_url.clone();
907 let thread_module_name = module_name.clone();
908 let thread_label = connection_label.clone();
909 let connection_thread = thread::spawn(move || {
910 let mut last_error = None;
911 for attempt in 0..=CONNECT_RETRY_DELAYS_MS.len() {
912 match build(&target) {
913 Ok(connection) => return Ok(connection),
914 Err(error) => {
915 last_error = Some(error);
916 if let Some(delay_ms) = CONNECT_RETRY_DELAYS_MS.get(attempt) {
917 thread::sleep(Duration::from_millis(*delay_ms));
918 }
919 }
920 }
921 }
922 Err(last_error.unwrap_or_else(|| {
923 format!(
924 "SpacetimeDB {thread_label} connection to {thread_module_name} at {thread_endpoint_url} failed"
925 )
926 }))
927 });
928
929 let mut connection = match connection_thread.join() {
930 Ok(Ok(connection)) => connection,
931 Ok(Err(error)) => {
932 if optional_read {
933 record_optional_read_failure(error.as_str());
934 }
935 return Err(error);
936 }
937 Err(payload) => {
938 let message = panic_message(
939 &payload,
940 format!(
941 "SpacetimeDB {connection_label} connection to {module_name} at {endpoint_url} panicked"
942 ),
943 );
944 if optional_read {
945 record_optional_read_failure(message.as_str());
946 }
947 return Err(message);
948 }
949 };
950 connection.set_cleanup_thread_name(cleanup_thread_name);
951 Ok(connection)
952}
953
954fn cleanup_thread_name(connection_label: &str) -> String {
955 let label = connection_label
956 .chars()
957 .map(|value| {
958 if value.is_ascii_alphanumeric() {
959 value.to_ascii_lowercase()
960 } else {
961 '-'
962 }
963 })
964 .collect::<String>()
965 .trim_matches('-')
966 .to_string();
967 match label.as_str() {
968 "" => "spacetimedb-cleanup".to_owned(),
969 _ => format!("{label}-cleanup"),
970 }
971}
972
973fn panic_message(payload: &Box<dyn std::any::Any + Send>, fallback: String) -> String {
974 if let Some(message) = payload.downcast_ref::<&str>() {
975 format!("{fallback}: {message}")
976 } else if let Some(message) = payload.downcast_ref::<String>() {
977 format!("{fallback}: {message}")
978 } else {
979 fallback
980 }
981}
982
983fn optional_read_degraded_until() -> &'static Mutex<Option<Instant>> {
984 OPTIONAL_READ_DEGRADED_UNTIL.get_or_init(|| Mutex::new(None))
985}
986
987fn subscribe_once_queue() -> &'static ReadProjectionQueue {
988 SUBSCRIBE_ONCE_QUEUE.get_or_init(ReadProjectionQueue::default)
989}
990
991fn ensure_optional_read_not_degraded() -> Result<(), String> {
992 let mut degraded_until = optional_read_degraded_until()
993 .lock()
994 .map_err(|_| "SpacetimeDB optional read circuit breaker was poisoned".to_string())?;
995 let Some(until) = *degraded_until else {
996 return Ok(());
997 };
998 if Instant::now() < until {
999 return Err(
1000 "SpacetimeDB optional read projections are temporarily degraded after a recent live-state failure"
1001 .to_string(),
1002 );
1003 }
1004 *degraded_until = None;
1005 Ok(())
1006}
1007
1008pub fn record_optional_read_failure(error: &str) {
1010 if !is_optional_read_circuit_breaking_error(error) {
1011 return;
1012 }
1013 if let Ok(mut degraded_until) = optional_read_degraded_until().lock() {
1014 *degraded_until = Some(Instant::now() + OPTIONAL_READ_DEGRADE_WINDOW);
1015 }
1016}
1017
1018pub fn is_optional_read_degrading_error(error: &str) -> bool {
1020 let error = error.to_ascii_lowercase();
1021 [
1022 "connection refused",
1023 "subscription did not apply",
1024 "subscription failed before read projection",
1025 "read projection is busy",
1026 "live-state connection",
1027 "panicked",
1028 "buffer should be unique",
1029 ]
1030 .into_iter()
1031 .any(|needle| error.contains(needle))
1032}
1033
1034fn is_optional_read_circuit_breaking_error(error: &str) -> bool {
1035 let error = error.to_ascii_lowercase();
1036 [
1037 "connection refused",
1038 "live-state connection",
1039 "panicked",
1040 "buffer should be unique",
1041 ]
1042 .into_iter()
1043 .any(|needle| error.contains(needle))
1044}
1045
1046#[cfg(test)]
1047mod tests {
1048 use super::{
1049 ConnectionTarget, MAX_PENDING_READ_PROJECTIONS, READ_PROJECTION_CANCELLED,
1050 ReadProjectionError, ReadProjectionQueue, ReadProjectionWaiter,
1051 SpacetimeDbConnectionStatus, is_optional_read_circuit_breaking_error,
1052 is_optional_read_degrading_error, runtime_endpoint_url, scoped_key, scoped_key3,
1053 set_runtime_endpoint_url, with_serialized_read_projection,
1054 with_serialized_read_projection_typed,
1055 };
1056 use std::sync::atomic::AtomicU8;
1057 use std::sync::{Arc, Mutex, mpsc};
1058 use std::thread;
1059 use std::time::{Duration, Instant};
1060
1061 #[test]
1062 fn connection_target_debug_output_redacts_auth_token() {
1063 let target = ConnectionTarget::new("https://database.example.invalid", "module")
1064 .with_auth_token("visible-auth-token");
1065
1066 let debug = format!("{target:?}");
1067
1068 assert!(!debug.contains("visible-auth-token"));
1069 assert!(debug.contains("[REDACTED]"));
1070 }
1071
1072 #[test]
1073 fn reconnect_status_is_shared_without_relaxed_visibility() {
1074 let status = SpacetimeDbConnectionStatus::default();
1075 let callback_status = status.clone();
1076
1077 assert!(!status.is_connected());
1078 status.mark_connected();
1079 assert!(callback_status.is_connected());
1080 callback_status.mark_disconnected();
1081 assert!(!status.is_connected());
1082 }
1083
1084 #[test]
1085 fn scoped_key_rejects_blank_parts() {
1086 assert!(scoped_key("", "session").is_err());
1087 assert!(scoped_key("owner", " ").is_err());
1088 assert_eq!(
1089 scoped_key("owner", "session").expect("valid key"),
1090 "owner:session"
1091 );
1092 assert!(scoped_key("owner:project", "timeline").is_err());
1093 assert!(scoped_key(" owner", "timeline").is_err());
1094 assert_eq!(
1095 scoped_key3("owner", "project", "timeline").expect("valid key"),
1096 "owner:project:timeline"
1097 );
1098 }
1099
1100 #[test]
1101 fn runtime_endpoint_url_ignores_blank_values() {
1102 set_runtime_endpoint_url(" ");
1103
1104 assert_eq!(runtime_endpoint_url(), None);
1105 }
1106
1107 #[test]
1108 fn optional_read_degrade_detection_matches_live_state_failures() {
1109 assert!(is_optional_read_degrading_error(
1110 "SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
1111 ));
1112 assert!(is_optional_read_degrading_error(
1113 "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
1114 ));
1115 assert!(is_optional_read_degrading_error(
1116 "thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
1117 ));
1118 }
1119
1120 #[test]
1121 fn optional_read_degrade_detection_ignores_domain_errors() {
1122 assert!(!is_optional_read_degrading_error(
1123 "requested record does not exist"
1124 ));
1125 assert!(!is_optional_read_degrading_error(
1126 "stored payload could not be deserialized"
1127 ));
1128 }
1129
1130 #[test]
1131 fn optional_read_circuit_breaker_ignores_subscription_apply_timeouts() {
1132 assert!(!is_optional_read_circuit_breaking_error(
1133 "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
1134 ));
1135 assert!(is_optional_read_degrading_error(
1136 "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
1137 ));
1138 assert!(is_optional_read_circuit_breaking_error(
1139 "SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
1140 ));
1141 assert!(is_optional_read_circuit_breaking_error(
1142 "thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
1143 ));
1144 }
1145
1146 #[test]
1147 fn typed_projection_errors_preserve_operation_failures_without_string_classification() {
1148 let typed_error = with_serialized_read_projection_typed::<()>(|| {
1149 Err("domain projection failed".to_string())
1150 })
1151 .expect_err("typed projection failure");
1152 assert_eq!(
1153 typed_error,
1154 ReadProjectionError::Operation("domain projection failed".to_string())
1155 );
1156
1157 let legacy_error =
1158 with_serialized_read_projection::<()>(|| Err("domain projection failed".to_string()))
1159 .expect_err("legacy projection failure");
1160 assert_eq!(legacy_error, "domain projection failed");
1161 }
1162
1163 #[test]
1164 fn serialized_read_projection_reports_typed_bounded_admission_failure() {
1165 let queue = ReadProjectionQueue::default();
1166 let _lease = queue
1167 .acquire(Duration::from_secs(1))
1168 .expect("first projection lease");
1169 let started_at = Instant::now();
1170
1171 let error = queue
1172 .acquire(Duration::from_millis(25))
1173 .expect_err("contended projection must remain bounded");
1174
1175 assert_eq!(error, ReadProjectionError::BusyBeforeStart);
1176 assert!(error.to_string().contains("read projection is busy"));
1177 assert!(started_at.elapsed() < Duration::from_secs(1));
1178 assert!(is_optional_read_degrading_error(error.to_string().as_str()));
1179 }
1180
1181 #[test]
1182 fn projection_admission_deadline_includes_queue_state_lock_contention() {
1183 let queue = Arc::new(ReadProjectionQueue::default());
1184 let state = queue.state.lock().expect("projection queue state");
1185 let contender_queue = Arc::clone(&queue);
1186 let (result_tx, result_rx) = mpsc::channel();
1187 let contender = thread::spawn(move || {
1188 result_tx
1189 .send(
1190 contender_queue
1191 .acquire(Duration::from_millis(25))
1192 .map(|_| ()),
1193 )
1194 .expect("contender result receiver");
1195 });
1196
1197 let started_at = Instant::now();
1198 assert_eq!(
1199 result_rx
1200 .recv_timeout(Duration::from_millis(100))
1201 .expect("queue-state contention must honor the admission deadline"),
1202 Err(ReadProjectionError::BusyBeforeStart)
1203 );
1204 assert!(started_at.elapsed() < Duration::from_millis(100));
1205 assert_eq!(state.next_ticket, 0);
1206 drop(state);
1207 contender.join().expect("projection contender");
1208 }
1209
1210 #[test]
1211 fn projection_waiter_is_not_admitted_after_its_deadline() {
1212 let queue = Arc::new(ReadProjectionQueue::default());
1213 let first_lease = queue
1214 .acquire(Duration::from_secs(1))
1215 .expect("first projection lease");
1216
1217 let waiter_queue = Arc::clone(&queue);
1218 let (waiter_result_tx, waiter_result_rx) = mpsc::channel();
1219 let waiter = thread::spawn(move || {
1220 waiter_result_tx
1221 .send(waiter_queue.acquire(Duration::from_millis(25)).map(|_| ()))
1222 .expect("waiter result receiver");
1223 });
1224 wait_for_projection_ticket_count(queue.as_ref(), 2);
1225
1226 let state = queue.state.lock().expect("projection queue state");
1227 let started_at = Instant::now();
1228 assert_eq!(
1229 waiter_result_rx
1230 .recv_timeout(Duration::from_millis(100))
1231 .expect("waiter must honor its deadline without reacquiring queue state"),
1232 Err(ReadProjectionError::BusyBeforeStart)
1233 );
1234 assert!(started_at.elapsed() < Duration::from_millis(100));
1235 drop(state);
1236 waiter.join().expect("projection waiter");
1237 drop(first_lease);
1238 }
1239
1240 #[test]
1241 fn cancelled_projection_backlog_is_bounded_and_released_in_one_pass() {
1242 let queue = ReadProjectionQueue::default();
1243 let first_lease = queue
1244 .acquire(Duration::from_secs(1))
1245 .expect("first projection lease");
1246 let mut cancelled_receivers = Vec::with_capacity(MAX_PENDING_READ_PROJECTIONS);
1247
1248 {
1249 let mut state = queue.state.lock().expect("projection queue state");
1250 for _ in 0..MAX_PENDING_READ_PROJECTIONS {
1251 let ticket = state.next_ticket;
1252 state.next_ticket = state.next_ticket.checked_add(1).expect("test ticket space");
1253 let (permit, receiver) = mpsc::channel();
1254 cancelled_receivers.push(receiver);
1255 state.waiters.push_back(ReadProjectionWaiter {
1256 ticket,
1257 deadline: Instant::now() + Duration::from_secs(1),
1258 status: Arc::new(AtomicU8::new(READ_PROJECTION_CANCELLED)),
1259 permit,
1260 });
1261 }
1262 }
1263
1264 assert_eq!(
1265 queue
1266 .acquire(Duration::from_millis(25))
1267 .expect_err("bounded backlog must reject another waiter"),
1268 ReadProjectionError::BusyBeforeStart
1269 );
1270
1271 let started_at = Instant::now();
1272 drop(first_lease);
1273 assert!(started_at.elapsed() < Duration::from_secs(1));
1274
1275 let state = queue.state.lock().expect("projection queue state");
1276 assert!(state.waiters.is_empty());
1277 assert!(!state.active);
1278 drop(cancelled_receivers);
1279 }
1280
1281 #[test]
1282 fn serialized_read_projection_serves_waiters_in_fifo_order() {
1283 let queue = Arc::new(ReadProjectionQueue::default());
1284 let first_lease = queue
1285 .acquire(Duration::from_secs(1))
1286 .expect("first projection lease");
1287 let order = Arc::new(Mutex::new(Vec::new()));
1288
1289 let second_queue = Arc::clone(&queue);
1290 let second_order = Arc::clone(&order);
1291 let second = thread::spawn(move || {
1292 let _lease = second_queue
1293 .acquire(Duration::from_secs(1))
1294 .expect("second projection lease");
1295 second_order.lock().expect("second order").push(2);
1296 });
1297 wait_for_projection_ticket_count(queue.as_ref(), 2);
1298
1299 let third_queue = Arc::clone(&queue);
1300 let third_order = Arc::clone(&order);
1301 let third = thread::spawn(move || {
1302 let _lease = third_queue
1303 .acquire(Duration::from_secs(1))
1304 .expect("third projection lease");
1305 third_order.lock().expect("third order").push(3);
1306 });
1307 wait_for_projection_ticket_count(queue.as_ref(), 3);
1308
1309 drop(first_lease);
1310 second.join().expect("second projection worker");
1311 third.join().expect("third projection worker");
1312
1313 assert_eq!(order.lock().expect("projection order").as_slice(), [2, 3]);
1314 }
1315
1316 #[test]
1317 fn cancelled_projection_ticket_does_not_block_the_next_waiter() {
1318 let queue = Arc::new(ReadProjectionQueue::default());
1319 let first_lease = queue
1320 .acquire(Duration::from_secs(1))
1321 .expect("first projection lease");
1322
1323 let cancelled_queue = Arc::clone(&queue);
1324 let cancelled = thread::spawn(move || {
1325 cancelled_queue
1326 .acquire(Duration::from_millis(25))
1327 .map(|_| ())
1328 });
1329 assert_eq!(
1330 cancelled.join().expect("cancelled projection worker"),
1331 Err(ReadProjectionError::BusyBeforeStart)
1332 );
1333
1334 let next_queue = Arc::clone(&queue);
1335 let next = thread::spawn(move || next_queue.acquire(Duration::from_secs(1)).is_ok());
1336 wait_for_projection_ticket_count(queue.as_ref(), 3);
1337 drop(first_lease);
1338
1339 assert!(next.join().expect("next projection worker"));
1340 }
1341
1342 #[test]
1343 fn panicking_projection_releases_its_fifo_lease() {
1344 let queue = ReadProjectionQueue::default();
1345
1346 let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1347 let _lease = queue
1348 .acquire(Duration::from_secs(1))
1349 .expect("panicking projection lease");
1350 panic!("projection panic fixture");
1351 }));
1352
1353 assert!(panic.is_err());
1354 assert!(queue.acquire(Duration::from_millis(25)).is_ok());
1355 }
1356
1357 fn wait_for_projection_ticket_count(queue: &ReadProjectionQueue, expected: u64) {
1358 let deadline = Instant::now() + Duration::from_secs(1);
1359 loop {
1360 if queue
1361 .state
1362 .lock()
1363 .expect("projection queue state")
1364 .next_ticket
1365 >= expected
1366 {
1367 return;
1368 }
1369 assert!(
1370 Instant::now() < deadline,
1371 "projection waiter did not enter the FIFO queue"
1372 );
1373 thread::yield_now();
1374 }
1375 }
1376}