1use std::fmt;
7use std::mem::ManuallyDrop;
8use std::ops::Deref;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::sync::{Arc, Mutex, OnceLock, RwLock, RwLockReadGuard};
11use std::thread;
12use std::time::{Duration, Instant};
13
14use spacetimedb_sdk::DbContext;
15
16static RUNTIME_ENDPOINT_URL: OnceLock<String> = OnceLock::new();
17static OPTIONAL_READ_DEGRADED_UNTIL: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
18static SUBSCRIBE_ONCE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
19
20const CONNECT_RETRY_DELAYS_MS: [u64; 2] = [150, 500];
21const OPTIONAL_READ_DEGRADE_WINDOW: Duration = Duration::from_secs(15);
22
23#[derive(Clone, PartialEq, Eq)]
25pub struct ConnectionTarget {
26 pub endpoint_url: String,
28 pub module_name: String,
30 pub auth_token: Option<String>,
32}
33
34impl ConnectionTarget {
35 #[must_use]
37 pub fn new(endpoint_url: impl Into<String>, module_name: impl Into<String>) -> Self {
38 Self {
39 endpoint_url: endpoint_url.into(),
40 module_name: module_name.into(),
41 auth_token: None,
42 }
43 }
44
45 #[must_use]
47 pub fn with_auth_token(mut self, auth_token: impl Into<String>) -> Self {
48 self.auth_token = Some(auth_token.into());
49 self
50 }
51}
52
53impl fmt::Debug for ConnectionTarget {
54 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
55 formatter
56 .debug_struct("ConnectionTarget")
57 .field("endpoint_url", &self.endpoint_url)
58 .field("module_name", &self.module_name)
59 .field(
60 "auth_token",
61 &self.auth_token.as_ref().map(|_| "[REDACTED]"),
62 )
63 .finish()
64 }
65}
66
67#[derive(Clone, Default)]
74pub struct SpacetimeDbConnectionStatus {
75 connected: Arc<AtomicBool>,
76}
77
78impl SpacetimeDbConnectionStatus {
79 pub fn mark_connected(&self) {
81 self.connected.store(true, Ordering::Release);
82 }
83
84 pub fn mark_disconnected(&self) {
86 self.connected.store(false, Ordering::Release);
87 }
88
89 #[must_use]
91 pub fn is_connected(&self) -> bool {
92 self.connected.load(Ordering::Acquire)
93 }
94}
95
96pub struct ManagedSpacetimeDbConnection<Connection>
101where
102 Connection: DbContext + Send + 'static,
103{
104 connection: Option<Connection>,
105 worker: Option<thread::JoinHandle<()>>,
106 keepalive: Vec<Box<dyn Send + Sync>>,
107 cleanup_thread_name: String,
108 status: Option<SpacetimeDbConnectionStatus>,
109}
110
111impl<Connection> ManagedSpacetimeDbConnection<Connection>
112where
113 Connection: DbContext + Send + 'static,
114{
115 pub fn new(connection: Connection, worker: thread::JoinHandle<()>) -> Self {
117 Self {
118 connection: Some(connection),
119 worker: Some(worker),
120 keepalive: Vec::new(),
121 cleanup_thread_name: "spacetimedb-cleanup".to_owned(),
122 status: None,
123 }
124 }
125
126 pub fn connection(&self) -> &Connection {
128 self.connection
129 .as_ref()
130 .expect("SpacetimeDB connection is only taken during Drop")
131 }
132
133 pub fn keep_alive(&mut self, resource: impl Send + Sync + 'static) {
135 self.keepalive.push(Box::new(resource));
136 }
137
138 fn set_cleanup_thread_name(&mut self, cleanup_thread_name: String) {
139 self.cleanup_thread_name = cleanup_thread_name;
140 }
141
142 fn set_status(&mut self, status: SpacetimeDbConnectionStatus) {
143 self.status = Some(status);
144 }
145}
146
147impl<Connection> Drop for ManagedSpacetimeDbConnection<Connection>
148where
149 Connection: DbContext + Send + 'static,
150{
151 fn drop(&mut self) {
152 let connection = self.connection.take();
153 let worker = self.worker.take();
154 let keepalive = std::mem::take(&mut self.keepalive);
155 if connection.is_none() && worker.is_none() && keepalive.is_empty() {
156 return;
157 }
158
159 let cleanup_thread_name = self.cleanup_thread_name.clone();
160 let _ = thread::Builder::new()
161 .name(cleanup_thread_name)
162 .spawn(move || {
163 drop(keepalive);
164 if let Some(connection) = connection {
165 let connection = ManuallyDrop::new(connection);
166 let disconnected =
167 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
168 let _ = connection.disconnect();
169 }))
170 .is_ok();
171 if disconnected {
172 let connection = ManuallyDrop::into_inner(connection);
173 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
174 drop(connection);
175 }));
176 }
177 }
178 if let Some(worker) = worker {
179 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
180 let _ = worker.join();
181 }));
182 }
183 });
184 }
185}
186
187type ReconnectBuilder<Connection> = dyn Fn(
188 &ConnectionTarget,
189 SpacetimeDbConnectionStatus,
190 ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
191 + Send
192 + Sync;
193
194enum SpacetimeDbClientConnection<Connection>
195where
196 Connection: DbContext + Send + 'static,
197{
198 Fixed(ManagedSpacetimeDbConnection<Connection>),
199 Reconnecting(RwLock<ManagedSpacetimeDbConnection<Connection>>),
200}
201
202pub struct SpacetimeDbClient<Connection>
209where
210 Connection: DbContext + Send + 'static,
211{
212 target: ConnectionTarget,
213 connection_label: String,
214 connection: SpacetimeDbClientConnection<Connection>,
215 reconnect_builder: Option<Arc<ReconnectBuilder<Connection>>>,
216}
217
218pub struct SpacetimeDbConnectionGuard<'a, Connection>
220where
221 Connection: DbContext + Send + 'static,
222{
223 connection: RwLockReadGuard<'a, ManagedSpacetimeDbConnection<Connection>>,
224}
225
226impl<Connection> Deref for SpacetimeDbConnectionGuard<'_, Connection>
227where
228 Connection: DbContext + Send + 'static,
229{
230 type Target = Connection;
231
232 fn deref(&self) -> &Self::Target {
233 self.connection.connection()
234 }
235}
236
237impl<Connection> SpacetimeDbClient<Connection>
238where
239 Connection: DbContext + Send + 'static,
240{
241 pub fn connect<Build>(
243 target: ConnectionTarget,
244 connection_label: impl Into<String>,
245 build: Build,
246 ) -> Result<Self, String>
247 where
248 Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
249 + Send
250 + 'static,
251 {
252 let connection_label = connection_label.into();
253 let connection =
254 connect_with_retries(target.clone(), connection_label.clone(), build, false)?;
255 Ok(Self {
256 target,
257 connection_label,
258 connection: SpacetimeDbClientConnection::Fixed(connection),
259 reconnect_builder: None,
260 })
261 }
262
263 pub fn connect_optional_read<Build>(
265 target: ConnectionTarget,
266 connection_label: impl Into<String>,
267 build: Build,
268 ) -> Result<Self, String>
269 where
270 Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
271 + Send
272 + 'static,
273 {
274 ensure_optional_read_not_degraded()?;
275 let connection_label = connection_label.into();
276 let connection =
277 connect_with_retries(target.clone(), connection_label.clone(), build, true)?;
278 Ok(Self {
279 target,
280 connection_label,
281 connection: SpacetimeDbClientConnection::Fixed(connection),
282 reconnect_builder: None,
283 })
284 }
285
286 pub fn connect_reconnecting<Build>(
288 target: ConnectionTarget,
289 connection_label: impl Into<String>,
290 build: Build,
291 ) -> Result<Self, String>
292 where
293 Build: Fn(
294 &ConnectionTarget,
295 SpacetimeDbConnectionStatus,
296 ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
297 + Send
298 + Sync
299 + 'static,
300 {
301 let connection_label = connection_label.into();
302 let reconnect_builder: Arc<ReconnectBuilder<Connection>> = Arc::new(build);
303 let connection = connect_reconnecting_with_retries(
304 target.clone(),
305 connection_label.clone(),
306 reconnect_builder.clone(),
307 )?;
308 Ok(Self {
309 target,
310 connection_label,
311 connection: SpacetimeDbClientConnection::Reconnecting(RwLock::new(connection)),
312 reconnect_builder: Some(reconnect_builder),
313 })
314 }
315
316 pub fn connection(&self) -> &Connection {
321 let SpacetimeDbClientConnection::Fixed(connection) = &self.connection else {
322 panic!("reconnecting SpacetimeDB clients use reconnecting_connection")
323 };
324 connection.connection()
325 }
326
327 pub fn reconnecting_connection(
329 &self,
330 ) -> Result<SpacetimeDbConnectionGuard<'_, Connection>, String> {
331 self.ensure_connected()?;
332 let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
333 return Err("fixed SpacetimeDB clients use connection".to_owned());
334 };
335 connection
336 .read()
337 .map(|connection| SpacetimeDbConnectionGuard { connection })
338 .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())
339 }
340
341 #[must_use]
343 pub fn is_connected(&self) -> bool {
344 match &self.connection {
345 SpacetimeDbClientConnection::Fixed(_) => true,
346 SpacetimeDbClientConnection::Reconnecting(connection) => {
347 connection.read().is_ok_and(|connection| {
348 connection
349 .status
350 .as_ref()
351 .is_some_and(SpacetimeDbConnectionStatus::is_connected)
352 })
353 }
354 }
355 }
356
357 fn ensure_connected(&self) -> Result<(), String> {
358 let Some(reconnect_builder) = self.reconnect_builder.as_ref() else {
359 return Ok(());
360 };
361 let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
362 return Err("SpacetimeDB reconnect builder has no reconnecting connection".to_owned());
363 };
364 {
365 let connection = connection
366 .read()
367 .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
368 if connection
369 .status
370 .as_ref()
371 .is_some_and(SpacetimeDbConnectionStatus::is_connected)
372 {
373 return Ok(());
374 }
375 }
376
377 let mut connection = connection
378 .write()
379 .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
380 if connection
381 .status
382 .as_ref()
383 .is_some_and(SpacetimeDbConnectionStatus::is_connected)
384 {
385 return Ok(());
386 }
387 *connection = connect_reconnecting_with_retries(
388 self.target.clone(),
389 self.connection_label.clone(),
390 reconnect_builder.clone(),
391 )?;
392 Ok(())
393 }
394}
395
396pub fn with_serialized_read_projection<T>(
398 read_projection: impl FnOnce() -> Result<T, String>,
399) -> Result<T, String> {
400 let _guard = subscribe_once_lock()
401 .lock()
402 .map_err(|_| "SpacetimeDB subscribe-once read lock was poisoned".to_string())?;
403 let result = read_projection();
404 if let Err(error) = &result {
405 record_optional_read_failure(error.as_str());
406 }
407 result
408}
409
410pub fn set_runtime_endpoint_url(endpoint_url: impl Into<String>) {
414 let endpoint_url = endpoint_url.into().trim().to_string();
415 if endpoint_url.is_empty() {
416 return;
417 }
418
419 let _ = RUNTIME_ENDPOINT_URL.set(endpoint_url);
420}
421
422pub fn runtime_endpoint_url() -> Option<String> {
424 RUNTIME_ENDPOINT_URL.get().cloned()
425}
426
427#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)]
429#[non_exhaustive]
430pub enum ScopedKeyError {
431 #[error("scoped key parts must not be empty")]
433 EmptyPart,
434 #[error("scoped key parts must not contain ':'")]
436 ContainsSeparator,
437 #[error("scoped key parts must not contain surrounding whitespace")]
439 SurroundingWhitespace,
440}
441
442pub fn scoped_key(left: &str, right: &str) -> Result<String, ScopedKeyError> {
444 validate_scoped_key_part(left)?;
445 validate_scoped_key_part(right)?;
446 Ok(format!("{left}:{right}"))
447}
448
449pub fn scoped_key3(left: &str, middle: &str, right: &str) -> Result<String, ScopedKeyError> {
451 validate_scoped_key_part(left)?;
452 validate_scoped_key_part(middle)?;
453 validate_scoped_key_part(right)?;
454 Ok(format!("{left}:{middle}:{right}"))
455}
456
457fn validate_scoped_key_part(part: &str) -> Result<(), ScopedKeyError> {
458 if part.trim().is_empty() {
459 return Err(ScopedKeyError::EmptyPart);
460 }
461 if part.trim() != part {
462 return Err(ScopedKeyError::SurroundingWhitespace);
463 }
464 if part.contains(':') {
465 return Err(ScopedKeyError::ContainsSeparator);
466 }
467 Ok(())
468}
469
470fn connect_with_retries<Connection, Build>(
471 target: ConnectionTarget,
472 connection_label: String,
473 mut build: Build,
474 optional_read: bool,
475) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
476where
477 Connection: DbContext + Send + 'static,
478 Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
479 + Send
480 + 'static,
481{
482 connect_managed_with_retries(
483 target,
484 connection_label,
485 move |target| {
486 let (connection, worker) = build(target)?;
487 Ok(ManagedSpacetimeDbConnection::new(connection, worker))
488 },
489 optional_read,
490 )
491}
492
493fn connect_reconnecting_with_retries<Connection>(
494 target: ConnectionTarget,
495 connection_label: String,
496 build: Arc<ReconnectBuilder<Connection>>,
497) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
498where
499 Connection: DbContext + Send + 'static,
500{
501 let readiness_label = connection_label.clone();
502 connect_managed_with_retries(
503 target,
504 connection_label,
505 move |target| {
506 let status = SpacetimeDbConnectionStatus::default();
507 let mut connection = build(target, status.clone())?;
508 if !status.is_connected() {
509 return Err(format!(
510 "SpacetimeDB {readiness_label} connection setup did not mark the connection ready"
511 ));
512 }
513 connection.set_status(status);
514 Ok(connection)
515 },
516 false,
517 )
518}
519
520fn connect_managed_with_retries<Connection, Build>(
521 target: ConnectionTarget,
522 connection_label: String,
523 mut build: Build,
524 optional_read: bool,
525) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
526where
527 Connection: DbContext + Send + 'static,
528 Build: FnMut(&ConnectionTarget) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
529 + Send
530 + 'static,
531{
532 let cleanup_thread_name = cleanup_thread_name(connection_label.as_str());
533 let endpoint_url = target.endpoint_url.clone();
534 let module_name = target.module_name.clone();
535 let thread_endpoint_url = endpoint_url.clone();
536 let thread_module_name = module_name.clone();
537 let thread_label = connection_label.clone();
538 let connection_thread = thread::spawn(move || {
539 let mut last_error = None;
540 for attempt in 0..=CONNECT_RETRY_DELAYS_MS.len() {
541 match build(&target) {
542 Ok(connection) => return Ok(connection),
543 Err(error) => {
544 last_error = Some(error);
545 if let Some(delay_ms) = CONNECT_RETRY_DELAYS_MS.get(attempt) {
546 thread::sleep(Duration::from_millis(*delay_ms));
547 }
548 }
549 }
550 }
551 Err(last_error.unwrap_or_else(|| {
552 format!(
553 "SpacetimeDB {thread_label} connection to {thread_module_name} at {thread_endpoint_url} failed"
554 )
555 }))
556 });
557
558 let mut connection = match connection_thread.join() {
559 Ok(Ok(connection)) => connection,
560 Ok(Err(error)) => {
561 if optional_read {
562 record_optional_read_failure(error.as_str());
563 }
564 return Err(error);
565 }
566 Err(payload) => {
567 let message = panic_message(
568 &payload,
569 format!(
570 "SpacetimeDB {connection_label} connection to {module_name} at {endpoint_url} panicked"
571 ),
572 );
573 if optional_read {
574 record_optional_read_failure(message.as_str());
575 }
576 return Err(message);
577 }
578 };
579 connection.set_cleanup_thread_name(cleanup_thread_name);
580 Ok(connection)
581}
582
583fn cleanup_thread_name(connection_label: &str) -> String {
584 let label = connection_label
585 .chars()
586 .map(|value| {
587 if value.is_ascii_alphanumeric() {
588 value.to_ascii_lowercase()
589 } else {
590 '-'
591 }
592 })
593 .collect::<String>()
594 .trim_matches('-')
595 .to_string();
596 match label.as_str() {
597 "" => "spacetimedb-cleanup".to_owned(),
598 _ => format!("{label}-cleanup"),
599 }
600}
601
602fn panic_message(payload: &Box<dyn std::any::Any + Send>, fallback: String) -> String {
603 if let Some(message) = payload.downcast_ref::<&str>() {
604 format!("{fallback}: {message}")
605 } else if let Some(message) = payload.downcast_ref::<String>() {
606 format!("{fallback}: {message}")
607 } else {
608 fallback
609 }
610}
611
612fn optional_read_degraded_until() -> &'static Mutex<Option<Instant>> {
613 OPTIONAL_READ_DEGRADED_UNTIL.get_or_init(|| Mutex::new(None))
614}
615
616fn subscribe_once_lock() -> &'static Mutex<()> {
617 SUBSCRIBE_ONCE_LOCK.get_or_init(|| Mutex::new(()))
618}
619
620fn ensure_optional_read_not_degraded() -> Result<(), String> {
621 let mut degraded_until = optional_read_degraded_until()
622 .lock()
623 .map_err(|_| "SpacetimeDB optional read circuit breaker was poisoned".to_string())?;
624 let Some(until) = *degraded_until else {
625 return Ok(());
626 };
627 if Instant::now() < until {
628 return Err(
629 "SpacetimeDB optional read projections are temporarily degraded after a recent live-state failure"
630 .to_string(),
631 );
632 }
633 *degraded_until = None;
634 Ok(())
635}
636
637pub fn record_optional_read_failure(error: &str) {
639 if !is_optional_read_circuit_breaking_error(error) {
640 return;
641 }
642 if let Ok(mut degraded_until) = optional_read_degraded_until().lock() {
643 *degraded_until = Some(Instant::now() + OPTIONAL_READ_DEGRADE_WINDOW);
644 }
645}
646
647pub fn is_optional_read_degrading_error(error: &str) -> bool {
649 let error = error.to_ascii_lowercase();
650 [
651 "connection refused",
652 "subscription did not apply",
653 "subscription failed before read projection",
654 "live-state connection",
655 "panicked",
656 "buffer should be unique",
657 ]
658 .into_iter()
659 .any(|needle| error.contains(needle))
660}
661
662fn is_optional_read_circuit_breaking_error(error: &str) -> bool {
663 let error = error.to_ascii_lowercase();
664 [
665 "connection refused",
666 "live-state connection",
667 "panicked",
668 "buffer should be unique",
669 ]
670 .into_iter()
671 .any(|needle| error.contains(needle))
672}
673
674#[cfg(test)]
675mod tests {
676 use super::{
677 ConnectionTarget, SpacetimeDbConnectionStatus, is_optional_read_circuit_breaking_error,
678 is_optional_read_degrading_error, runtime_endpoint_url, scoped_key, scoped_key3,
679 set_runtime_endpoint_url,
680 };
681
682 #[test]
683 fn connection_target_debug_output_redacts_auth_token() {
684 let target = ConnectionTarget::new("https://database.example.invalid", "module")
685 .with_auth_token("visible-auth-token");
686
687 let debug = format!("{target:?}");
688
689 assert!(!debug.contains("visible-auth-token"));
690 assert!(debug.contains("[REDACTED]"));
691 }
692
693 #[test]
694 fn reconnect_status_is_shared_without_relaxed_visibility() {
695 let status = SpacetimeDbConnectionStatus::default();
696 let callback_status = status.clone();
697
698 assert!(!status.is_connected());
699 status.mark_connected();
700 assert!(callback_status.is_connected());
701 callback_status.mark_disconnected();
702 assert!(!status.is_connected());
703 }
704
705 #[test]
706 fn scoped_key_rejects_blank_parts() {
707 assert!(scoped_key("", "session").is_err());
708 assert!(scoped_key("owner", " ").is_err());
709 assert_eq!(
710 scoped_key("owner", "session").expect("valid key"),
711 "owner:session"
712 );
713 assert!(scoped_key("owner:project", "timeline").is_err());
714 assert!(scoped_key(" owner", "timeline").is_err());
715 assert_eq!(
716 scoped_key3("owner", "project", "timeline").expect("valid key"),
717 "owner:project:timeline"
718 );
719 }
720
721 #[test]
722 fn runtime_endpoint_url_ignores_blank_values() {
723 set_runtime_endpoint_url(" ");
724
725 assert_eq!(runtime_endpoint_url(), None);
726 }
727
728 #[test]
729 fn optional_read_degrade_detection_matches_live_state_failures() {
730 assert!(is_optional_read_degrading_error(
731 "SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
732 ));
733 assert!(is_optional_read_degrading_error(
734 "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
735 ));
736 assert!(is_optional_read_degrading_error(
737 "thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
738 ));
739 }
740
741 #[test]
742 fn optional_read_degrade_detection_ignores_domain_errors() {
743 assert!(!is_optional_read_degrading_error(
744 "requested record does not exist"
745 ));
746 assert!(!is_optional_read_degrading_error(
747 "stored payload could not be deserialized"
748 ));
749 }
750
751 #[test]
752 fn optional_read_circuit_breaker_ignores_subscription_apply_timeouts() {
753 assert!(!is_optional_read_circuit_breaking_error(
754 "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
755 ));
756 assert!(is_optional_read_degrading_error(
757 "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
758 ));
759 assert!(is_optional_read_circuit_breaking_error(
760 "SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
761 ));
762 assert!(is_optional_read_circuit_breaking_error(
763 "thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
764 ));
765 }
766}