use std::fmt;
use std::mem::ManuallyDrop;
use std::ops::Deref;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, OnceLock, RwLock, RwLockReadGuard};
use std::thread;
use std::time::{Duration, Instant};
use spacetimedb_sdk::DbContext;
static RUNTIME_ENDPOINT_URL: OnceLock<String> = OnceLock::new();
static OPTIONAL_READ_DEGRADED_UNTIL: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
static SUBSCRIBE_ONCE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
const CONNECT_RETRY_DELAYS_MS: [u64; 2] = [150, 500];
const OPTIONAL_READ_DEGRADE_WINDOW: Duration = Duration::from_secs(15);
#[derive(Clone, PartialEq, Eq)]
pub struct ConnectionTarget {
pub endpoint_url: String,
pub module_name: String,
pub auth_token: Option<String>,
}
impl ConnectionTarget {
#[must_use]
pub fn new(endpoint_url: impl Into<String>, module_name: impl Into<String>) -> Self {
Self {
endpoint_url: endpoint_url.into(),
module_name: module_name.into(),
auth_token: None,
}
}
#[must_use]
pub fn with_auth_token(mut self, auth_token: impl Into<String>) -> Self {
self.auth_token = Some(auth_token.into());
self
}
}
impl fmt::Debug for ConnectionTarget {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ConnectionTarget")
.field("endpoint_url", &self.endpoint_url)
.field("module_name", &self.module_name)
.field(
"auth_token",
&self.auth_token.as_ref().map(|_| "[REDACTED]"),
)
.finish()
}
}
#[derive(Clone, Default)]
pub struct SpacetimeDbConnectionStatus {
connected: Arc<AtomicBool>,
}
impl SpacetimeDbConnectionStatus {
pub fn mark_connected(&self) {
self.connected.store(true, Ordering::Release);
}
pub fn mark_disconnected(&self) {
self.connected.store(false, Ordering::Release);
}
#[must_use]
pub fn is_connected(&self) -> bool {
self.connected.load(Ordering::Acquire)
}
}
pub struct ManagedSpacetimeDbConnection<Connection>
where
Connection: DbContext + Send + 'static,
{
connection: Option<Connection>,
worker: Option<thread::JoinHandle<()>>,
keepalive: Vec<Box<dyn Send + Sync>>,
cleanup_thread_name: String,
status: Option<SpacetimeDbConnectionStatus>,
}
impl<Connection> ManagedSpacetimeDbConnection<Connection>
where
Connection: DbContext + Send + 'static,
{
pub fn new(connection: Connection, worker: thread::JoinHandle<()>) -> Self {
Self {
connection: Some(connection),
worker: Some(worker),
keepalive: Vec::new(),
cleanup_thread_name: "spacetimedb-cleanup".to_owned(),
status: None,
}
}
pub fn connection(&self) -> &Connection {
self.connection
.as_ref()
.expect("SpacetimeDB connection is only taken during Drop")
}
pub fn keep_alive(&mut self, resource: impl Send + Sync + 'static) {
self.keepalive.push(Box::new(resource));
}
fn set_cleanup_thread_name(&mut self, cleanup_thread_name: String) {
self.cleanup_thread_name = cleanup_thread_name;
}
fn set_status(&mut self, status: SpacetimeDbConnectionStatus) {
self.status = Some(status);
}
}
impl<Connection> Drop for ManagedSpacetimeDbConnection<Connection>
where
Connection: DbContext + Send + 'static,
{
fn drop(&mut self) {
let connection = self.connection.take();
let worker = self.worker.take();
let keepalive = std::mem::take(&mut self.keepalive);
if connection.is_none() && worker.is_none() && keepalive.is_empty() {
return;
}
let cleanup_thread_name = self.cleanup_thread_name.clone();
let _ = thread::Builder::new()
.name(cleanup_thread_name)
.spawn(move || {
drop(keepalive);
if let Some(connection) = connection {
let connection = ManuallyDrop::new(connection);
let disconnected =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _ = connection.disconnect();
}))
.is_ok();
if disconnected {
let connection = ManuallyDrop::into_inner(connection);
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
drop(connection);
}));
}
}
if let Some(worker) = worker {
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _ = worker.join();
}));
}
});
}
}
type ReconnectBuilder<Connection> = dyn Fn(
&ConnectionTarget,
SpacetimeDbConnectionStatus,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
+ Send
+ Sync;
enum SpacetimeDbClientConnection<Connection>
where
Connection: DbContext + Send + 'static,
{
Fixed(ManagedSpacetimeDbConnection<Connection>),
Reconnecting(RwLock<ManagedSpacetimeDbConnection<Connection>>),
}
pub struct SpacetimeDbClient<Connection>
where
Connection: DbContext + Send + 'static,
{
target: ConnectionTarget,
connection_label: String,
connection: SpacetimeDbClientConnection<Connection>,
reconnect_builder: Option<Arc<ReconnectBuilder<Connection>>>,
}
pub struct SpacetimeDbConnectionGuard<'a, Connection>
where
Connection: DbContext + Send + 'static,
{
connection: RwLockReadGuard<'a, ManagedSpacetimeDbConnection<Connection>>,
}
impl<Connection> Deref for SpacetimeDbConnectionGuard<'_, Connection>
where
Connection: DbContext + Send + 'static,
{
type Target = Connection;
fn deref(&self) -> &Self::Target {
self.connection.connection()
}
}
impl<Connection> SpacetimeDbClient<Connection>
where
Connection: DbContext + Send + 'static,
{
pub fn connect<Build>(
target: ConnectionTarget,
connection_label: impl Into<String>,
build: Build,
) -> Result<Self, String>
where
Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
+ Send
+ 'static,
{
let connection_label = connection_label.into();
let connection =
connect_with_retries(target.clone(), connection_label.clone(), build, false)?;
Ok(Self {
target,
connection_label,
connection: SpacetimeDbClientConnection::Fixed(connection),
reconnect_builder: None,
})
}
pub fn connect_optional_read<Build>(
target: ConnectionTarget,
connection_label: impl Into<String>,
build: Build,
) -> Result<Self, String>
where
Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
+ Send
+ 'static,
{
ensure_optional_read_not_degraded()?;
let connection_label = connection_label.into();
let connection =
connect_with_retries(target.clone(), connection_label.clone(), build, true)?;
Ok(Self {
target,
connection_label,
connection: SpacetimeDbClientConnection::Fixed(connection),
reconnect_builder: None,
})
}
pub fn connect_reconnecting<Build>(
target: ConnectionTarget,
connection_label: impl Into<String>,
build: Build,
) -> Result<Self, String>
where
Build: Fn(
&ConnectionTarget,
SpacetimeDbConnectionStatus,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
+ Send
+ Sync
+ 'static,
{
let connection_label = connection_label.into();
let reconnect_builder: Arc<ReconnectBuilder<Connection>> = Arc::new(build);
let connection = connect_reconnecting_with_retries(
target.clone(),
connection_label.clone(),
reconnect_builder.clone(),
)?;
Ok(Self {
target,
connection_label,
connection: SpacetimeDbClientConnection::Reconnecting(RwLock::new(connection)),
reconnect_builder: Some(reconnect_builder),
})
}
pub fn connection(&self) -> &Connection {
let SpacetimeDbClientConnection::Fixed(connection) = &self.connection else {
panic!("reconnecting SpacetimeDB clients use reconnecting_connection")
};
connection.connection()
}
pub fn reconnecting_connection(
&self,
) -> Result<SpacetimeDbConnectionGuard<'_, Connection>, String> {
self.ensure_connected()?;
let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
return Err("fixed SpacetimeDB clients use connection".to_owned());
};
connection
.read()
.map(|connection| SpacetimeDbConnectionGuard { connection })
.map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())
}
#[must_use]
pub fn is_connected(&self) -> bool {
match &self.connection {
SpacetimeDbClientConnection::Fixed(_) => true,
SpacetimeDbClientConnection::Reconnecting(connection) => {
connection.read().is_ok_and(|connection| {
connection
.status
.as_ref()
.is_some_and(SpacetimeDbConnectionStatus::is_connected)
})
}
}
}
fn ensure_connected(&self) -> Result<(), String> {
let Some(reconnect_builder) = self.reconnect_builder.as_ref() else {
return Ok(());
};
let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
return Err("SpacetimeDB reconnect builder has no reconnecting connection".to_owned());
};
{
let connection = connection
.read()
.map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
if connection
.status
.as_ref()
.is_some_and(SpacetimeDbConnectionStatus::is_connected)
{
return Ok(());
}
}
let mut connection = connection
.write()
.map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
if connection
.status
.as_ref()
.is_some_and(SpacetimeDbConnectionStatus::is_connected)
{
return Ok(());
}
*connection = connect_reconnecting_with_retries(
self.target.clone(),
self.connection_label.clone(),
reconnect_builder.clone(),
)?;
Ok(())
}
}
pub fn with_serialized_read_projection<T>(
read_projection: impl FnOnce() -> Result<T, String>,
) -> Result<T, String> {
let _guard = subscribe_once_lock()
.lock()
.map_err(|_| "SpacetimeDB subscribe-once read lock was poisoned".to_string())?;
let result = read_projection();
if let Err(error) = &result {
record_optional_read_failure(error.as_str());
}
result
}
pub fn set_runtime_endpoint_url(endpoint_url: impl Into<String>) {
let endpoint_url = endpoint_url.into().trim().to_string();
if endpoint_url.is_empty() {
return;
}
let _ = RUNTIME_ENDPOINT_URL.set(endpoint_url);
}
pub fn runtime_endpoint_url() -> Option<String> {
RUNTIME_ENDPOINT_URL.get().cloned()
}
#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)]
#[non_exhaustive]
pub enum ScopedKeyError {
#[error("scoped key parts must not be empty")]
EmptyPart,
#[error("scoped key parts must not contain ':'")]
ContainsSeparator,
#[error("scoped key parts must not contain surrounding whitespace")]
SurroundingWhitespace,
}
pub fn scoped_key(left: &str, right: &str) -> Result<String, ScopedKeyError> {
validate_scoped_key_part(left)?;
validate_scoped_key_part(right)?;
Ok(format!("{left}:{right}"))
}
pub fn scoped_key3(left: &str, middle: &str, right: &str) -> Result<String, ScopedKeyError> {
validate_scoped_key_part(left)?;
validate_scoped_key_part(middle)?;
validate_scoped_key_part(right)?;
Ok(format!("{left}:{middle}:{right}"))
}
fn validate_scoped_key_part(part: &str) -> Result<(), ScopedKeyError> {
if part.trim().is_empty() {
return Err(ScopedKeyError::EmptyPart);
}
if part.trim() != part {
return Err(ScopedKeyError::SurroundingWhitespace);
}
if part.contains(':') {
return Err(ScopedKeyError::ContainsSeparator);
}
Ok(())
}
fn connect_with_retries<Connection, Build>(
target: ConnectionTarget,
connection_label: String,
mut build: Build,
optional_read: bool,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
where
Connection: DbContext + Send + 'static,
Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
+ Send
+ 'static,
{
connect_managed_with_retries(
target,
connection_label,
move |target| {
let (connection, worker) = build(target)?;
Ok(ManagedSpacetimeDbConnection::new(connection, worker))
},
optional_read,
)
}
fn connect_reconnecting_with_retries<Connection>(
target: ConnectionTarget,
connection_label: String,
build: Arc<ReconnectBuilder<Connection>>,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
where
Connection: DbContext + Send + 'static,
{
let readiness_label = connection_label.clone();
connect_managed_with_retries(
target,
connection_label,
move |target| {
let status = SpacetimeDbConnectionStatus::default();
let mut connection = build(target, status.clone())?;
if !status.is_connected() {
return Err(format!(
"SpacetimeDB {readiness_label} connection setup did not mark the connection ready"
));
}
connection.set_status(status);
Ok(connection)
},
false,
)
}
fn connect_managed_with_retries<Connection, Build>(
target: ConnectionTarget,
connection_label: String,
mut build: Build,
optional_read: bool,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
where
Connection: DbContext + Send + 'static,
Build: FnMut(&ConnectionTarget) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
+ Send
+ 'static,
{
let cleanup_thread_name = cleanup_thread_name(connection_label.as_str());
let endpoint_url = target.endpoint_url.clone();
let module_name = target.module_name.clone();
let thread_endpoint_url = endpoint_url.clone();
let thread_module_name = module_name.clone();
let thread_label = connection_label.clone();
let connection_thread = thread::spawn(move || {
let mut last_error = None;
for attempt in 0..=CONNECT_RETRY_DELAYS_MS.len() {
match build(&target) {
Ok(connection) => return Ok(connection),
Err(error) => {
last_error = Some(error);
if let Some(delay_ms) = CONNECT_RETRY_DELAYS_MS.get(attempt) {
thread::sleep(Duration::from_millis(*delay_ms));
}
}
}
}
Err(last_error.unwrap_or_else(|| {
format!(
"SpacetimeDB {thread_label} connection to {thread_module_name} at {thread_endpoint_url} failed"
)
}))
});
let mut connection = match connection_thread.join() {
Ok(Ok(connection)) => connection,
Ok(Err(error)) => {
if optional_read {
record_optional_read_failure(error.as_str());
}
return Err(error);
}
Err(payload) => {
let message = panic_message(
&payload,
format!(
"SpacetimeDB {connection_label} connection to {module_name} at {endpoint_url} panicked"
),
);
if optional_read {
record_optional_read_failure(message.as_str());
}
return Err(message);
}
};
connection.set_cleanup_thread_name(cleanup_thread_name);
Ok(connection)
}
fn cleanup_thread_name(connection_label: &str) -> String {
let label = connection_label
.chars()
.map(|value| {
if value.is_ascii_alphanumeric() {
value.to_ascii_lowercase()
} else {
'-'
}
})
.collect::<String>()
.trim_matches('-')
.to_string();
match label.as_str() {
"" => "spacetimedb-cleanup".to_owned(),
_ => format!("{label}-cleanup"),
}
}
fn panic_message(payload: &Box<dyn std::any::Any + Send>, fallback: String) -> String {
if let Some(message) = payload.downcast_ref::<&str>() {
format!("{fallback}: {message}")
} else if let Some(message) = payload.downcast_ref::<String>() {
format!("{fallback}: {message}")
} else {
fallback
}
}
fn optional_read_degraded_until() -> &'static Mutex<Option<Instant>> {
OPTIONAL_READ_DEGRADED_UNTIL.get_or_init(|| Mutex::new(None))
}
fn subscribe_once_lock() -> &'static Mutex<()> {
SUBSCRIBE_ONCE_LOCK.get_or_init(|| Mutex::new(()))
}
fn ensure_optional_read_not_degraded() -> Result<(), String> {
let mut degraded_until = optional_read_degraded_until()
.lock()
.map_err(|_| "SpacetimeDB optional read circuit breaker was poisoned".to_string())?;
let Some(until) = *degraded_until else {
return Ok(());
};
if Instant::now() < until {
return Err(
"SpacetimeDB optional read projections are temporarily degraded after a recent live-state failure"
.to_string(),
);
}
*degraded_until = None;
Ok(())
}
pub fn record_optional_read_failure(error: &str) {
if !is_optional_read_circuit_breaking_error(error) {
return;
}
if let Ok(mut degraded_until) = optional_read_degraded_until().lock() {
*degraded_until = Some(Instant::now() + OPTIONAL_READ_DEGRADE_WINDOW);
}
}
pub fn is_optional_read_degrading_error(error: &str) -> bool {
let error = error.to_ascii_lowercase();
[
"connection refused",
"subscription did not apply",
"subscription failed before read projection",
"live-state connection",
"panicked",
"buffer should be unique",
]
.into_iter()
.any(|needle| error.contains(needle))
}
fn is_optional_read_circuit_breaking_error(error: &str) -> bool {
let error = error.to_ascii_lowercase();
[
"connection refused",
"live-state connection",
"panicked",
"buffer should be unique",
]
.into_iter()
.any(|needle| error.contains(needle))
}
#[cfg(test)]
mod tests {
use super::{
ConnectionTarget, SpacetimeDbConnectionStatus, is_optional_read_circuit_breaking_error,
is_optional_read_degrading_error, runtime_endpoint_url, scoped_key, scoped_key3,
set_runtime_endpoint_url,
};
#[test]
fn connection_target_debug_output_redacts_auth_token() {
let target = ConnectionTarget::new("https://database.example.invalid", "module")
.with_auth_token("visible-auth-token");
let debug = format!("{target:?}");
assert!(!debug.contains("visible-auth-token"));
assert!(debug.contains("[REDACTED]"));
}
#[test]
fn reconnect_status_is_shared_without_relaxed_visibility() {
let status = SpacetimeDbConnectionStatus::default();
let callback_status = status.clone();
assert!(!status.is_connected());
status.mark_connected();
assert!(callback_status.is_connected());
callback_status.mark_disconnected();
assert!(!status.is_connected());
}
#[test]
fn scoped_key_rejects_blank_parts() {
assert!(scoped_key("", "session").is_err());
assert!(scoped_key("owner", " ").is_err());
assert_eq!(
scoped_key("owner", "session").expect("valid key"),
"owner:session"
);
assert!(scoped_key("owner:project", "timeline").is_err());
assert!(scoped_key(" owner", "timeline").is_err());
assert_eq!(
scoped_key3("owner", "project", "timeline").expect("valid key"),
"owner:project:timeline"
);
}
#[test]
fn runtime_endpoint_url_ignores_blank_values() {
set_runtime_endpoint_url(" ");
assert_eq!(runtime_endpoint_url(), None);
}
#[test]
fn optional_read_degrade_detection_matches_live_state_failures() {
assert!(is_optional_read_degrading_error(
"SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
));
assert!(is_optional_read_degrading_error(
"SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
));
assert!(is_optional_read_degrading_error(
"thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
));
}
#[test]
fn optional_read_degrade_detection_ignores_domain_errors() {
assert!(!is_optional_read_degrading_error(
"requested record does not exist"
));
assert!(!is_optional_read_degrading_error(
"stored payload could not be deserialized"
));
}
#[test]
fn optional_read_circuit_breaker_ignores_subscription_apply_timeouts() {
assert!(!is_optional_read_circuit_breaking_error(
"SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
));
assert!(is_optional_read_degrading_error(
"SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
));
assert!(is_optional_read_circuit_breaking_error(
"SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
));
assert!(is_optional_read_circuit_breaking_error(
"thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
));
}
}