use std::cell::Cell;
use std::collections::VecDeque;
use std::fmt;
use std::mem::ManuallyDrop;
use std::ops::Deref;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, OnceLock, RwLock, RwLockReadGuard, TryLockError, mpsc};
use std::thread;
use std::time::{Duration, Instant};
use spacetimedb_sdk::DbContext;
static RUNTIME_ENDPOINT_URL: OnceLock<String> = OnceLock::new();
static OPTIONAL_READ_DEGRADED_UNTIL: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
static SUBSCRIBE_ONCE_QUEUE: OnceLock<ReadProjectionQueue> = OnceLock::new();
thread_local! {
static READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE: Cell<Option<Duration>> = const { Cell::new(None) };
}
const CONNECT_RETRY_DELAYS_MS: [u64; 2] = [150, 500];
const OPTIONAL_READ_DEGRADE_WINDOW: Duration = Duration::from_secs(15);
const DEFAULT_READ_PROJECTION_LOCK_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_PENDING_READ_PROJECTIONS: usize = 256;
const READ_PROJECTION_PENDING: u8 = 0;
const READ_PROJECTION_CANCELLED: u8 = 1;
const READ_PROJECTION_GRANTED: u8 = 2;
#[derive(Debug)]
struct ReadProjectionWaiter {
ticket: u64,
deadline: Instant,
status: Arc<AtomicU8>,
permit: mpsc::Sender<()>,
}
#[derive(Debug, Default)]
struct ReadProjectionQueueState {
next_ticket: u64,
serving_ticket: u64,
active: bool,
waiters: VecDeque<ReadProjectionWaiter>,
}
#[derive(Debug, Default)]
struct ReadProjectionQueue {
state: Mutex<ReadProjectionQueueState>,
}
#[derive(Debug)]
struct ReadProjectionLease<'a> {
queue: &'a ReadProjectionQueue,
ticket: u64,
}
impl Drop for ReadProjectionLease<'_> {
fn drop(&mut self) {
let Ok(mut state) = self.queue.state.lock() else {
return;
};
if !state.active || state.serving_ticket != self.ticket {
return;
}
state.active = false;
state.serving_ticket = self.ticket.saturating_add(1);
advance_read_projection_queue(&mut state);
}
}
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
#[non_exhaustive]
pub enum ReadProjectionError {
#[error(
"SpacetimeDB read projection is busy; FIFO admission timed out before the read started"
)]
BusyBeforeStart,
#[error("SpacetimeDB read projection queue was poisoned")]
QueuePoisoned,
#[error("SpacetimeDB read projection queue exhausted its ticket space")]
TicketSpaceExhausted,
#[error("SpacetimeDB read projection admission timeout exceeded the monotonic clock range")]
AdmissionDeadlineOverflow,
#[error("{0}")]
Operation(String),
}
struct ReadProjectionLockTimeoutGuard<'a> {
cell: &'a Cell<Option<Duration>>,
previous: Option<Duration>,
}
impl Drop for ReadProjectionLockTimeoutGuard<'_> {
fn drop(&mut self) {
self.cell.set(self.previous);
}
}
impl ReadProjectionQueue {
fn acquire(&self, timeout: Duration) -> Result<ReadProjectionLease<'_>, ReadProjectionError> {
let deadline = Instant::now()
.checked_add(timeout)
.ok_or(ReadProjectionError::AdmissionDeadlineOverflow)?;
let mut state = self.lock_state_until(deadline)?;
if state.waiters.len() >= MAX_PENDING_READ_PROJECTIONS {
return Err(ReadProjectionError::BusyBeforeStart);
}
let ticket = state.next_ticket;
state.next_ticket = state
.next_ticket
.checked_add(1)
.ok_or(ReadProjectionError::TicketSpaceExhausted)?;
if !state.active && state.waiters.is_empty() {
state.active = true;
state.serving_ticket = ticket;
return Ok(ReadProjectionLease {
queue: self,
ticket,
});
}
let status = Arc::new(AtomicU8::new(READ_PROJECTION_PENDING));
let (permit, granted) = mpsc::channel();
state.waiters.push_back(ReadProjectionWaiter {
ticket,
deadline,
status: Arc::clone(&status),
permit,
});
drop(state);
let remaining = deadline.saturating_duration_since(Instant::now());
match granted.recv_timeout(remaining) {
Ok(()) => Ok(ReadProjectionLease {
queue: self,
ticket,
}),
Err(mpsc::RecvTimeoutError::Timeout) => {
if status
.compare_exchange(
READ_PROJECTION_PENDING,
READ_PROJECTION_CANCELLED,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
return Err(ReadProjectionError::BusyBeforeStart);
}
if status.load(Ordering::Acquire) != READ_PROJECTION_GRANTED {
return Err(ReadProjectionError::BusyBeforeStart);
}
granted
.recv()
.map_err(|_| ReadProjectionError::QueuePoisoned)?;
Ok(ReadProjectionLease {
queue: self,
ticket,
})
}
Err(mpsc::RecvTimeoutError::Disconnected) => Err(ReadProjectionError::BusyBeforeStart),
}
}
fn lock_state_until(
&self,
deadline: Instant,
) -> Result<MutexGuard<'_, ReadProjectionQueueState>, ReadProjectionError> {
let mut first_attempt = true;
loop {
if !first_attempt && Instant::now() >= deadline {
return Err(ReadProjectionError::BusyBeforeStart);
}
match self.state.try_lock() {
Ok(state) => return Ok(state),
Err(TryLockError::Poisoned(_)) => {
return Err(ReadProjectionError::QueuePoisoned);
}
Err(TryLockError::WouldBlock) => {
first_attempt = false;
let now = Instant::now();
if now >= deadline {
return Err(ReadProjectionError::BusyBeforeStart);
}
thread::park_timeout(
deadline
.saturating_duration_since(now)
.min(Duration::from_millis(1)),
);
}
}
}
}
}
fn advance_read_projection_queue(state: &mut ReadProjectionQueueState) {
debug_assert!(state.waiters.len() <= MAX_PENDING_READ_PROJECTIONS);
while let Some(waiter) = state.waiters.pop_front() {
state.serving_ticket = waiter.ticket;
if Instant::now() >= waiter.deadline {
let _ = waiter.status.compare_exchange(
READ_PROJECTION_PENDING,
READ_PROJECTION_CANCELLED,
Ordering::AcqRel,
Ordering::Acquire,
);
state.serving_ticket = waiter.ticket.saturating_add(1);
continue;
}
if waiter
.status
.compare_exchange(
READ_PROJECTION_PENDING,
READ_PROJECTION_GRANTED,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_err()
{
state.serving_ticket = waiter.ticket.saturating_add(1);
continue;
}
if waiter.permit.send(()).is_ok() {
state.active = true;
return;
}
waiter
.status
.store(READ_PROJECTION_CANCELLED, Ordering::Release);
state.serving_ticket = waiter.ticket.saturating_add(1);
}
}
#[derive(Clone, PartialEq, Eq)]
pub struct ConnectionTarget {
pub endpoint_url: String,
pub module_name: String,
pub auth_token: Option<String>,
}
impl ConnectionTarget {
#[must_use]
pub fn new(endpoint_url: impl Into<String>, module_name: impl Into<String>) -> Self {
Self {
endpoint_url: endpoint_url.into(),
module_name: module_name.into(),
auth_token: None,
}
}
#[must_use]
pub fn with_auth_token(mut self, auth_token: impl Into<String>) -> Self {
self.auth_token = Some(auth_token.into());
self
}
}
impl fmt::Debug for ConnectionTarget {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("ConnectionTarget")
.field("endpoint_url", &self.endpoint_url)
.field("module_name", &self.module_name)
.field(
"auth_token",
&self.auth_token.as_ref().map(|_| "[REDACTED]"),
)
.finish()
}
}
#[derive(Clone, Default)]
pub struct SpacetimeDbConnectionStatus {
connected: Arc<AtomicBool>,
}
impl SpacetimeDbConnectionStatus {
pub fn mark_connected(&self) {
self.connected.store(true, Ordering::Release);
}
pub fn mark_disconnected(&self) {
self.connected.store(false, Ordering::Release);
}
#[must_use]
pub fn is_connected(&self) -> bool {
self.connected.load(Ordering::Acquire)
}
}
pub struct ManagedSpacetimeDbConnection<Connection>
where
Connection: DbContext + Send + 'static,
{
connection: Option<Connection>,
worker: Option<thread::JoinHandle<()>>,
keepalive: Vec<Box<dyn Send + Sync>>,
cleanup_thread_name: String,
status: Option<SpacetimeDbConnectionStatus>,
}
impl<Connection> ManagedSpacetimeDbConnection<Connection>
where
Connection: DbContext + Send + 'static,
{
pub fn new(connection: Connection, worker: thread::JoinHandle<()>) -> Self {
Self {
connection: Some(connection),
worker: Some(worker),
keepalive: Vec::new(),
cleanup_thread_name: "spacetimedb-cleanup".to_owned(),
status: None,
}
}
pub fn connection(&self) -> &Connection {
self.connection
.as_ref()
.expect("SpacetimeDB connection is only taken during Drop")
}
pub fn keep_alive(&mut self, resource: impl Send + Sync + 'static) {
self.keepalive.push(Box::new(resource));
}
fn set_cleanup_thread_name(&mut self, cleanup_thread_name: String) {
self.cleanup_thread_name = cleanup_thread_name;
}
fn set_status(&mut self, status: SpacetimeDbConnectionStatus) {
self.status = Some(status);
}
}
impl<Connection> Drop for ManagedSpacetimeDbConnection<Connection>
where
Connection: DbContext + Send + 'static,
{
fn drop(&mut self) {
let connection = self.connection.take();
let worker = self.worker.take();
let keepalive = std::mem::take(&mut self.keepalive);
if connection.is_none() && worker.is_none() && keepalive.is_empty() {
return;
}
let cleanup_thread_name = self.cleanup_thread_name.clone();
let _ = thread::Builder::new()
.name(cleanup_thread_name)
.spawn(move || {
for resource in keepalive {
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
drop(resource);
}));
}
if let Some(connection) = connection {
let connection = ManuallyDrop::new(connection);
let disconnected =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _ = connection.disconnect();
}))
.is_ok();
if disconnected {
let connection = ManuallyDrop::into_inner(connection);
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
drop(connection);
}));
}
}
if let Some(worker) = worker {
let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _ = worker.join();
}));
}
});
}
}
type ReconnectBuilder<Connection> = dyn Fn(
&ConnectionTarget,
SpacetimeDbConnectionStatus,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
+ Send
+ Sync;
enum SpacetimeDbClientConnection<Connection>
where
Connection: DbContext + Send + 'static,
{
Fixed(ManagedSpacetimeDbConnection<Connection>),
Reconnecting(RwLock<ManagedSpacetimeDbConnection<Connection>>),
}
pub struct SpacetimeDbClient<Connection>
where
Connection: DbContext + Send + 'static,
{
target: ConnectionTarget,
connection_label: String,
connection: SpacetimeDbClientConnection<Connection>,
reconnect_builder: Option<Arc<ReconnectBuilder<Connection>>>,
}
pub struct SpacetimeDbClientRegistry<Connection>
where
Connection: DbContext + Send + 'static,
{
clients: Mutex<Vec<(ConnectionTarget, Arc<SpacetimeDbClient<Connection>>)>>,
}
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
#[non_exhaustive]
pub enum SpacetimeDbClientRegistryError {
#[error("SpacetimeDB client registry lock was poisoned")]
LockPoisoned,
#[error("SpacetimeDB client {label} could not connect: {message}")]
Connection {
label: String,
message: String,
},
}
impl<Connection> SpacetimeDbClientRegistry<Connection>
where
Connection: DbContext + Send + 'static,
{
#[must_use]
pub const fn new() -> Self {
Self {
clients: Mutex::new(Vec::new()),
}
}
pub fn get_or_connect_reconnecting<Build>(
&self,
target: ConnectionTarget,
connection_label: impl Into<String>,
build: Build,
) -> Result<Arc<SpacetimeDbClient<Connection>>, SpacetimeDbClientRegistryError>
where
Build: Fn(
&ConnectionTarget,
SpacetimeDbConnectionStatus,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
+ Send
+ Sync
+ 'static,
{
let connection_label = connection_label.into();
let mut clients = self
.clients
.lock()
.map_err(|_| SpacetimeDbClientRegistryError::LockPoisoned)?;
if let Some((_, client)) = clients.iter().find(|(candidate, _)| candidate == &target) {
return Ok(Arc::clone(client));
}
let client = Arc::new(
SpacetimeDbClient::connect_reconnecting(
target.clone(),
connection_label.clone(),
build,
)
.map_err(|message| SpacetimeDbClientRegistryError::Connection {
label: connection_label,
message,
})?,
);
clients.push((target, Arc::clone(&client)));
Ok(client)
}
}
impl<Connection> Default for SpacetimeDbClientRegistry<Connection>
where
Connection: DbContext + Send + 'static,
{
fn default() -> Self {
Self::new()
}
}
pub struct SpacetimeDbConnectionGuard<'a, Connection>
where
Connection: DbContext + Send + 'static,
{
connection: RwLockReadGuard<'a, ManagedSpacetimeDbConnection<Connection>>,
}
impl<Connection> Deref for SpacetimeDbConnectionGuard<'_, Connection>
where
Connection: DbContext + Send + 'static,
{
type Target = Connection;
fn deref(&self) -> &Self::Target {
self.connection.connection()
}
}
impl<Connection> SpacetimeDbClient<Connection>
where
Connection: DbContext + Send + 'static,
{
pub fn connect<Build>(
target: ConnectionTarget,
connection_label: impl Into<String>,
build: Build,
) -> Result<Self, String>
where
Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
+ Send
+ 'static,
{
let connection_label = connection_label.into();
let connection =
connect_with_retries(target.clone(), connection_label.clone(), build, false)?;
Ok(Self {
target,
connection_label,
connection: SpacetimeDbClientConnection::Fixed(connection),
reconnect_builder: None,
})
}
pub fn connect_optional_read<Build>(
target: ConnectionTarget,
connection_label: impl Into<String>,
build: Build,
) -> Result<Self, String>
where
Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
+ Send
+ 'static,
{
ensure_optional_read_not_degraded()?;
let connection_label = connection_label.into();
let connection =
connect_with_retries(target.clone(), connection_label.clone(), build, true)?;
Ok(Self {
target,
connection_label,
connection: SpacetimeDbClientConnection::Fixed(connection),
reconnect_builder: None,
})
}
pub fn connect_reconnecting<Build>(
target: ConnectionTarget,
connection_label: impl Into<String>,
build: Build,
) -> Result<Self, String>
where
Build: Fn(
&ConnectionTarget,
SpacetimeDbConnectionStatus,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
+ Send
+ Sync
+ 'static,
{
let connection_label = connection_label.into();
let reconnect_builder: Arc<ReconnectBuilder<Connection>> = Arc::new(build);
let connection = connect_reconnecting_with_retries(
target.clone(),
connection_label.clone(),
reconnect_builder.clone(),
)?;
Ok(Self {
target,
connection_label,
connection: SpacetimeDbClientConnection::Reconnecting(RwLock::new(connection)),
reconnect_builder: Some(reconnect_builder),
})
}
pub fn connection(&self) -> &Connection {
let SpacetimeDbClientConnection::Fixed(connection) = &self.connection else {
panic!("reconnecting SpacetimeDB clients use reconnecting_connection")
};
connection.connection()
}
pub fn reconnecting_connection(
&self,
) -> Result<SpacetimeDbConnectionGuard<'_, Connection>, String> {
self.ensure_connected()?;
let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
return Err("fixed SpacetimeDB clients use connection".to_owned());
};
connection
.read()
.map(|connection| SpacetimeDbConnectionGuard { connection })
.map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())
}
#[must_use]
pub fn is_connected(&self) -> bool {
match &self.connection {
SpacetimeDbClientConnection::Fixed(_) => true,
SpacetimeDbClientConnection::Reconnecting(connection) => {
connection.read().is_ok_and(|connection| {
connection
.status
.as_ref()
.is_some_and(SpacetimeDbConnectionStatus::is_connected)
})
}
}
}
pub fn invalidate_reconnecting_connection(&self) -> Result<(), String> {
let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
return Err("fixed SpacetimeDB clients cannot be invalidated".to_owned());
};
let connection = connection
.read()
.map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
let status = connection
.status
.as_ref()
.ok_or_else(|| "SpacetimeDB reconnecting connection has no status owner".to_owned())?;
status.mark_disconnected();
Ok(())
}
fn ensure_connected(&self) -> Result<(), String> {
let Some(reconnect_builder) = self.reconnect_builder.as_ref() else {
return Ok(());
};
let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
return Err("SpacetimeDB reconnect builder has no reconnecting connection".to_owned());
};
{
let connection = connection
.read()
.map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
if connection
.status
.as_ref()
.is_some_and(SpacetimeDbConnectionStatus::is_connected)
{
return Ok(());
}
}
let mut connection = connection
.write()
.map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
if connection
.status
.as_ref()
.is_some_and(SpacetimeDbConnectionStatus::is_connected)
{
return Ok(());
}
*connection = connect_reconnecting_with_retries(
self.target.clone(),
self.connection_label.clone(),
reconnect_builder.clone(),
)?;
Ok(())
}
}
pub fn with_serialized_read_projection<T>(
read_projection: impl FnOnce() -> Result<T, String>,
) -> Result<T, String> {
with_serialized_read_projection_typed(read_projection).map_err(|error| match error {
ReadProjectionError::Operation(error) => error,
error => error.to_string(),
})
}
pub fn with_serialized_read_projection_typed<T>(
read_projection: impl FnOnce() -> Result<T, String>,
) -> Result<T, ReadProjectionError> {
let lock_timeout = READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE
.with(|timeout| timeout.get())
.unwrap_or(DEFAULT_READ_PROJECTION_LOCK_TIMEOUT);
let _lease = subscribe_once_queue().acquire(lock_timeout)?;
let result = read_projection();
if let Err(error) = &result {
record_optional_read_failure(error.as_str());
}
result.map_err(ReadProjectionError::Operation)
}
pub fn with_read_projection_lock_timeout<T>(timeout: Duration, operation: impl FnOnce() -> T) -> T {
READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE.with(|cell| {
let _guard = ReadProjectionLockTimeoutGuard {
cell,
previous: cell.replace(Some(timeout)),
};
operation()
})
}
#[must_use]
pub fn read_projection_timeout_override() -> Option<Duration> {
READ_PROJECTION_LOCK_TIMEOUT_OVERRIDE.with(|timeout| timeout.get())
}
pub fn set_runtime_endpoint_url(endpoint_url: impl Into<String>) {
let endpoint_url = endpoint_url.into().trim().to_string();
if endpoint_url.is_empty() {
return;
}
let _ = RUNTIME_ENDPOINT_URL.set(endpoint_url);
}
pub fn runtime_endpoint_url() -> Option<String> {
RUNTIME_ENDPOINT_URL.get().cloned()
}
#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)]
#[non_exhaustive]
pub enum ScopedKeyError {
#[error("scoped key parts must not be empty")]
EmptyPart,
#[error("scoped key parts must not contain ':'")]
ContainsSeparator,
#[error("scoped key parts must not contain surrounding whitespace")]
SurroundingWhitespace,
}
pub fn scoped_key(left: &str, right: &str) -> Result<String, ScopedKeyError> {
validate_scoped_key_part(left)?;
validate_scoped_key_part(right)?;
Ok(format!("{left}:{right}"))
}
pub fn scoped_key3(left: &str, middle: &str, right: &str) -> Result<String, ScopedKeyError> {
validate_scoped_key_part(left)?;
validate_scoped_key_part(middle)?;
validate_scoped_key_part(right)?;
Ok(format!("{left}:{middle}:{right}"))
}
fn validate_scoped_key_part(part: &str) -> Result<(), ScopedKeyError> {
if part.trim().is_empty() {
return Err(ScopedKeyError::EmptyPart);
}
if part.trim() != part {
return Err(ScopedKeyError::SurroundingWhitespace);
}
if part.contains(':') {
return Err(ScopedKeyError::ContainsSeparator);
}
Ok(())
}
fn connect_with_retries<Connection, Build>(
target: ConnectionTarget,
connection_label: String,
mut build: Build,
optional_read: bool,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
where
Connection: DbContext + Send + 'static,
Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
+ Send
+ 'static,
{
connect_managed_with_retries(
target,
connection_label,
move |target| {
let (connection, worker) = build(target)?;
Ok(ManagedSpacetimeDbConnection::new(connection, worker))
},
optional_read,
)
}
fn connect_reconnecting_with_retries<Connection>(
target: ConnectionTarget,
connection_label: String,
build: Arc<ReconnectBuilder<Connection>>,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
where
Connection: DbContext + Send + 'static,
{
let readiness_label = connection_label.clone();
connect_managed_with_retries(
target,
connection_label,
move |target| {
let status = SpacetimeDbConnectionStatus::default();
let mut connection = build(target, status.clone())?;
if !status.is_connected() {
return Err(format!(
"SpacetimeDB {readiness_label} connection setup did not mark the connection ready"
));
}
connection.set_status(status);
Ok(connection)
},
false,
)
}
fn connect_managed_with_retries<Connection, Build>(
target: ConnectionTarget,
connection_label: String,
mut build: Build,
optional_read: bool,
) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
where
Connection: DbContext + Send + 'static,
Build: FnMut(&ConnectionTarget) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
+ Send
+ 'static,
{
let cleanup_thread_name = cleanup_thread_name(connection_label.as_str());
let endpoint_url = target.endpoint_url.clone();
let module_name = target.module_name.clone();
let thread_endpoint_url = endpoint_url.clone();
let thread_module_name = module_name.clone();
let thread_label = connection_label.clone();
let connection_thread = thread::spawn(move || {
let mut last_error = None;
for attempt in 0..=CONNECT_RETRY_DELAYS_MS.len() {
match build(&target) {
Ok(connection) => return Ok(connection),
Err(error) => {
last_error = Some(error);
if let Some(delay_ms) = CONNECT_RETRY_DELAYS_MS.get(attempt) {
thread::sleep(Duration::from_millis(*delay_ms));
}
}
}
}
Err(last_error.unwrap_or_else(|| {
format!(
"SpacetimeDB {thread_label} connection to {thread_module_name} at {thread_endpoint_url} failed"
)
}))
});
let mut connection = match connection_thread.join() {
Ok(Ok(connection)) => connection,
Ok(Err(error)) => {
if optional_read {
record_optional_read_failure(error.as_str());
}
return Err(error);
}
Err(payload) => {
let message = panic_message(
&payload,
format!(
"SpacetimeDB {connection_label} connection to {module_name} at {endpoint_url} panicked"
),
);
if optional_read {
record_optional_read_failure(message.as_str());
}
return Err(message);
}
};
connection.set_cleanup_thread_name(cleanup_thread_name);
Ok(connection)
}
fn cleanup_thread_name(connection_label: &str) -> String {
let label = connection_label
.chars()
.map(|value| {
if value.is_ascii_alphanumeric() {
value.to_ascii_lowercase()
} else {
'-'
}
})
.collect::<String>()
.trim_matches('-')
.to_string();
match label.as_str() {
"" => "spacetimedb-cleanup".to_owned(),
_ => format!("{label}-cleanup"),
}
}
fn panic_message(payload: &Box<dyn std::any::Any + Send>, fallback: String) -> String {
if let Some(message) = payload.downcast_ref::<&str>() {
format!("{fallback}: {message}")
} else if let Some(message) = payload.downcast_ref::<String>() {
format!("{fallback}: {message}")
} else {
fallback
}
}
fn optional_read_degraded_until() -> &'static Mutex<Option<Instant>> {
OPTIONAL_READ_DEGRADED_UNTIL.get_or_init(|| Mutex::new(None))
}
fn subscribe_once_queue() -> &'static ReadProjectionQueue {
SUBSCRIBE_ONCE_QUEUE.get_or_init(ReadProjectionQueue::default)
}
fn ensure_optional_read_not_degraded() -> Result<(), String> {
let mut degraded_until = optional_read_degraded_until()
.lock()
.map_err(|_| "SpacetimeDB optional read circuit breaker was poisoned".to_string())?;
let Some(until) = *degraded_until else {
return Ok(());
};
if Instant::now() < until {
return Err(
"SpacetimeDB optional read projections are temporarily degraded after a recent live-state failure"
.to_string(),
);
}
*degraded_until = None;
Ok(())
}
pub fn record_optional_read_failure(error: &str) {
if !is_optional_read_circuit_breaking_error(error) {
return;
}
if let Ok(mut degraded_until) = optional_read_degraded_until().lock() {
*degraded_until = Some(Instant::now() + OPTIONAL_READ_DEGRADE_WINDOW);
}
}
pub fn is_optional_read_degrading_error(error: &str) -> bool {
let error = error.to_ascii_lowercase();
[
"connection refused",
"subscription did not apply",
"subscription failed before read projection",
"read projection is busy",
"live-state connection",
"panicked",
"buffer should be unique",
]
.into_iter()
.any(|needle| error.contains(needle))
}
fn is_optional_read_circuit_breaking_error(error: &str) -> bool {
let error = error.to_ascii_lowercase();
[
"connection refused",
"live-state connection",
"panicked",
"buffer should be unique",
]
.into_iter()
.any(|needle| error.contains(needle))
}
#[cfg(test)]
mod tests {
use super::{
ConnectionTarget, MAX_PENDING_READ_PROJECTIONS, READ_PROJECTION_CANCELLED,
ReadProjectionError, ReadProjectionQueue, ReadProjectionWaiter,
SpacetimeDbConnectionStatus, is_optional_read_circuit_breaking_error,
is_optional_read_degrading_error, runtime_endpoint_url, scoped_key, scoped_key3,
set_runtime_endpoint_url, with_serialized_read_projection,
with_serialized_read_projection_typed,
};
use std::sync::atomic::AtomicU8;
use std::sync::{Arc, Mutex, mpsc};
use std::thread;
use std::time::{Duration, Instant};
#[test]
fn connection_target_debug_output_redacts_auth_token() {
let target = ConnectionTarget::new("https://database.example.invalid", "module")
.with_auth_token("visible-auth-token");
let debug = format!("{target:?}");
assert!(!debug.contains("visible-auth-token"));
assert!(debug.contains("[REDACTED]"));
}
#[test]
fn reconnect_status_is_shared_without_relaxed_visibility() {
let status = SpacetimeDbConnectionStatus::default();
let callback_status = status.clone();
assert!(!status.is_connected());
status.mark_connected();
assert!(callback_status.is_connected());
callback_status.mark_disconnected();
assert!(!status.is_connected());
}
#[test]
fn scoped_key_rejects_blank_parts() {
assert!(scoped_key("", "session").is_err());
assert!(scoped_key("owner", " ").is_err());
assert_eq!(
scoped_key("owner", "session").expect("valid key"),
"owner:session"
);
assert!(scoped_key("owner:project", "timeline").is_err());
assert!(scoped_key(" owner", "timeline").is_err());
assert_eq!(
scoped_key3("owner", "project", "timeline").expect("valid key"),
"owner:project:timeline"
);
}
#[test]
fn runtime_endpoint_url_ignores_blank_values() {
set_runtime_endpoint_url(" ");
assert_eq!(runtime_endpoint_url(), None);
}
#[test]
fn optional_read_degrade_detection_matches_live_state_failures() {
assert!(is_optional_read_degrading_error(
"SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
));
assert!(is_optional_read_degrading_error(
"SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
));
assert!(is_optional_read_degrading_error(
"thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
));
}
#[test]
fn optional_read_degrade_detection_ignores_domain_errors() {
assert!(!is_optional_read_degrading_error(
"requested record does not exist"
));
assert!(!is_optional_read_degrading_error(
"stored payload could not be deserialized"
));
}
#[test]
fn optional_read_circuit_breaker_ignores_subscription_apply_timeouts() {
assert!(!is_optional_read_circuit_breaking_error(
"SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
));
assert!(is_optional_read_degrading_error(
"SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
));
assert!(is_optional_read_circuit_breaking_error(
"SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
));
assert!(is_optional_read_circuit_breaking_error(
"thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
));
}
#[test]
fn typed_projection_errors_preserve_operation_failures_without_string_classification() {
let typed_error = with_serialized_read_projection_typed::<()>(|| {
Err("domain projection failed".to_string())
})
.expect_err("typed projection failure");
assert_eq!(
typed_error,
ReadProjectionError::Operation("domain projection failed".to_string())
);
let legacy_error =
with_serialized_read_projection::<()>(|| Err("domain projection failed".to_string()))
.expect_err("legacy projection failure");
assert_eq!(legacy_error, "domain projection failed");
}
#[test]
fn serialized_read_projection_reports_typed_bounded_admission_failure() {
let queue = ReadProjectionQueue::default();
let _lease = queue
.acquire(Duration::from_secs(1))
.expect("first projection lease");
let started_at = Instant::now();
let error = queue
.acquire(Duration::from_millis(25))
.expect_err("contended projection must remain bounded");
assert_eq!(error, ReadProjectionError::BusyBeforeStart);
assert!(error.to_string().contains("read projection is busy"));
assert!(started_at.elapsed() < Duration::from_secs(1));
assert!(is_optional_read_degrading_error(error.to_string().as_str()));
}
#[test]
fn projection_admission_deadline_includes_queue_state_lock_contention() {
let queue = Arc::new(ReadProjectionQueue::default());
let state = queue.state.lock().expect("projection queue state");
let contender_queue = Arc::clone(&queue);
let (result_tx, result_rx) = mpsc::channel();
let contender = thread::spawn(move || {
result_tx
.send(
contender_queue
.acquire(Duration::from_millis(25))
.map(|_| ()),
)
.expect("contender result receiver");
});
let started_at = Instant::now();
assert_eq!(
result_rx
.recv_timeout(Duration::from_millis(100))
.expect("queue-state contention must honor the admission deadline"),
Err(ReadProjectionError::BusyBeforeStart)
);
assert!(started_at.elapsed() < Duration::from_millis(100));
assert_eq!(state.next_ticket, 0);
drop(state);
contender.join().expect("projection contender");
}
#[test]
fn projection_waiter_is_not_admitted_after_its_deadline() {
let queue = Arc::new(ReadProjectionQueue::default());
let first_lease = queue
.acquire(Duration::from_secs(1))
.expect("first projection lease");
let waiter_queue = Arc::clone(&queue);
let (waiter_result_tx, waiter_result_rx) = mpsc::channel();
let waiter = thread::spawn(move || {
waiter_result_tx
.send(waiter_queue.acquire(Duration::from_millis(25)).map(|_| ()))
.expect("waiter result receiver");
});
wait_for_projection_ticket_count(queue.as_ref(), 2);
let state = queue.state.lock().expect("projection queue state");
let started_at = Instant::now();
assert_eq!(
waiter_result_rx
.recv_timeout(Duration::from_millis(100))
.expect("waiter must honor its deadline without reacquiring queue state"),
Err(ReadProjectionError::BusyBeforeStart)
);
assert!(started_at.elapsed() < Duration::from_millis(100));
drop(state);
waiter.join().expect("projection waiter");
drop(first_lease);
}
#[test]
fn cancelled_projection_backlog_is_bounded_and_released_in_one_pass() {
let queue = ReadProjectionQueue::default();
let first_lease = queue
.acquire(Duration::from_secs(1))
.expect("first projection lease");
let mut cancelled_receivers = Vec::with_capacity(MAX_PENDING_READ_PROJECTIONS);
{
let mut state = queue.state.lock().expect("projection queue state");
for _ in 0..MAX_PENDING_READ_PROJECTIONS {
let ticket = state.next_ticket;
state.next_ticket = state.next_ticket.checked_add(1).expect("test ticket space");
let (permit, receiver) = mpsc::channel();
cancelled_receivers.push(receiver);
state.waiters.push_back(ReadProjectionWaiter {
ticket,
deadline: Instant::now() + Duration::from_secs(1),
status: Arc::new(AtomicU8::new(READ_PROJECTION_CANCELLED)),
permit,
});
}
}
assert_eq!(
queue
.acquire(Duration::from_millis(25))
.expect_err("bounded backlog must reject another waiter"),
ReadProjectionError::BusyBeforeStart
);
let started_at = Instant::now();
drop(first_lease);
assert!(started_at.elapsed() < Duration::from_secs(1));
let state = queue.state.lock().expect("projection queue state");
assert!(state.waiters.is_empty());
assert!(!state.active);
drop(cancelled_receivers);
}
#[test]
fn serialized_read_projection_serves_waiters_in_fifo_order() {
let queue = Arc::new(ReadProjectionQueue::default());
let first_lease = queue
.acquire(Duration::from_secs(1))
.expect("first projection lease");
let order = Arc::new(Mutex::new(Vec::new()));
let second_queue = Arc::clone(&queue);
let second_order = Arc::clone(&order);
let second = thread::spawn(move || {
let _lease = second_queue
.acquire(Duration::from_secs(1))
.expect("second projection lease");
second_order.lock().expect("second order").push(2);
});
wait_for_projection_ticket_count(queue.as_ref(), 2);
let third_queue = Arc::clone(&queue);
let third_order = Arc::clone(&order);
let third = thread::spawn(move || {
let _lease = third_queue
.acquire(Duration::from_secs(1))
.expect("third projection lease");
third_order.lock().expect("third order").push(3);
});
wait_for_projection_ticket_count(queue.as_ref(), 3);
drop(first_lease);
second.join().expect("second projection worker");
third.join().expect("third projection worker");
assert_eq!(order.lock().expect("projection order").as_slice(), [2, 3]);
}
#[test]
fn cancelled_projection_ticket_does_not_block_the_next_waiter() {
let queue = Arc::new(ReadProjectionQueue::default());
let first_lease = queue
.acquire(Duration::from_secs(1))
.expect("first projection lease");
let cancelled_queue = Arc::clone(&queue);
let cancelled = thread::spawn(move || {
cancelled_queue
.acquire(Duration::from_millis(25))
.map(|_| ())
});
assert_eq!(
cancelled.join().expect("cancelled projection worker"),
Err(ReadProjectionError::BusyBeforeStart)
);
let next_queue = Arc::clone(&queue);
let next = thread::spawn(move || next_queue.acquire(Duration::from_secs(1)).is_ok());
wait_for_projection_ticket_count(queue.as_ref(), 3);
drop(first_lease);
assert!(next.join().expect("next projection worker"));
}
#[test]
fn panicking_projection_releases_its_fifo_lease() {
let queue = ReadProjectionQueue::default();
let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _lease = queue
.acquire(Duration::from_secs(1))
.expect("panicking projection lease");
panic!("projection panic fixture");
}));
assert!(panic.is_err());
assert!(queue.acquire(Duration::from_millis(25)).is_ok());
}
fn wait_for_projection_ticket_count(queue: &ReadProjectionQueue, expected: u64) {
let deadline = Instant::now() + Duration::from_secs(1);
loop {
if queue
.state
.lock()
.expect("projection queue state")
.next_ticket
>= expected
{
return;
}
assert!(
Instant::now() < deadline,
"projection waiter did not enter the FIFO queue"
);
thread::yield_now();
}
}
}