Skip to main content

ch_database/spacetimedb/
mod.rs

1//! Shared SpacetimeDB helper utilities.
2//!
3//! App-owned SpacetimeDB tables, reducers, generated bindings, and concrete
4//! clients live in the owning app database crates.
5
6use std::fmt;
7use std::mem::ManuallyDrop;
8use std::ops::Deref;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::sync::{Arc, Mutex, OnceLock, RwLock, RwLockReadGuard};
11use std::thread;
12use std::time::{Duration, Instant};
13
14use spacetimedb_sdk::DbContext;
15
16static RUNTIME_ENDPOINT_URL: OnceLock<String> = OnceLock::new();
17static OPTIONAL_READ_DEGRADED_UNTIL: OnceLock<Mutex<Option<Instant>>> = OnceLock::new();
18static SUBSCRIBE_ONCE_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
19
20const CONNECT_RETRY_DELAYS_MS: [u64; 2] = [150, 500];
21const OPTIONAL_READ_DEGRADE_WINDOW: Duration = Duration::from_secs(15);
22
23/// Concrete SpacetimeDB database target shared by app-owned clients.
24#[derive(Clone, PartialEq, Eq)]
25pub struct ConnectionTarget {
26    /// HTTP or WebSocket endpoint used by the generated client.
27    pub endpoint_url: String,
28    /// Deployed SpacetimeDB module name.
29    pub module_name: String,
30    /// Optional bearer token used to authenticate the connection.
31    pub auth_token: Option<String>,
32}
33
34impl ConnectionTarget {
35    /// Creates an unauthenticated connection target.
36    #[must_use]
37    pub fn new(endpoint_url: impl Into<String>, module_name: impl Into<String>) -> Self {
38        Self {
39            endpoint_url: endpoint_url.into(),
40            module_name: module_name.into(),
41            auth_token: None,
42        }
43    }
44
45    /// Attaches a bearer token while keeping it redacted from debug output.
46    #[must_use]
47    pub fn with_auth_token(mut self, auth_token: impl Into<String>) -> Self {
48        self.auth_token = Some(auth_token.into());
49        self
50    }
51}
52
53impl fmt::Debug for ConnectionTarget {
54    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
55        formatter
56            .debug_struct("ConnectionTarget")
57            .field("endpoint_url", &self.endpoint_url)
58            .field("module_name", &self.module_name)
59            .field(
60                "auth_token",
61                &self.auth_token.as_ref().map(|_| "[REDACTED]"),
62            )
63            .finish()
64    }
65}
66
67/// Shared connection state reported by an app-owned generated connection builder.
68///
69/// The app builder wires `mark_disconnected` into the generated SDK's
70/// `on_disconnect` callback and calls `mark_connected` only after its required
71/// subscriptions have been applied. The shared client uses that state to rebuild
72/// the complete connection without duplicating reconnect policy in each app.
73#[derive(Clone, Default)]
74pub struct SpacetimeDbConnectionStatus {
75    connected: Arc<AtomicBool>,
76}
77
78impl SpacetimeDbConnectionStatus {
79    /// Marks the generated connection and its required subscriptions ready.
80    pub fn mark_connected(&self) {
81        self.connected.store(true, Ordering::Release);
82    }
83
84    /// Marks the generated connection unavailable so the next access reconnects.
85    pub fn mark_disconnected(&self) {
86        self.connected.store(false, Ordering::Release);
87    }
88
89    /// Returns whether the complete app-owned connection is ready for use.
90    #[must_use]
91    pub fn is_connected(&self) -> bool {
92        self.connected.load(Ordering::Acquire)
93    }
94}
95
96/// One generated connection, its worker, and resources tied to that connection.
97///
98/// App clients may retain generated subscription handles through `keep_alive`.
99/// All retained resources are released before disconnecting the generated client.
100pub struct ManagedSpacetimeDbConnection<Connection>
101where
102    Connection: DbContext + Send + 'static,
103{
104    connection: Option<Connection>,
105    worker: Option<thread::JoinHandle<()>>,
106    keepalive: Vec<Box<dyn Send + Sync>>,
107    cleanup_thread_name: String,
108    status: Option<SpacetimeDbConnectionStatus>,
109}
110
111impl<Connection> ManagedSpacetimeDbConnection<Connection>
112where
113    Connection: DbContext + Send + 'static,
114{
115    /// Wraps a generated connection and its background worker in the shared owner.
116    pub fn new(connection: Connection, worker: thread::JoinHandle<()>) -> Self {
117        Self {
118            connection: Some(connection),
119            worker: Some(worker),
120            keepalive: Vec::new(),
121            cleanup_thread_name: "spacetimedb-cleanup".to_owned(),
122            status: None,
123        }
124    }
125
126    /// Returns the generated connection while an app builder finishes setup.
127    pub fn connection(&self) -> &Connection {
128        self.connection
129            .as_ref()
130            .expect("SpacetimeDB connection is only taken during Drop")
131    }
132
133    /// Keeps a connection-scoped callback or subscription handle alive.
134    pub fn keep_alive(&mut self, resource: impl Send + Sync + 'static) {
135        self.keepalive.push(Box::new(resource));
136    }
137
138    fn set_cleanup_thread_name(&mut self, cleanup_thread_name: String) {
139        self.cleanup_thread_name = cleanup_thread_name;
140    }
141
142    fn set_status(&mut self, status: SpacetimeDbConnectionStatus) {
143        self.status = Some(status);
144    }
145}
146
147impl<Connection> Drop for ManagedSpacetimeDbConnection<Connection>
148where
149    Connection: DbContext + Send + 'static,
150{
151    fn drop(&mut self) {
152        let connection = self.connection.take();
153        let worker = self.worker.take();
154        let keepalive = std::mem::take(&mut self.keepalive);
155        if connection.is_none() && worker.is_none() && keepalive.is_empty() {
156            return;
157        }
158
159        let cleanup_thread_name = self.cleanup_thread_name.clone();
160        let _ = thread::Builder::new()
161            .name(cleanup_thread_name)
162            .spawn(move || {
163                drop(keepalive);
164                if let Some(connection) = connection {
165                    let connection = ManuallyDrop::new(connection);
166                    let disconnected =
167                        std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
168                            let _ = connection.disconnect();
169                        }))
170                        .is_ok();
171                    if disconnected {
172                        let connection = ManuallyDrop::into_inner(connection);
173                        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
174                            drop(connection);
175                        }));
176                    }
177                }
178                if let Some(worker) = worker {
179                    let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
180                        let _ = worker.join();
181                    }));
182                }
183            });
184    }
185}
186
187type ReconnectBuilder<Connection> = dyn Fn(
188        &ConnectionTarget,
189        SpacetimeDbConnectionStatus,
190    ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
191    + Send
192    + Sync;
193
194enum SpacetimeDbClientConnection<Connection>
195where
196    Connection: DbContext + Send + 'static,
197{
198    Fixed(ManagedSpacetimeDbConnection<Connection>),
199    Reconnecting(RwLock<ManagedSpacetimeDbConnection<Connection>>),
200}
201
202/// Shared lifecycle owner for generated app-specific SpacetimeDB connections.
203///
204/// Apps still own their generated `DbConnection` type, table accessors, reducers,
205/// subscriptions, and row projection logic. This shared owner centralizes retry,
206/// reconnect, worker ownership, optional-read degradation, serial read-projection
207/// throttling, and panic-tolerant cleanup.
208pub struct SpacetimeDbClient<Connection>
209where
210    Connection: DbContext + Send + 'static,
211{
212    target: ConnectionTarget,
213    connection_label: String,
214    connection: SpacetimeDbClientConnection<Connection>,
215    reconnect_builder: Option<Arc<ReconnectBuilder<Connection>>>,
216}
217
218/// Read guard for the generated connection currently owned by the shared client.
219pub struct SpacetimeDbConnectionGuard<'a, Connection>
220where
221    Connection: DbContext + Send + 'static,
222{
223    connection: RwLockReadGuard<'a, ManagedSpacetimeDbConnection<Connection>>,
224}
225
226impl<Connection> Deref for SpacetimeDbConnectionGuard<'_, Connection>
227where
228    Connection: DbContext + Send + 'static,
229{
230    type Target = Connection;
231
232    fn deref(&self) -> &Self::Target {
233        self.connection.connection()
234    }
235}
236
237impl<Connection> SpacetimeDbClient<Connection>
238where
239    Connection: DbContext + Send + 'static,
240{
241    /// Connects a required app-owned client using the shared retry and cleanup policy.
242    pub fn connect<Build>(
243        target: ConnectionTarget,
244        connection_label: impl Into<String>,
245        build: Build,
246    ) -> Result<Self, String>
247    where
248        Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
249            + Send
250            + 'static,
251    {
252        let connection_label = connection_label.into();
253        let connection =
254            connect_with_retries(target.clone(), connection_label.clone(), build, false)?;
255        Ok(Self {
256            target,
257            connection_label,
258            connection: SpacetimeDbClientConnection::Fixed(connection),
259            reconnect_builder: None,
260        })
261    }
262
263    /// Connects an optional read projection unless its shared circuit breaker is degraded.
264    pub fn connect_optional_read<Build>(
265        target: ConnectionTarget,
266        connection_label: impl Into<String>,
267        build: Build,
268    ) -> Result<Self, String>
269    where
270        Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
271            + Send
272            + 'static,
273    {
274        ensure_optional_read_not_degraded()?;
275        let connection_label = connection_label.into();
276        let connection =
277            connect_with_retries(target.clone(), connection_label.clone(), build, true)?;
278        Ok(Self {
279            target,
280            connection_label,
281            connection: SpacetimeDbClientConnection::Fixed(connection),
282            reconnect_builder: None,
283        })
284    }
285
286    /// Connects a required app-owned client and restores it after disconnection.
287    pub fn connect_reconnecting<Build>(
288        target: ConnectionTarget,
289        connection_label: impl Into<String>,
290        build: Build,
291    ) -> Result<Self, String>
292    where
293        Build: Fn(
294                &ConnectionTarget,
295                SpacetimeDbConnectionStatus,
296            ) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
297            + Send
298            + Sync
299            + 'static,
300    {
301        let connection_label = connection_label.into();
302        let reconnect_builder: Arc<ReconnectBuilder<Connection>> = Arc::new(build);
303        let connection = connect_reconnecting_with_retries(
304            target.clone(),
305            connection_label.clone(),
306            reconnect_builder.clone(),
307        )?;
308        Ok(Self {
309            target,
310            connection_label,
311            connection: SpacetimeDbClientConnection::Reconnecting(RwLock::new(connection)),
312            reconnect_builder: Some(reconnect_builder),
313        })
314    }
315
316    /// Returns a fixed generated connection owned by this lifecycle guard.
317    ///
318    /// Reconnecting clients use `reconnecting_connection` so access remains locked
319    /// while a disconnected connection is replaced.
320    pub fn connection(&self) -> &Connection {
321        let SpacetimeDbClientConnection::Fixed(connection) = &self.connection else {
322            panic!("reconnecting SpacetimeDB clients use reconnecting_connection")
323        };
324        connection.connection()
325    }
326
327    /// Returns the current reconnecting connection, rebuilding it when required.
328    pub fn reconnecting_connection(
329        &self,
330    ) -> Result<SpacetimeDbConnectionGuard<'_, Connection>, String> {
331        self.ensure_connected()?;
332        let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
333            return Err("fixed SpacetimeDB clients use connection".to_owned());
334        };
335        connection
336            .read()
337            .map(|connection| SpacetimeDbConnectionGuard { connection })
338            .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())
339    }
340
341    /// Returns the last connection status reported by a reconnecting client.
342    #[must_use]
343    pub fn is_connected(&self) -> bool {
344        match &self.connection {
345            SpacetimeDbClientConnection::Fixed(_) => true,
346            SpacetimeDbClientConnection::Reconnecting(connection) => {
347                connection.read().is_ok_and(|connection| {
348                    connection
349                        .status
350                        .as_ref()
351                        .is_some_and(SpacetimeDbConnectionStatus::is_connected)
352                })
353            }
354        }
355    }
356
357    fn ensure_connected(&self) -> Result<(), String> {
358        let Some(reconnect_builder) = self.reconnect_builder.as_ref() else {
359            return Ok(());
360        };
361        let SpacetimeDbClientConnection::Reconnecting(connection) = &self.connection else {
362            return Err("SpacetimeDB reconnect builder has no reconnecting connection".to_owned());
363        };
364        {
365            let connection = connection
366                .read()
367                .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
368            if connection
369                .status
370                .as_ref()
371                .is_some_and(SpacetimeDbConnectionStatus::is_connected)
372            {
373                return Ok(());
374            }
375        }
376
377        let mut connection = connection
378            .write()
379            .map_err(|_| "SpacetimeDB connection lock was poisoned".to_owned())?;
380        if connection
381            .status
382            .as_ref()
383            .is_some_and(SpacetimeDbConnectionStatus::is_connected)
384        {
385            return Ok(());
386        }
387        *connection = connect_reconnecting_with_retries(
388            self.target.clone(),
389            self.connection_label.clone(),
390            reconnect_builder.clone(),
391        )?;
392        Ok(())
393    }
394}
395
396/// Runs one read projection while holding the shared subscription projection lock.
397pub fn with_serialized_read_projection<T>(
398    read_projection: impl FnOnce() -> Result<T, String>,
399) -> Result<T, String> {
400    let _guard = subscribe_once_lock()
401        .lock()
402        .map_err(|_| "SpacetimeDB subscribe-once read lock was poisoned".to_string())?;
403    let result = read_projection();
404    if let Err(error) = &result {
405        record_optional_read_failure(error.as_str());
406    }
407    result
408}
409
410/// Sets the process-wide SpacetimeDB endpoint discovered by the runtime adapter.
411///
412/// The first non-empty endpoint wins for the lifetime of the process.
413pub fn set_runtime_endpoint_url(endpoint_url: impl Into<String>) {
414    let endpoint_url = endpoint_url.into().trim().to_string();
415    if endpoint_url.is_empty() {
416        return;
417    }
418
419    let _ = RUNTIME_ENDPOINT_URL.set(endpoint_url);
420}
421
422/// Returns the process-wide SpacetimeDB endpoint, when one has been configured.
423pub fn runtime_endpoint_url() -> Option<String> {
424    RUNTIME_ENDPOINT_URL.get().cloned()
425}
426
427/// Invalid component in a scoped storage key.
428#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)]
429#[non_exhaustive]
430pub enum ScopedKeyError {
431    /// A component was empty after trimming.
432    #[error("scoped key parts must not be empty")]
433    EmptyPart,
434    /// A component contains the separator and would make the key ambiguous.
435    #[error("scoped key parts must not contain ':'")]
436    ContainsSeparator,
437    /// Leading or trailing whitespace would change the key silently.
438    #[error("scoped key parts must not contain surrounding whitespace")]
439    SurroundingWhitespace,
440}
441
442/// Builds a validated two-part colon-delimited key.
443pub fn scoped_key(left: &str, right: &str) -> Result<String, ScopedKeyError> {
444    validate_scoped_key_part(left)?;
445    validate_scoped_key_part(right)?;
446    Ok(format!("{left}:{right}"))
447}
448
449/// Builds a validated three-part colon-delimited key.
450pub fn scoped_key3(left: &str, middle: &str, right: &str) -> Result<String, ScopedKeyError> {
451    validate_scoped_key_part(left)?;
452    validate_scoped_key_part(middle)?;
453    validate_scoped_key_part(right)?;
454    Ok(format!("{left}:{middle}:{right}"))
455}
456
457fn validate_scoped_key_part(part: &str) -> Result<(), ScopedKeyError> {
458    if part.trim().is_empty() {
459        return Err(ScopedKeyError::EmptyPart);
460    }
461    if part.trim() != part {
462        return Err(ScopedKeyError::SurroundingWhitespace);
463    }
464    if part.contains(':') {
465        return Err(ScopedKeyError::ContainsSeparator);
466    }
467    Ok(())
468}
469
470fn connect_with_retries<Connection, Build>(
471    target: ConnectionTarget,
472    connection_label: String,
473    mut build: Build,
474    optional_read: bool,
475) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
476where
477    Connection: DbContext + Send + 'static,
478    Build: FnMut(&ConnectionTarget) -> Result<(Connection, thread::JoinHandle<()>), String>
479        + Send
480        + 'static,
481{
482    connect_managed_with_retries(
483        target,
484        connection_label,
485        move |target| {
486            let (connection, worker) = build(target)?;
487            Ok(ManagedSpacetimeDbConnection::new(connection, worker))
488        },
489        optional_read,
490    )
491}
492
493fn connect_reconnecting_with_retries<Connection>(
494    target: ConnectionTarget,
495    connection_label: String,
496    build: Arc<ReconnectBuilder<Connection>>,
497) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
498where
499    Connection: DbContext + Send + 'static,
500{
501    let readiness_label = connection_label.clone();
502    connect_managed_with_retries(
503        target,
504        connection_label,
505        move |target| {
506            let status = SpacetimeDbConnectionStatus::default();
507            let mut connection = build(target, status.clone())?;
508            if !status.is_connected() {
509                return Err(format!(
510                    "SpacetimeDB {readiness_label} connection setup did not mark the connection ready"
511                ));
512            }
513            connection.set_status(status);
514            Ok(connection)
515        },
516        false,
517    )
518}
519
520fn connect_managed_with_retries<Connection, Build>(
521    target: ConnectionTarget,
522    connection_label: String,
523    mut build: Build,
524    optional_read: bool,
525) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
526where
527    Connection: DbContext + Send + 'static,
528    Build: FnMut(&ConnectionTarget) -> Result<ManagedSpacetimeDbConnection<Connection>, String>
529        + Send
530        + 'static,
531{
532    let cleanup_thread_name = cleanup_thread_name(connection_label.as_str());
533    let endpoint_url = target.endpoint_url.clone();
534    let module_name = target.module_name.clone();
535    let thread_endpoint_url = endpoint_url.clone();
536    let thread_module_name = module_name.clone();
537    let thread_label = connection_label.clone();
538    let connection_thread = thread::spawn(move || {
539        let mut last_error = None;
540        for attempt in 0..=CONNECT_RETRY_DELAYS_MS.len() {
541            match build(&target) {
542                Ok(connection) => return Ok(connection),
543                Err(error) => {
544                    last_error = Some(error);
545                    if let Some(delay_ms) = CONNECT_RETRY_DELAYS_MS.get(attempt) {
546                        thread::sleep(Duration::from_millis(*delay_ms));
547                    }
548                }
549            }
550        }
551        Err(last_error.unwrap_or_else(|| {
552            format!(
553                "SpacetimeDB {thread_label} connection to {thread_module_name} at {thread_endpoint_url} failed"
554            )
555        }))
556    });
557
558    let mut connection = match connection_thread.join() {
559        Ok(Ok(connection)) => connection,
560        Ok(Err(error)) => {
561            if optional_read {
562                record_optional_read_failure(error.as_str());
563            }
564            return Err(error);
565        }
566        Err(payload) => {
567            let message = panic_message(
568                &payload,
569                format!(
570                    "SpacetimeDB {connection_label} connection to {module_name} at {endpoint_url} panicked"
571                ),
572            );
573            if optional_read {
574                record_optional_read_failure(message.as_str());
575            }
576            return Err(message);
577        }
578    };
579    connection.set_cleanup_thread_name(cleanup_thread_name);
580    Ok(connection)
581}
582
583fn cleanup_thread_name(connection_label: &str) -> String {
584    let label = connection_label
585        .chars()
586        .map(|value| {
587            if value.is_ascii_alphanumeric() {
588                value.to_ascii_lowercase()
589            } else {
590                '-'
591            }
592        })
593        .collect::<String>()
594        .trim_matches('-')
595        .to_string();
596    match label.as_str() {
597        "" => "spacetimedb-cleanup".to_owned(),
598        _ => format!("{label}-cleanup"),
599    }
600}
601
602fn panic_message(payload: &Box<dyn std::any::Any + Send>, fallback: String) -> String {
603    if let Some(message) = payload.downcast_ref::<&str>() {
604        format!("{fallback}: {message}")
605    } else if let Some(message) = payload.downcast_ref::<String>() {
606        format!("{fallback}: {message}")
607    } else {
608        fallback
609    }
610}
611
612fn optional_read_degraded_until() -> &'static Mutex<Option<Instant>> {
613    OPTIONAL_READ_DEGRADED_UNTIL.get_or_init(|| Mutex::new(None))
614}
615
616fn subscribe_once_lock() -> &'static Mutex<()> {
617    SUBSCRIBE_ONCE_LOCK.get_or_init(|| Mutex::new(()))
618}
619
620fn ensure_optional_read_not_degraded() -> Result<(), String> {
621    let mut degraded_until = optional_read_degraded_until()
622        .lock()
623        .map_err(|_| "SpacetimeDB optional read circuit breaker was poisoned".to_string())?;
624    let Some(until) = *degraded_until else {
625        return Ok(());
626    };
627    if Instant::now() < until {
628        return Err(
629            "SpacetimeDB optional read projections are temporarily degraded after a recent live-state failure"
630                .to_string(),
631        );
632    }
633    *degraded_until = None;
634    Ok(())
635}
636
637/// Records a transport-level optional-read failure in the shared circuit breaker.
638pub fn record_optional_read_failure(error: &str) {
639    if !is_optional_read_circuit_breaking_error(error) {
640        return;
641    }
642    if let Ok(mut degraded_until) = optional_read_degraded_until().lock() {
643        *degraded_until = Some(Instant::now() + OPTIONAL_READ_DEGRADE_WINDOW);
644    }
645}
646
647/// Reports whether an optional-read error should degrade its caller-facing feature.
648pub fn is_optional_read_degrading_error(error: &str) -> bool {
649    let error = error.to_ascii_lowercase();
650    [
651        "connection refused",
652        "subscription did not apply",
653        "subscription failed before read projection",
654        "live-state connection",
655        "panicked",
656        "buffer should be unique",
657    ]
658    .into_iter()
659    .any(|needle| error.contains(needle))
660}
661
662fn is_optional_read_circuit_breaking_error(error: &str) -> bool {
663    let error = error.to_ascii_lowercase();
664    [
665        "connection refused",
666        "live-state connection",
667        "panicked",
668        "buffer should be unique",
669    ]
670    .into_iter()
671    .any(|needle| error.contains(needle))
672}
673
674#[cfg(test)]
675mod tests {
676    use super::{
677        ConnectionTarget, SpacetimeDbConnectionStatus, is_optional_read_circuit_breaking_error,
678        is_optional_read_degrading_error, runtime_endpoint_url, scoped_key, scoped_key3,
679        set_runtime_endpoint_url,
680    };
681
682    #[test]
683    fn connection_target_debug_output_redacts_auth_token() {
684        let target = ConnectionTarget::new("https://database.example.invalid", "module")
685            .with_auth_token("visible-auth-token");
686
687        let debug = format!("{target:?}");
688
689        assert!(!debug.contains("visible-auth-token"));
690        assert!(debug.contains("[REDACTED]"));
691    }
692
693    #[test]
694    fn reconnect_status_is_shared_without_relaxed_visibility() {
695        let status = SpacetimeDbConnectionStatus::default();
696        let callback_status = status.clone();
697
698        assert!(!status.is_connected());
699        status.mark_connected();
700        assert!(callback_status.is_connected());
701        callback_status.mark_disconnected();
702        assert!(!status.is_connected());
703    }
704
705    #[test]
706    fn scoped_key_rejects_blank_parts() {
707        assert!(scoped_key("", "session").is_err());
708        assert!(scoped_key("owner", " ").is_err());
709        assert_eq!(
710            scoped_key("owner", "session").expect("valid key"),
711            "owner:session"
712        );
713        assert!(scoped_key("owner:project", "timeline").is_err());
714        assert!(scoped_key(" owner", "timeline").is_err());
715        assert_eq!(
716            scoped_key3("owner", "project", "timeline").expect("valid key"),
717            "owner:project:timeline"
718        );
719    }
720
721    #[test]
722    fn runtime_endpoint_url_ignores_blank_values() {
723        set_runtime_endpoint_url(" ");
724
725        assert_eq!(runtime_endpoint_url(), None);
726    }
727
728    #[test]
729    fn optional_read_degrade_detection_matches_live_state_failures() {
730        assert!(is_optional_read_degrading_error(
731            "SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
732        ));
733        assert!(is_optional_read_degrading_error(
734            "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
735        ));
736        assert!(is_optional_read_degrading_error(
737            "thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
738        ));
739    }
740
741    #[test]
742    fn optional_read_degrade_detection_ignores_domain_errors() {
743        assert!(!is_optional_read_degrading_error(
744            "requested record does not exist"
745        ));
746        assert!(!is_optional_read_degrading_error(
747            "stored payload could not be deserialized"
748        ));
749    }
750
751    #[test]
752    fn optional_read_circuit_breaker_ignores_subscription_apply_timeouts() {
753        assert!(!is_optional_read_circuit_breaking_error(
754            "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
755        ));
756        assert!(is_optional_read_degrading_error(
757            "SpacetimeDB subscription did not apply before read projection timeout: timed out waiting on channel"
758        ));
759        assert!(is_optional_read_circuit_breaking_error(
760            "SpacetimeDB live-state connection to module at http://127.0.0.1:4531 failed: connection refused"
761        ));
762        assert!(is_optional_read_circuit_breaking_error(
763            "thread panicked at crates/client-api/src/routes/subscribe.rs:1366:48: buffer should be unique"
764        ));
765    }
766}