Skip to main content

surrealdb_engine_local/
lib.rs

1//! The embedded SurrealDB engine driven by the Rust SDK.
2//!
3//! This crate owns everything that needs the database core: it opens a
4//! [`Datastore`], which runs its own background maintenance [`tasks`], and
5//! serves it to the SDK as a [`SurrealEngine`](surrealdb_engine_api::SurrealEngine).
6//! The SDK reaches it only through [`surrealdb_engine_api`], which is why
7//! enabling one of this crate's storage backends is the only thing that puts
8//! the core in a Rust SDK build.
9//!
10//! # Stability
11//!
12//! This is an internal interface between crates released together. It carries
13//! no stability guarantee and may change in any release, including a patch
14//! release. Application code should use the
15//! [`surrealdb`](https://docs.rs/surrealdb) crate and select a storage backend
16//! with its `kv-*` features.
17
18#![recursion_limit = "256"]
19// TODO: Remove once this crate no longer needs it; see `absolute_paths` in the workspace lints.
20#![allow(clippy::absolute_paths)]
21
22#[macro_use]
23extern crate tracing;
24
25use std::future::Future;
26use std::path::PathBuf;
27use std::sync::Arc;
28use std::time::Duration;
29
30use async_channel::Receiver;
31use surrealdb_cnf::ConfigMap;
32#[doc(inline)]
33pub use surrealdb_core::kvs::Datastore;
34#[doc(inline)]
35pub use surrealdb_core::options::EngineOptions;
36use surrealdb_engine_api::SessionId;
37use surrealdb_rpc::capabilities::Capabilities;
38use surrealdb_types::{Error, Notification};
39
40mod engine;
41mod session;
42
43/// The datastore's background maintenance tasks.
44///
45/// Re-exported because the datastore starts them itself; this path is what
46/// `surrealdb::engine::tasks` resolves to.
47#[doc(inline)]
48pub use surrealdb_core::kvs::tasks;
49
50pub use crate::engine::LocalEngine;
51use crate::session::SessionRegistry;
52
53/// The root credentials to initialise the datastore with on first start.
54#[derive(Debug, Clone)]
55pub struct Root {
56	/// The username of the root user.
57	pub username: String,
58	/// The password of the root user.
59	pub password: String,
60}
61
62/// Everything the engine needs to open a datastore and start serving routes.
63///
64/// The SDK derives this from the endpoint the caller connected to; the engine
65/// never sees the SDK's own connection configuration.
66#[derive(Debug, Clone, Default)]
67pub struct LocalConfig {
68	/// The datastore to open, as a scheme-prefixed path (`mem://`,
69	/// `surrealkv://path`).
70	pub path: String,
71	/// The root user to create when the datastore has no credentials yet.
72	/// `None` starts the datastore with authentication disabled.
73	pub root: Option<Root>,
74	/// The maximum time a single query may run for.
75	pub query_timeout: Option<Duration>,
76	/// The maximum time a single transaction may run for.
77	pub transaction_timeout: Option<Duration>,
78	/// What the datastore is allowed to do.
79	pub capabilities: Capabilities,
80	/// The directories a local `file:` bucket backend may be defined in. Empty
81	/// refuses every such bucket.
82	pub bucket_folder_allowlist: Vec<PathBuf>,
83	/// Where the datastore spills large intermediate results, when the storage
84	/// backends that need one are compiled in.
85	pub temporary_directory: Option<PathBuf>,
86	/// Overrides for the corresponding [`EngineOptions`] intervals. `None`
87	/// keeps the default.
88	pub node_membership_refresh_interval: Option<Duration>,
89	/// Overrides the node membership check interval.
90	pub node_membership_check_interval: Option<Duration>,
91	/// Overrides the node membership cleanup interval.
92	pub node_membership_cleanup_interval: Option<Duration>,
93	/// Overrides the changefeed garbage collection interval.
94	pub changefeed_gc_interval: Option<Duration>,
95	/// Datastore configuration keys and values, as the server reads them
96	/// from `SURREAL_<KEY>`: storage backend settings such as
97	/// `surrealkv_block_cache_capacity` and runtime settings such as
98	/// `memory_limit`. A parameter on `path` wins over a backend setting here.
99	/// `bucket_folder_allowlist` is refused: it is set by its own field.
100	pub datastore_options: Vec<(String, String)>,
101}
102
103impl LocalConfig {
104	/// The engine options this configuration implies, leaving every interval it
105	/// does not override at its default.
106	fn engine_options(&self) -> EngineOptions {
107		let mut opt = EngineOptions::default();
108		if let Some(interval) = self.node_membership_refresh_interval {
109			opt.node_membership_refresh_interval = interval;
110		}
111		if let Some(interval) = self.node_membership_check_interval {
112			opt.node_membership_check_interval = interval;
113		}
114		if let Some(interval) = self.node_membership_cleanup_interval {
115			opt.node_membership_cleanup_interval = interval;
116		}
117		if let Some(interval) = self.changefeed_gc_interval {
118			opt.changefeed_gc_interval = interval;
119		}
120		opt
121	}
122
123	/// The configuration map the datastore is built with.
124	///
125	/// It holds only the keys this configuration sets. Nothing is read from the
126	/// process environment, so an embedded datastore can do only what its
127	/// caller configured.
128	fn datastore_config(&self) -> Result<ConfigMap, Error> {
129		let mut map = ConfigMap::empty();
130		for (key, value) in &self.datastore_options {
131			// Keys are matched as the environment produces them.
132			let key = key.to_lowercase();
133			if key == "bucket_folder_allowlist" {
134				return Err(std_error_to_types_error(
135					"the bucket folder allowlist is set by its own option, not as a datastore option",
136				));
137			}
138			map = map.with_key_value(key, value.clone());
139		}
140		if !self.bucket_folder_allowlist.is_empty() {
141			// The datastore reads the allowlist as a path list, the form the
142			// server's environment variable takes.
143			let joined = std::env::join_paths(&self.bucket_folder_allowlist)
144				.map_err(|e| {
145					std_error_to_types_error(format!("invalid bucket folder allowlist: {e}"))
146				})?
147				.into_string()
148				.map_err(|_| {
149					std_error_to_types_error("the bucket folder allowlist is not valid UTF-8")
150				})?;
151			map = map.with_key_value("bucket_folder_allowlist", joined);
152		}
153		Ok(map)
154	}
155}
156
157/// Convert an error that is only reportable as text into the SDK's error type.
158fn std_error_to_types_error(error: impl std::fmt::Display) -> surrealdb_types::Error {
159	surrealdb_types::Error::internal(error.to_string())
160}
161
162/// Spawns a detached task on whichever executor this target has.
163///
164/// Nothing joins one of these, which is what lets the browser's `spawn_local`
165/// -- which returns no handle at all -- stand in for `tokio::spawn`. Every
166/// other target spawns on the ambient tokio runtime.
167#[cfg(not(all(target_family = "wasm", target_os = "unknown")))]
168fn spawn<F>(future: F)
169where
170	F: Future<Output = ()> + Send + 'static,
171{
172	drop(tokio::spawn(future));
173}
174
175#[cfg(all(target_family = "wasm", target_os = "unknown"))]
176fn spawn<F>(future: F)
177where
178	F: Future<Output = ()> + 'static,
179{
180	wasm_bindgen_futures::spawn_local(future);
181}
182
183/// Opens the datastore `config` describes and serves it to the SDK.
184///
185/// The returned engine owns the datastore. `session_rx` is the SDK's session
186/// lifecycle channel: it registers the sessions requests run under, and its
187/// closing -- which happens when the last connection handle is dropped -- is
188/// what shuts the datastore down.
189pub async fn connect(
190	config: LocalConfig,
191	session_rx: Receiver<SessionId>,
192) -> Result<Arc<LocalEngine>, Error> {
193	let builder = Datastore::builder()
194		.with_config(config.datastore_config()?)
195		// The datastore starts its own maintenance tasks, so the cadences go to
196		// the builder rather than to a `tasks::init` call here.
197		.with_engine_options(config.engine_options())
198		.with_query_timeout(config.query_timeout)
199		.with_transaction_timeout(config.transaction_timeout)
200		.with_auth(config.root.is_some());
201
202	#[cfg(storage)]
203	let builder = builder.with_temporary_directory(config.temporary_directory);
204
205	let (notifications, builder) = if config.capabilities.allows_live_query_notifications() {
206		let (send, recv) = async_channel::bounded(surrealdb_cnf::NOTIFICATIONS_CHANNEL_SIZE);
207		(Some(recv), builder.with_notify(send))
208	} else {
209		(None, builder)
210	};
211
212	let builder = builder.with_capabilities(config.capabilities);
213
214	// The maintenance tasks are started by hand below rather than at
215	// construction. Every one of them writes, and a write that lands before the
216	// version key does makes a datastore this process just created look like one
217	// written before that key existed — which `check_version` is obliged to
218	// refuse, permanently, on this and every later open.
219	let kvs = builder
220		.without_maintenance_tasks()
221		.build_with_path(&config.path)
222		.await
223		.map_err(std_error_to_types_error)?;
224	// A failure past this point must close the datastore: the caller never
225	// receives a handle to close, and an engine that holds an exclusive lock on
226	// its storage would otherwise keep it until the process ends.
227	let started = async {
228		kvs.check_version().await.map_err(std_error_to_types_error)?;
229		kvs.bootstrap().await.map_err(std_error_to_types_error)?;
230		kvs.start_maintenance_tasks_checked().await.map_err(std_error_to_types_error)?;
231		// If a root user is specified, setup the initial datastore credentials. A datastore
232		// that is open read-only refuses the write, so the step is skipped and the store
233		// serves what it already holds.
234		if let Some(root) = &config.root {
235			if kvs.is_read_only() {
236				tracing::warn!(
237					"The datastore is open read-only, so the root credentials were not initialised"
238				);
239			} else {
240				kvs.initialise_credentials(&root.username, &root.password)
241					.await
242					.map_err(std_error_to_types_error)?;
243			}
244		}
245		Ok(())
246	}
247	.await;
248	if let Err(e) = started {
249		kvs.shutdown().await.ok();
250		return Err(e);
251	}
252
253	Ok(serve(kvs, notifications, session_rx))
254}
255
256/// Serves an already-open datastore to the SDK.
257///
258/// The datastore arrives with its own maintenance tasks and its own
259/// cancellation, both established when it was built, so neither is passed in
260/// here. A caller that wants to control either sets them on the
261/// [`Builder`](surrealdb_core::kvs::ds::builder::Builder) it constructs the
262/// datastore with.
263pub fn from_datastore(
264	datastore: Arc<Datastore>,
265	notifications: Option<Receiver<Notification>>,
266	session_rx: Receiver<SessionId>,
267) -> Arc<LocalEngine> {
268	serve(datastore, notifications, session_rx)
269}
270
271/// Builds the engine and starts the two tasks that outlive an individual
272/// request: the one tracking the SDK's sessions, and the one delivering
273/// live-query notifications to them.
274fn serve(
275	kvs: Arc<Datastore>,
276	notifications: Option<Receiver<Notification>>,
277	session_rx: Receiver<SessionId>,
278) -> Arc<LocalEngine> {
279	let sessions = Arc::new(SessionRegistry::default());
280
281	// Both tasks end without reference to the engine: `run` on the session
282	// channel closing, and `pump` on the notification channel `run` then closes.
283	// Holding the engine instead would make the engine's own liveness the
284	// condition for its tasks ending, which nothing would ever satisfy.
285	let pump = notifications
286		.clone()
287		.map(|notifications| session::pump(Arc::clone(&kvs), Arc::clone(&sessions), notifications));
288	let run = session::run(Arc::clone(&kvs), Arc::clone(&sessions), session_rx, notifications);
289	// Answers once, when both tasks have finished and dropped their handles on
290	// the datastore: what `run` reported of shutting it down.
291	let (stopped_tx, stopped) = async_channel::bounded::<Result<(), Error>>(1);
292	spawn(async move {
293		let outcome = match pump {
294			Some(pump) => futures::future::join(run, pump).await.0,
295			None => run.await,
296		};
297		stopped_tx.send(outcome).await.ok();
298	});
299
300	Arc::new(LocalEngine {
301		kvs,
302		sessions,
303		stopped,
304	})
305}
306
307#[cfg(all(test, not(target_family = "wasm")))]
308mod tests {
309	use surrealdb_kvs_surrealkv::SurrealKvConfig;
310
311	use super::LocalConfig;
312
313	fn options(pairs: &[(&str, &str)]) -> LocalConfig {
314		LocalConfig {
315			datastore_options: pairs.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(),
316			..LocalConfig::default()
317		}
318	}
319
320	/// The storage settings an embedded caller sets reach the backend
321	/// configuration the datastore is opened with, whatever their case.
322	#[test]
323	fn datastore_options_reach_the_backend_config() {
324		let config = options(&[
325			("surrealkv_block_cache_capacity", "268435456"),
326			("SURREALKV_MAX_MEMTABLE_SIZE", "67108864"),
327		]);
328		let engine = config.datastore_config().unwrap().load::<SurrealKvConfig>();
329		assert_eq!(engine.block_cache_capacity, 256 * 1024 * 1024);
330		assert_eq!(engine.max_memtable_size, 64 * 1024 * 1024);
331	}
332
333	/// Unset options leave the backend's defaults in place.
334	#[test]
335	fn unset_datastore_options_keep_the_backend_defaults() {
336		let engine = LocalConfig::default().datastore_config().unwrap().load::<SurrealKvConfig>();
337		let defaults = SurrealKvConfig::default();
338		assert_eq!(engine.block_cache_capacity, defaults.block_cache_capacity);
339		assert_eq!(engine.max_memtable_size, defaults.max_memtable_size);
340	}
341
342	/// The allowlist is a capability, so it is set only by its own field.
343	#[test]
344	fn bucket_folder_allowlist_is_refused_as_a_datastore_option() {
345		for key in ["bucket_folder_allowlist", "BUCKET_FOLDER_ALLOWLIST"] {
346			assert!(options(&[(key, "/")]).datastore_config().is_err(), "{key}");
347		}
348	}
349}