objectiveai_mcp_plugin_framework/db.rs
1//! The plugin's database.
2//!
3//! Every plugin container gets its own Postgres, tunnelled in by the
4//! laboratory host: the host binds a listener, hands the daemon's
5//! role, password and database through, and stamps the assembled URL
6//! as `OBJECTIVEAI_POSTGRES_URL` at create time. A plugin therefore
7//! never configures a database — it asks for one.
8//!
9//! ```no_run
10//! # use std::sync::Arc;
11//! # async fn example() -> Result<(), Arc<objectiveai_mcp_plugin_framework::db::Error>> {
12//! // Connects once, however many times or from however many tasks
13//! // this is called.
14//! let pool = objectiveai_mcp_plugin_framework::db::connect(Default::default()).await?;
15//! # let _ = pool;
16//! # Ok(())
17//! # }
18//! ```
19//!
20//! **[`connect`] connects at most once.** A plugin has one database
21//! for its whole life, so handing out a second pool would only spend
22//! connections on a duplicate of the first. What the memoization has
23//! to get right is what happens when callers overlap, and when the
24//! connection fails:
25//!
26//! - **Once it succeeds, that is the answer forever.** Later calls
27//! return the same pool without taking a lock at all.
28//! - **Overlapping callers share ONE attempt.** The first starts it;
29//! everyone arriving while it runs waits on that same attempt and
30//! receives its outcome, success or failure alike. Only one
31//! connection attempt is ever in flight.
32//! - **Failure is not cached.** Once a failed attempt finishes, the
33//! slot is cleared and the next call starts fresh. A tunnel that was
34//! not up yet does not poison the process.
35//!
36//! Sharing an outcome means cloning it, so the error is an
37//! `Arc<Error>` — `sqlx::Error` is not `Clone`, and every waiter on a
38//! failed attempt gets that same error.
39
40use std::sync::{Arc, LazyLock, Mutex, OnceLock};
41use std::time::Duration;
42
43use futures::FutureExt as _;
44use futures::future::{BoxFuture, Shared};
45use sqlx::postgres::PgPoolOptions;
46
47/// A handle to the plugin's database.
48///
49/// A POOL rather than a single connection: it is cheap to clone,
50/// shares one set of connections between every clone, and is what
51/// every `sqlx` query API already takes. Hold one and clone it around.
52pub type Pool = sqlx::PgPool;
53
54/// Everything [`connect`] can fail with.
55///
56/// Handed out as `Arc<Error>`, because overlapping callers share one
57/// attempt and [`Shared`] gives each of them a CLONE of its output —
58/// which `sqlx::Error` is not.
59#[derive(Debug, thiserror::Error)]
60pub enum Error {
61 /// No `OBJECTIVEAI_POSTGRES_URL` in the environment.
62 ///
63 /// Not a misconfiguration a plugin can fix: the laboratory host
64 /// stamps this variable on every plugin container it creates, so
65 /// its absence means this process is not running as one.
66 #[error(
67 "no database: OBJECTIVEAI_POSTGRES_URL is unset, which means this process is not running as a plugin container"
68 )]
69 NoDatabase,
70 /// The URL was there and the connection still failed — the tunnel
71 /// is down, the credentials are stale, or Postgres refused.
72 #[error("connect to the plugin database")]
73 Connect(#[source] sqlx::Error),
74}
75
76/// How the pool is built.
77///
78/// [`Default`] suits a plugin container as it actually runs: nothing
79/// held open when idle, a generous ceiling for when it is not, and a
80/// bounded wait. Reach for the fields only to tighten them.
81///
82/// Only the attempt that actually CONNECTS is shaped by this — see
83/// [`connect`].
84#[derive(Debug, Clone, PartialEq, Eq)]
85pub struct Config {
86 /// Ceiling on open connections.
87 ///
88 /// High by default, and cheap to leave high: the pool opens
89 /// connections ON DEMAND and holds none idle (see
90 /// [`Config::min_connections`]), so a plugin that uses three opens
91 /// three. The ceiling only bites under real concurrency, and a
92 /// plugin that fans out should not have to discover a config knob
93 /// to do so.
94 ///
95 /// It is deliberately NOT a throttle, and it matches the server's
96 /// own `max_connections=1024`. Exhaustion therefore surfaces as
97 /// Postgres refusing ("too many clients already"), not as this
98 /// pool queueing — which is the right place for it: one plugin
99 /// should not be silently rationed to a fraction of a budget it
100 /// may be the only claimant on.
101 pub max_connections: u32,
102 /// Connections kept open when idle. Zero by default — a container
103 /// that finishes its completion in a second should not have held
104 /// connections open waiting for work that will never come.
105 pub min_connections: u32,
106 /// How long `acquire` waits for a free connection before giving
107 /// up. Bounded so a query storm surfaces as an error rather than
108 /// as a plugin that hangs until the completion times out.
109 pub acquire_timeout: Duration,
110}
111
112impl Default for Config {
113 fn default() -> Self {
114 Self {
115 max_connections: 1024,
116 min_connections: 0,
117 acquire_timeout: Duration::from_secs(30),
118 }
119 }
120}
121
122/// The connection, once it exists. TERMINAL: nothing ever replaces or
123/// clears it, which is what lets the fast path skip the lock entirely.
124static POOL: OnceLock<Pool> = OnceLock::new();
125
126/// The attempt currently in flight, if any.
127///
128/// A plain `Mutex`, and not something cleverer, because it is only
129/// ever held to clone a handle or clear a slot — never across an await
130/// — and after the first success [`POOL`] short-circuits before this
131/// is even looked at.
132static ATTEMPT: LazyLock<Mutex<Attempt>> = LazyLock::new(|| {
133 Mutex::new(Attempt {
134 current: None,
135 generation: 0,
136 })
137});
138
139struct Attempt {
140 /// The live attempt, and the generation identifying it.
141 current: Option<(u64, Shared<BoxFuture<'static, Result<Pool, Arc<Error>>>>)>,
142 /// Bumped per attempt, so a finishing attempt can tell whether the
143 /// slot still holds ITSELF before clearing it. Without it, a slow
144 /// failure could clear the slot out from under a newer attempt
145 /// that had already replaced it.
146 generation: u64,
147}
148
149/// Connect to the plugin's database, or return the connection already
150/// made.
151///
152/// The URL is NOT a parameter: it comes from the environment the
153/// laboratory host stamped, which is the only authority on where this
154/// container's Postgres lives. A plugin that could pass its own URL
155/// could dial somewhere it was never granted.
156///
157/// `config` shapes only the attempt that actually connects. A call
158/// that finds a pool already made returns it as-is, and a call that
159/// joins a running attempt gets that attempt's config — so passing
160/// different configs from different call sites is not meaningful, and
161/// whoever connects first decides.
162///
163/// Connects EAGERLY, so a bad tunnel or stale credentials surface here
164/// rather than at the first query somewhere deep in a tool call.
165pub async fn connect(config: Config) -> Result<Pool, Arc<Error>> {
166 coalesce(|| connect_once(config).boxed()).await
167}
168
169/// The memoization itself, separated from what it memoizes.
170///
171/// `start` is only called when there is no pool and no attempt in
172/// flight. Splitting it out is what makes the coalescing testable: the
173/// real attempt fails synchronously when there is no database, so it
174/// never suspends, so nothing can ever overlap it — and a test that
175/// drove the real thing would prove nothing about joining.
176async fn coalesce(
177 start: impl FnOnce() -> BoxFuture<'static, Result<Pool, Arc<Error>>>,
178) -> Result<Pool, Arc<Error>> {
179 // After the first success, this is the entire function.
180 if let Some(pool) = POOL.get() {
181 return Ok(pool.clone());
182 }
183
184 // Join the running attempt, or start one. The guard is dropped
185 // before the await: a lock held across a connection attempt would
186 // serialize the very callers this exists to coalesce.
187 let (generation, attempt) = {
188 let mut state = ATTEMPT.lock().unwrap_or_else(|e| e.into_inner());
189
190 // RETIRE a finished attempt before deciding anything.
191 //
192 // Without this there is a window — between an attempt
193 // resolving and a waiter clearing the slot — in which the slot
194 // still holds a FINISHED attempt. A caller arriving then would
195 // join it and receive its cached outcome, which for a failure
196 // is exactly the stale error that is supposed to be
197 // unreachable. Worse, if every waiter is dropped in that
198 // window, nobody ever clears and the next caller is guaranteed
199 // the dead attempt.
200 //
201 // Deciding it HERE, under the same lock that hands out joins,
202 // is what makes the boundary exact: "arrived before it
203 // finished" and "arrived after" are separated by this critical
204 // section rather than by a clear that races it.
205 let finished = state.current.as_ref().and_then(|(_, a)| a.peek().cloned());
206 match finished {
207 // It succeeded and we are the first to notice. Publish and
208 // hand it back: starting a fresh connection here would
209 // waste one, and joining a completed future would leave it
210 // in the slot forever.
211 Some(Ok(pool)) => {
212 state.current = None;
213 drop(state);
214 let _ = POOL.set(pool.clone());
215 return Ok(pool);
216 }
217 // It failed, and we arrived after. Retire it and start over
218 // — the whole point of not caching failure.
219 Some(Err(_)) => state.current = None,
220 // Still running, so joinable.
221 None => {}
222 }
223
224 match &state.current {
225 Some((generation, attempt)) => (*generation, attempt.clone()),
226 None => {
227 state.generation += 1;
228 let generation = state.generation;
229 let attempt = start().shared();
230 state.current = Some((generation, attempt.clone()));
231 (generation, attempt)
232 }
233 }
234 };
235
236 match attempt.await {
237 Ok(pool) => {
238 // Whichever waiter arrives first wins the `set`; the rest
239 // return the same pool regardless, since they are all
240 // holding clones of one `PgPool`.
241 let _ = POOL.set(pool.clone());
242 // Release the slot too. `POOL` is the answer from here on,
243 // so holding the completed attempt would retain a finished
244 // future and a duplicate pool handle for the life of the
245 // process.
246 let mut state = ATTEMPT.lock().unwrap_or_else(|e| e.into_inner());
247 if state.current.as_ref().is_some_and(|(g, _)| *g == generation) {
248 state.current = None;
249 }
250 Ok(pool)
251 }
252 Err(error) => {
253 // Clear the slot so the NEXT caller starts fresh — but
254 // only if it still holds THIS attempt, since a newer one
255 // may already have replaced it.
256 let mut state = ATTEMPT.lock().unwrap_or_else(|e| e.into_inner());
257 if state.current.as_ref().is_some_and(|(g, _)| *g == generation) {
258 state.current = None;
259 }
260 Err(error)
261 }
262 }
263}
264
265/// One connection attempt, with no memoization of its own.
266async fn connect_once(config: Config) -> Result<Pool, Arc<Error>> {
267 let url = crate::environment::postgres_url()
268 .ok_or_else(|| Arc::new(Error::NoDatabase))?;
269 PgPoolOptions::new()
270 .max_connections(config.max_connections)
271 .min_connections(config.min_connections)
272 .acquire_timeout(config.acquire_timeout)
273 .connect(url)
274 .await
275 .map_err(|e| Arc::new(Error::Connect(e)))
276}
277
278#[cfg(test)]
279mod tests {
280 use super::*;
281
282 /// Every test here depends on there being no database to reach.
283 /// Inside a real plugin container there is one, and asserting the
284 /// opposite of the truth would be worse than skipping.
285 fn outside_a_plugin_container() -> bool {
286 crate::environment::postgres_url().is_none()
287 }
288
289 /// The memoization is PROCESS-global, so tests that count attempts
290 /// would otherwise count each other — libtest runs them on
291 /// separate threads by default. Every test touching `ATTEMPT`
292 /// takes this first.
293 static SERIAL: Mutex<()> = Mutex::new(());
294
295 fn exclusive() -> std::sync::MutexGuard<'static, ()> {
296 SERIAL.lock().unwrap_or_else(|e| e.into_inner())
297 }
298
299 /// A generous ceiling, nothing held idle, and a bounded wait — the
300 /// pool never costs what it does not use, but never throttles a
301 /// plugin that genuinely fans out either.
302 #[test]
303 fn default_config_is_generous_but_bounded() {
304 let config = Config::default();
305 assert_eq!(config.max_connections, 1024);
306 assert_eq!(config.min_connections, 0);
307 assert_eq!(config.acquire_timeout, Duration::from_secs(30));
308 }
309
310 /// The missing-URL case has to be a named error a caller can match
311 /// on, not an opaque connection failure — it means "not running as
312 /// a plugin", which is a different problem with a different fix.
313 #[tokio::test]
314 async fn no_url_is_its_own_error() {
315 let _serial = exclusive();
316 if !outside_a_plugin_container() {
317 return;
318 }
319 let error = connect(Config::default()).await.expect_err("no database");
320 assert!(matches!(*error, Error::NoDatabase));
321 }
322
323 /// Failure is not cached: each later call starts over. A plugin
324 /// that raced the tunnel coming up must be able to recover, which
325 /// a remembered error would prevent forever.
326 #[tokio::test]
327 async fn a_failed_attempt_leaves_no_trace() {
328 let _serial = exclusive();
329 if !outside_a_plugin_container() {
330 return;
331 }
332 let before = ATTEMPT.lock().unwrap().generation;
333 for _ in 0..3 {
334 assert!(connect(Config::default()).await.is_err());
335 }
336 let state = ATTEMPT.lock().unwrap();
337 assert!(
338 state.current.is_none(),
339 "the slot is empty, so the next caller starts fresh",
340 );
341 assert_eq!(
342 state.generation - before,
343 3,
344 "and each sequential call really did start its own attempt",
345 );
346 }
347
348 /// Overlapping callers share ONE attempt and all receive its
349 /// outcome — the coalescing that stops a burst of tool calls
350 /// opening a burst of connections.
351 ///
352 /// Driven through [`coalesce`] with an attempt that SUSPENDS,
353 /// because the real one does not: with no database it fails
354 /// without ever yielding, so callers can never overlap it and a
355 /// test against it would pass while proving nothing.
356 #[tokio::test]
357 async fn overlapping_callers_share_one_attempt() {
358 let _serial = exclusive();
359 let before = ATTEMPT.lock().unwrap().generation;
360
361 let results = futures::future::join_all((0..8).map(|_| {
362 coalesce(|| {
363 async {
364 // A real pause, not a single `yield_now`: one
365 // reschedule guarantees only ONE further poll,
366 // which can let the attempt finish before the last
367 // callers have been polled even once. A timer holds
368 // the window open for all of them.
369 tokio::time::sleep(Duration::from_millis(50)).await;
370 Err(Arc::new(Error::NoDatabase))
371 }
372 .boxed()
373 })
374 }))
375 .await;
376
377 let after = ATTEMPT.lock().unwrap().generation;
378 assert_eq!(results.len(), 8);
379 assert!(results.iter().all(|r| r.is_err()), "all eight see the failure");
380 assert_eq!(
381 after - before,
382 1,
383 "eight concurrent callers, one attempt between them",
384 );
385 }
386
387 /// A caller arriving AFTER a failed attempt resolved must start
388 /// over, not inherit its error.
389 ///
390 /// The natural window is too narrow to hit on purpose, so the
391 /// state it produces is built directly: a slot holding an attempt
392 /// that has already finished with an error, exactly as it looks
393 /// between resolution and the waiter clearing it.
394 #[tokio::test]
395 async fn a_finished_failure_is_retired_not_joined() {
396 let _serial = exclusive();
397
398 // An attempt that is already resolved, with a DISTINGUISHABLE
399 // error.
400 let stale: Shared<BoxFuture<'static, Result<Pool, Arc<Error>>>> =
401 async { Err(Arc::new(Error::NoDatabase)) }.boxed().shared();
402 assert!(
403 stale.clone().now_or_never().is_some(),
404 "resolved, so it is what the slot holds after a failure",
405 );
406
407 let before = {
408 let mut state = ATTEMPT.lock().unwrap();
409 state.generation += 1;
410 state.current = Some((state.generation, stale));
411 state.generation
412 };
413
414 // A fresh attempt with a different error, so the two are
415 // telling apart.
416 let outcome = coalesce(|| {
417 async { Err(Arc::new(Error::Connect(sqlx::Error::PoolClosed))) }.boxed()
418 })
419 .await;
420
421 let error = outcome.expect_err("still no database");
422 assert!(
423 matches!(*error, Error::Connect(_)),
424 "the FRESH attempt answered, not the retired one",
425 );
426 assert_eq!(
427 ATTEMPT.lock().unwrap().generation - before,
428 1,
429 "and it really was a new attempt",
430 );
431 assert!(
432 ATTEMPT.lock().unwrap().current.is_none(),
433 "which then retired itself in turn",
434 );
435 }
436
437 /// And once that shared attempt has failed, the slot is empty
438 /// again — the eight callers coalesced, but they did not leave the
439 /// failure behind for a ninth.
440 #[tokio::test]
441 async fn a_shared_failure_still_clears_the_slot() {
442 let _serial = exclusive();
443 let _ = coalesce(|| {
444 async {
445 tokio::time::sleep(Duration::from_millis(10)).await;
446 Err(Arc::new(Error::NoDatabase))
447 }
448 .boxed()
449 })
450 .await;
451 assert!(ATTEMPT.lock().unwrap().current.is_none());
452 }
453}