halephant_core/pool/checkout.rs
1//! Connection checkout — the hot path through the pool. Walks
2//! candidate nodes, falls back to opening new connections, and
3//! enqueues on the shared `(database, user, role)` wait queue when
4//! every candidate is at its per-node ceiling.
5
6use std::sync::Arc;
7use std::sync::atomic::Ordering;
8use std::time::Instant;
9
10use tokio::sync::oneshot;
11use tracing::{debug, trace};
12
13use crate::clients;
14use crate::connections::server;
15use crate::errors::ResolveError;
16use crate::o11y;
17use crate::pool::PoolManager;
18use crate::pool::types::{ConnGuard, PoolKey, Routing, TryCheckout, WaitKey};
19
20impl PoolManager {
21 /// Check out a server connection.
22 ///
23 /// When `read_only` is true, routes to a replica. When false, routes
24 /// to the primary. Returns an RAII guard that auto-discards on drop;
25 /// call `checkin` to return the connection for reuse.
26 ///
27 /// If every candidate node for the resolved role is at its per-node
28 /// `max_connections`, the checkout enqueues on a shared
29 /// `(database, user, role)` wait queue and blocks until either:
30 ///
31 /// 1. a connection is released on any candidate node and this
32 /// waiter is the next in the FIFO queue, or
33 /// 2. `server.checkout_timeout` elapses since the first enqueue,
34 /// in which case the checkout fails with a classified
35 /// `checkout_timeout` error.
36 ///
37 /// The `ClientGuard` is flipped to `ClientState::Waiting` for the
38 /// duration of any blocked wait and back to `ClientState::Idle` on
39 /// success, timeout, or spurious wakeup.
40 #[tracing::instrument(name = "pool.checkout", skip_all, err(Display), fields(
41 db.namespace = %database,
42 user = %user,
43 halephant.client.id = client.id().as_u64(),
44 read_only,
45 pool.reused,
46 pool.waited,
47 otel.status_code,
48 otel.status_description,
49 ))]
50 pub async fn checkout(
51 self: &Arc<Self>,
52 client: &clients::ClientGuard,
53 database: &str,
54 user: &str,
55 read_only: bool,
56 ) -> anyhow::Result<ConnGuard> {
57 let routing = if read_only {
58 Routing::Replica
59 } else {
60 Routing::Primary
61 };
62 let wait_key = WaitKey {
63 database: database.to_owned(),
64 user: user.to_owned(),
65 role: routing,
66 };
67
68 // Snapshot config once for the whole checkout. An atomic swap
69 // during this operation does not affect in-flight checkouts —
70 // they finish under the old view and the next checkout picks
71 // up the new one. `cfg` owns an `Arc<Config>`, so borrows
72 // derived from it are valid for the function lifetime and
73 // survive across `.await`.
74 let cfg = self.config.load_full();
75 let max_prepared_statements = cfg.server.max_prepared_statements;
76
77 // Resolve config-derived parameters once. The per-pool
78 // `checkout_timeout` override takes precedence over the
79 // server-wide default.
80 let (
81 cluster_name,
82 connect_timeout,
83 idle_timeout,
84 max_lifetime,
85 checkout_timeout,
86 upstream_database,
87 ) = {
88 let Some((name, cluster, pool_config)) = cfg.find_pool(database) else {
89 return Err(ResolveError::UnknownDatabase {
90 database: database.to_owned(),
91 }
92 .into());
93 };
94 (
95 name.to_owned(),
96 cluster.connect_timeout,
97 pool_config.idle_timeout,
98 pool_config.max_lifetime,
99 pool_config
100 .checkout_timeout
101 .unwrap_or(cfg.server.checkout_timeout),
102 pool_config.database_name(database).to_owned(),
103 )
104 };
105
106 // Look up user-specific connection parameters once for the batch.
107 // These are only used if we actually open a new connection.
108 let (upstream_user, user_params) = {
109 let user_config = cfg.find_user(database, user);
110 let upstream_user = user_config
111 .map_or(user, |u| u.upstream_name(user))
112 .to_owned();
113 let params = user_config
114 .map(|u| u.parameters.clone())
115 .unwrap_or_default();
116 (upstream_user, params)
117 };
118
119 let checkout_start = Instant::now();
120 let mut queue_start: Option<Instant> = None;
121
122 async {
123 loop {
124 // Recompute candidates each loop iteration so a
125 // topology change (e.g., primary failover) during a
126 // wait is picked up on retry.
127 let candidates = self.candidate_nodes(&cfg, database, user, routing);
128 if candidates.is_empty() {
129 return Err(match routing {
130 Routing::Primary => ResolveError::NoPrimary {
131 cluster: cluster_name.clone(),
132 },
133 Routing::Replica => ResolveError::NoReplica {
134 cluster: cluster_name.clone(),
135 },
136 }
137 .into());
138 }
139
140 // 1. Try every candidate atomically before giving up.
141 // `any_full` tracks whether any candidate was at its
142 // ceiling (meaning a slot will eventually free up and
143 // waiting is productive); `last_connect_error` keeps
144 // the most recent connect/auth failure so we can
145 // surface it if no candidate is worth waiting on.
146 let mut any_full = false;
147 let mut last_connect_error: Option<anyhow::Error> = None;
148 for key in &candidates {
149 let max = self.max_for_key(&cfg, key);
150 match self.try_checkout(key, max, idle_timeout, max_lifetime) {
151 TryCheckout::TookIdle(idle) => {
152 if let Some(queued_at) = queue_start {
153 o11y::metrics::record_wait_duration(
154 queued_at.elapsed(),
155 database,
156 user,
157 routing,
158 );
159 }
160 o11y::metrics::record_checkout(
161 checkout_start,
162 database,
163 user,
164 &key.node,
165 );
166 let span = tracing::Span::current();
167 span.record("pool.reused", true);
168 span.record("pool.waited", queue_start.is_some());
169 debug!(
170 database,
171 user,
172 node = %key.node,
173 "reusing idle connection",
174 );
175 return Ok(ConnGuard {
176 pools: Arc::clone(self),
177 key: key.clone(),
178 conn: Some(idle.conn),
179 });
180 }
181 TryCheckout::MayOpen => {
182 let password =
183 self.lookup_password(&key.node, &upstream_database, &upstream_user);
184 match server::connect_server(
185 &key.node,
186 &upstream_database,
187 &upstream_user,
188 password.as_deref(),
189 &user_params,
190 max_prepared_statements,
191 connect_timeout,
192 )
193 .await
194 {
195 Ok(conn) => {
196 if let Some(queued_at) = queue_start {
197 o11y::metrics::record_wait_duration(
198 queued_at.elapsed(),
199 database,
200 user,
201 routing,
202 );
203 }
204 o11y::metrics::record_checkout(
205 checkout_start,
206 database,
207 user,
208 &key.node,
209 );
210 let span = tracing::Span::current();
211 span.record("pool.reused", false);
212 span.record("pool.waited", queue_start.is_some());
213 debug!(
214 database,
215 user,
216 node = %key.node,
217 "opened new server connection",
218 );
219 return Ok(ConnGuard {
220 pools: Arc::clone(self),
221 key: key.clone(),
222 conn: Some(conn),
223 });
224 }
225 Err(e) => {
226 // Connect/auth failed on this
227 // candidate — release the slot we
228 // speculatively claimed and fall
229 // through to the next candidate so
230 // a transient failure on one replica
231 // doesn't forfeit a healthy one
232 // later in the round-robin order.
233 self.discard_internal(key);
234 last_connect_error = Some(e);
235 }
236 }
237 }
238 TryCheckout::Full => {
239 any_full = true;
240 }
241 }
242 }
243
244 // 2. Every candidate either refused connections or hit
245 // its ceiling. If nothing was Full, there's no slot
246 // to wait for — surface the connect error directly.
247 // Otherwise fall through to the enqueue-and-wait
248 // path so the Full candidate can wake us when a
249 // slot frees up.
250 if !any_full && let Some(e) = last_connect_error {
251 return Err(e);
252 }
253
254 // 3. Every candidate is at capacity. Enqueue and wait.
255 // Record Waiting state and the target role only while
256 // actually blocked so fast checkouts don't flicker
257 // through Waiting in admin output.
258 let rx = {
259 let (tx, rx) = oneshot::channel();
260 let enqueued_at = Instant::now();
261 let mut inner = self.inner.lock();
262 inner
263 .waits
264 .entry(wait_key.clone())
265 .or_default()
266 .waiters
267 .push_back((enqueued_at, tx));
268 rx
269 };
270 client.set_waiting_for(Some(clients::WaitTarget {
271 database: database.to_owned(),
272 user: user.to_owned(),
273 role: match routing {
274 Routing::Primary => clients::WaitRole::Primary,
275 Routing::Replica => clients::WaitRole::Replica,
276 },
277 }));
278 client.set_state(clients::ClientState::Waiting);
279 // First enqueue stamps the queue clock; subsequent
280 // re-enqueues after a spurious wakeup keep the
281 // original value so the metric reflects total queue
282 // time, not per-wait segments.
283 queue_start.get_or_insert_with(Instant::now);
284
285 // Single timeout budget across the full checkout —
286 // spurious wakeups don't get a fresh clock. Measured
287 // from `checkout_start` so pre-enqueue candidate
288 // attempts count toward the overall deadline.
289 let elapsed = checkout_start.elapsed();
290 let remaining = checkout_timeout.saturating_sub(elapsed);
291 if remaining.is_zero() {
292 client.set_waiting_for(None);
293 client.set_state(clients::ClientState::Idle);
294 anyhow::bail!(
295 "checkout timed out after {checkout_timeout:?} for {database}/{user}",
296 );
297 }
298 let wake = tokio::time::timeout(remaining, rx).await;
299 client.set_waiting_for(None);
300 client.set_state(clients::ClientState::Idle);
301
302 if wake.is_err() {
303 // Outer `tokio::time::timeout` elapsed.
304 anyhow::bail!(
305 "checkout timed out after {checkout_timeout:?} for {database}/{user}",
306 );
307 }
308 // Shutdown wakes everyone via `wake_all_waiters`. A
309 // waiter that was spuriously woken by an unrelated
310 // checkin right as shutdown started would also loop
311 // here, so the check happens after the wake.
312 if self.shutting_down.load(Ordering::Acquire) {
313 anyhow::bail!("halephant is shutting down");
314 }
315 // Woken — loop top re-runs the candidate check.
316 }
317 }
318 .await
319 .inspect(|_| {
320 tracing::Span::current().record("otel.status_code", "OK");
321 })
322 .inspect_err(|e| {
323 // Classify the failure so `db.client.connection.errors` time
324 // series break down cleanly in dashboards.
325 let msg = e.to_string();
326 let error_type = if let Some(resolve_err) = e.downcast_ref::<ResolveError>() {
327 match resolve_err {
328 ResolveError::UnknownDatabase { .. } => "unknown_database",
329 ResolveError::NoPrimary { .. } => "no_primary",
330 ResolveError::NoReplica { .. } => "no_replica",
331 }
332 } else if msg.contains("shutting down") {
333 "shutting_down"
334 } else if msg.contains("checkout timed out") {
335 "checkout_timeout"
336 } else {
337 "connect_failed"
338 };
339 o11y::metrics::record_checkout_error(database, user, error_type);
340 let span = tracing::Span::current();
341 span.record("otel.status_code", "ERROR");
342 span.record("otel.status_description", "checkout failed");
343 })
344 }
345
346 /// Atomic check-and-claim for a single pool key.
347 ///
348 /// Discards expired idle connections inline (idle_timeout /
349 /// max_lifetime). Increments `pool.active` on success so a concurrent
350 /// checkout can't double-claim the same slot. Holds the inner lock
351 /// for the duration of one HashMap lookup + VecDeque pop — no
352 /// `.await` while locked.
353 fn try_checkout(
354 &self,
355 key: &PoolKey,
356 max: u32,
357 idle_timeout: std::time::Duration,
358 max_lifetime: std::time::Duration,
359 ) -> TryCheckout {
360 let mut inner = self.inner.lock();
361 let pool = inner.pools.entry(key.clone()).or_default();
362
363 // Drain expired idle entries until we find a usable one or the
364 // queue is empty. Each iteration drops the front entry.
365 while let Some(idle) = pool.idle.pop_front() {
366 if idle.idle_since.elapsed() > idle_timeout {
367 trace!(
368 database = %key.database,
369 user = %key.user,
370 "discarding idle connection (idle timeout)",
371 );
372 continue;
373 }
374 if idle.conn.created_at.elapsed() > max_lifetime {
375 trace!(
376 database = %key.database,
377 user = %key.user,
378 "discarding idle connection (max lifetime)",
379 );
380 continue;
381 }
382 pool.active += 1;
383 return TryCheckout::TookIdle(Box::new(idle));
384 }
385
386 if max == 0 || pool.total() < max {
387 pool.active += 1;
388 TryCheckout::MayOpen
389 } else {
390 TryCheckout::Full
391 }
392 }
393
394 /// Release a slot that was speculatively claimed by a failed
395 /// connect attempt or a dropped [`ConnGuard`]. Wakes a waiter if
396 /// there is one — the just-freed slot is a retry opportunity.
397 pub(in crate::pool) fn discard_internal(&self, key: &PoolKey) {
398 {
399 let mut inner = self.inner.lock();
400 if let Some(pool) = inner.pools.get_mut(key) {
401 pool.active = pool.active.saturating_sub(1);
402 trace!(
403 node = key.node,
404 database = key.database,
405 user = key.user,
406 "connection discarded"
407 );
408 }
409 }
410 // Freeing an active slot may allow a waiter to grow the pool.
411 self.wake_one_for_node(key);
412 }
413}