Skip to main content

halephant_core/pool/
checkout.rs

1//! Connection checkout — the hot path through the pool. Walks
2//! candidate nodes, falls back to opening new connections, and
3//! enqueues on the shared `(database, user, role)` wait queue when
4//! every candidate is at its per-node ceiling.
5
6use std::sync::Arc;
7use std::sync::atomic::Ordering;
8use std::time::Instant;
9
10use tokio::sync::oneshot;
11use tracing::{debug, trace};
12
13use crate::clients;
14use crate::connections::server;
15use crate::errors::ResolveError;
16use crate::o11y;
17use crate::pool::PoolManager;
18use crate::pool::types::{ConnGuard, PoolKey, Routing, TryCheckout, WaitKey};
19
20impl PoolManager {
21    /// Check out a server connection.
22    ///
23    /// When `read_only` is true, routes to a replica. When false, routes
24    /// to the primary. Returns an RAII guard that auto-discards on drop;
25    /// call `checkin` to return the connection for reuse.
26    ///
27    /// If every candidate node for the resolved role is at its per-node
28    /// `max_connections`, the checkout enqueues on a shared
29    /// `(database, user, role)` wait queue and blocks until either:
30    ///
31    /// 1. a connection is released on any candidate node and this
32    ///    waiter is the next in the FIFO queue, or
33    /// 2. `server.checkout_timeout` elapses since the first enqueue,
34    ///    in which case the checkout fails with a classified
35    ///    `checkout_timeout` error.
36    ///
37    /// The `ClientGuard` is flipped to `ClientState::Waiting` for the
38    /// duration of any blocked wait and back to `ClientState::Idle` on
39    /// success, timeout, or spurious wakeup.
40    #[tracing::instrument(name = "pool.checkout", skip_all, err(Display), fields(
41        db.namespace = %database,
42        user = %user,
43        halephant.client.id = client.id().as_u64(),
44        read_only,
45        pool.reused,
46        pool.waited,
47        otel.status_code,
48        otel.status_description,
49    ))]
50    pub async fn checkout(
51        self: &Arc<Self>,
52        client: &clients::ClientGuard,
53        database: &str,
54        user: &str,
55        read_only: bool,
56    ) -> anyhow::Result<ConnGuard> {
57        let routing = if read_only {
58            Routing::Replica
59        } else {
60            Routing::Primary
61        };
62        let wait_key = WaitKey {
63            database: database.to_owned(),
64            user: user.to_owned(),
65            role: routing,
66        };
67
68        // Snapshot config once for the whole checkout. An atomic swap
69        // during this operation does not affect in-flight checkouts —
70        // they finish under the old view and the next checkout picks
71        // up the new one. `cfg` owns an `Arc<Config>`, so borrows
72        // derived from it are valid for the function lifetime and
73        // survive across `.await`.
74        let cfg = self.config.load_full();
75        let max_prepared_statements = cfg.server.max_prepared_statements;
76
77        // Resolve config-derived parameters once. The per-pool
78        // `checkout_timeout` override takes precedence over the
79        // server-wide default.
80        let (
81            cluster_name,
82            connect_timeout,
83            idle_timeout,
84            max_lifetime,
85            checkout_timeout,
86            upstream_database,
87        ) = {
88            let Some((name, cluster, pool_config)) = cfg.find_pool(database) else {
89                return Err(ResolveError::UnknownDatabase {
90                    database: database.to_owned(),
91                }
92                .into());
93            };
94            (
95                name.to_owned(),
96                cluster.connect_timeout,
97                pool_config.idle_timeout,
98                pool_config.max_lifetime,
99                pool_config
100                    .checkout_timeout
101                    .unwrap_or(cfg.server.checkout_timeout),
102                pool_config.database_name(database).to_owned(),
103            )
104        };
105
106        // Look up user-specific connection parameters once for the batch.
107        // These are only used if we actually open a new connection.
108        let (upstream_user, user_params) = {
109            let user_config = cfg.find_user(database, user);
110            let upstream_user = user_config
111                .map_or(user, |u| u.upstream_name(user))
112                .to_owned();
113            let params = user_config
114                .map(|u| u.parameters.clone())
115                .unwrap_or_default();
116            (upstream_user, params)
117        };
118
119        let checkout_start = Instant::now();
120        let mut queue_start: Option<Instant> = None;
121
122        async {
123            loop {
124                // Recompute candidates each loop iteration so a
125                // topology change (e.g., primary failover) during a
126                // wait is picked up on retry.
127                let candidates = self.candidate_nodes(&cfg, database, user, routing);
128                if candidates.is_empty() {
129                    return Err(match routing {
130                        Routing::Primary => ResolveError::NoPrimary {
131                            cluster: cluster_name.clone(),
132                        },
133                        Routing::Replica => ResolveError::NoReplica {
134                            cluster: cluster_name.clone(),
135                        },
136                    }
137                    .into());
138                }
139
140                // 1. Try every candidate atomically before giving up.
141                //    `any_full` tracks whether any candidate was at its
142                //    ceiling (meaning a slot will eventually free up and
143                //    waiting is productive); `last_connect_error` keeps
144                //    the most recent connect/auth failure so we can
145                //    surface it if no candidate is worth waiting on.
146                let mut any_full = false;
147                let mut last_connect_error: Option<anyhow::Error> = None;
148                for key in &candidates {
149                    let max = self.max_for_key(&cfg, key);
150                    match self.try_checkout(key, max, idle_timeout, max_lifetime) {
151                        TryCheckout::TookIdle(idle) => {
152                            if let Some(queued_at) = queue_start {
153                                o11y::metrics::record_wait_duration(
154                                    queued_at.elapsed(),
155                                    database,
156                                    user,
157                                    routing,
158                                );
159                            }
160                            o11y::metrics::record_checkout(
161                                checkout_start,
162                                database,
163                                user,
164                                &key.node,
165                            );
166                            let span = tracing::Span::current();
167                            span.record("pool.reused", true);
168                            span.record("pool.waited", queue_start.is_some());
169                            debug!(
170                                database,
171                                user,
172                                node = %key.node,
173                                "reusing idle connection",
174                            );
175                            return Ok(ConnGuard {
176                                pools: Arc::clone(self),
177                                key: key.clone(),
178                                conn: Some(idle.conn),
179                            });
180                        }
181                        TryCheckout::MayOpen => {
182                            let password =
183                                self.lookup_password(&key.node, &upstream_database, &upstream_user);
184                            match server::connect_server(
185                                &key.node,
186                                &upstream_database,
187                                &upstream_user,
188                                password.as_deref(),
189                                &user_params,
190                                max_prepared_statements,
191                                connect_timeout,
192                            )
193                            .await
194                            {
195                                Ok(conn) => {
196                                    if let Some(queued_at) = queue_start {
197                                        o11y::metrics::record_wait_duration(
198                                            queued_at.elapsed(),
199                                            database,
200                                            user,
201                                            routing,
202                                        );
203                                    }
204                                    o11y::metrics::record_checkout(
205                                        checkout_start,
206                                        database,
207                                        user,
208                                        &key.node,
209                                    );
210                                    let span = tracing::Span::current();
211                                    span.record("pool.reused", false);
212                                    span.record("pool.waited", queue_start.is_some());
213                                    debug!(
214                                        database,
215                                        user,
216                                        node = %key.node,
217                                        "opened new server connection",
218                                    );
219                                    return Ok(ConnGuard {
220                                        pools: Arc::clone(self),
221                                        key: key.clone(),
222                                        conn: Some(conn),
223                                    });
224                                }
225                                Err(e) => {
226                                    // Connect/auth failed on this
227                                    // candidate — release the slot we
228                                    // speculatively claimed and fall
229                                    // through to the next candidate so
230                                    // a transient failure on one replica
231                                    // doesn't forfeit a healthy one
232                                    // later in the round-robin order.
233                                    self.discard_internal(key);
234                                    last_connect_error = Some(e);
235                                }
236                            }
237                        }
238                        TryCheckout::Full => {
239                            any_full = true;
240                        }
241                    }
242                }
243
244                // 2. Every candidate either refused connections or hit
245                //    its ceiling. If nothing was Full, there's no slot
246                //    to wait for — surface the connect error directly.
247                //    Otherwise fall through to the enqueue-and-wait
248                //    path so the Full candidate can wake us when a
249                //    slot frees up.
250                if !any_full && let Some(e) = last_connect_error {
251                    return Err(e);
252                }
253
254                // 3. Every candidate is at capacity. Enqueue and wait.
255                //    Record Waiting state and the target role only while
256                //    actually blocked so fast checkouts don't flicker
257                //    through Waiting in admin output.
258                let rx = {
259                    let (tx, rx) = oneshot::channel();
260                    let enqueued_at = Instant::now();
261                    let mut inner = self.inner.lock();
262                    inner
263                        .waits
264                        .entry(wait_key.clone())
265                        .or_default()
266                        .waiters
267                        .push_back((enqueued_at, tx));
268                    rx
269                };
270                client.set_waiting_for(Some(clients::WaitTarget {
271                    database: database.to_owned(),
272                    user: user.to_owned(),
273                    role: match routing {
274                        Routing::Primary => clients::WaitRole::Primary,
275                        Routing::Replica => clients::WaitRole::Replica,
276                    },
277                }));
278                client.set_state(clients::ClientState::Waiting);
279                // First enqueue stamps the queue clock; subsequent
280                // re-enqueues after a spurious wakeup keep the
281                // original value so the metric reflects total queue
282                // time, not per-wait segments.
283                queue_start.get_or_insert_with(Instant::now);
284
285                // Single timeout budget across the full checkout —
286                // spurious wakeups don't get a fresh clock. Measured
287                // from `checkout_start` so pre-enqueue candidate
288                // attempts count toward the overall deadline.
289                let elapsed = checkout_start.elapsed();
290                let remaining = checkout_timeout.saturating_sub(elapsed);
291                if remaining.is_zero() {
292                    client.set_waiting_for(None);
293                    client.set_state(clients::ClientState::Idle);
294                    anyhow::bail!(
295                        "checkout timed out after {checkout_timeout:?} for {database}/{user}",
296                    );
297                }
298                let wake = tokio::time::timeout(remaining, rx).await;
299                client.set_waiting_for(None);
300                client.set_state(clients::ClientState::Idle);
301
302                if wake.is_err() {
303                    // Outer `tokio::time::timeout` elapsed.
304                    anyhow::bail!(
305                        "checkout timed out after {checkout_timeout:?} for {database}/{user}",
306                    );
307                }
308                // Shutdown wakes everyone via `wake_all_waiters`. A
309                // waiter that was spuriously woken by an unrelated
310                // checkin right as shutdown started would also loop
311                // here, so the check happens after the wake.
312                if self.shutting_down.load(Ordering::Acquire) {
313                    anyhow::bail!("halephant is shutting down");
314                }
315                // Woken — loop top re-runs the candidate check.
316            }
317        }
318        .await
319        .inspect(|_| {
320            tracing::Span::current().record("otel.status_code", "OK");
321        })
322        .inspect_err(|e| {
323            // Classify the failure so `db.client.connection.errors` time
324            // series break down cleanly in dashboards.
325            let msg = e.to_string();
326            let error_type = if let Some(resolve_err) = e.downcast_ref::<ResolveError>() {
327                match resolve_err {
328                    ResolveError::UnknownDatabase { .. } => "unknown_database",
329                    ResolveError::NoPrimary { .. } => "no_primary",
330                    ResolveError::NoReplica { .. } => "no_replica",
331                }
332            } else if msg.contains("shutting down") {
333                "shutting_down"
334            } else if msg.contains("checkout timed out") {
335                "checkout_timeout"
336            } else {
337                "connect_failed"
338            };
339            o11y::metrics::record_checkout_error(database, user, error_type);
340            let span = tracing::Span::current();
341            span.record("otel.status_code", "ERROR");
342            span.record("otel.status_description", "checkout failed");
343        })
344    }
345
346    /// Atomic check-and-claim for a single pool key.
347    ///
348    /// Discards expired idle connections inline (idle_timeout /
349    /// max_lifetime). Increments `pool.active` on success so a concurrent
350    /// checkout can't double-claim the same slot. Holds the inner lock
351    /// for the duration of one HashMap lookup + VecDeque pop — no
352    /// `.await` while locked.
353    fn try_checkout(
354        &self,
355        key: &PoolKey,
356        max: u32,
357        idle_timeout: std::time::Duration,
358        max_lifetime: std::time::Duration,
359    ) -> TryCheckout {
360        let mut inner = self.inner.lock();
361        let pool = inner.pools.entry(key.clone()).or_default();
362
363        // Drain expired idle entries until we find a usable one or the
364        // queue is empty. Each iteration drops the front entry.
365        while let Some(idle) = pool.idle.pop_front() {
366            if idle.idle_since.elapsed() > idle_timeout {
367                trace!(
368                    database = %key.database,
369                    user = %key.user,
370                    "discarding idle connection (idle timeout)",
371                );
372                continue;
373            }
374            if idle.conn.created_at.elapsed() > max_lifetime {
375                trace!(
376                    database = %key.database,
377                    user = %key.user,
378                    "discarding idle connection (max lifetime)",
379                );
380                continue;
381            }
382            pool.active += 1;
383            return TryCheckout::TookIdle(Box::new(idle));
384        }
385
386        if max == 0 || pool.total() < max {
387            pool.active += 1;
388            TryCheckout::MayOpen
389        } else {
390            TryCheckout::Full
391        }
392    }
393
394    /// Release a slot that was speculatively claimed by a failed
395    /// connect attempt or a dropped [`ConnGuard`]. Wakes a waiter if
396    /// there is one — the just-freed slot is a retry opportunity.
397    pub(in crate::pool) fn discard_internal(&self, key: &PoolKey) {
398        {
399            let mut inner = self.inner.lock();
400            if let Some(pool) = inner.pools.get_mut(key) {
401                pool.active = pool.active.saturating_sub(1);
402                trace!(
403                    node = key.node,
404                    database = key.database,
405                    user = key.user,
406                    "connection discarded"
407                );
408            }
409        }
410        // Freeing an active slot may allow a waiter to grow the pool.
411        self.wake_one_for_node(key);
412    }
413}