Skip to main content

halephant_core/pool/
checkout.rs

1//! Connection checkout — the hot path through the pool.
2
3use std::sync::Arc;
4use std::sync::atomic::Ordering;
5use std::time::{Duration, Instant};
6
7use tokio::sync::oneshot;
8use tracing::debug;
9
10use crate::clients;
11use crate::connections::server;
12use crate::errors::ResolveError;
13use crate::o11y;
14use crate::pool::PoolManager;
15use crate::pool::types::{CandidateResult, ConnGuard, ConnId, PoolKey, Routing, WaitKey, Waiter};
16
17impl PoolManager {
18    #[tracing::instrument(name = "pool.checkout", skip_all, err(Display), fields(
19        db.namespace = %database,
20        user = %user,
21        halephant.client.id = client.id().as_u64(),
22        read_only,
23        pool.reused,
24        pool.waited,
25        otel.status_code,
26        otel.status_description,
27    ))]
28    pub async fn checkout(
29        self: &Arc<Self>,
30        client: &clients::ClientGuard,
31        database: &str,
32        user: &str,
33        read_only: bool,
34    ) -> anyhow::Result<ConnGuard> {
35        let routing = if read_only {
36            Routing::Replica
37        } else {
38            Routing::Primary
39        };
40        let wait_key = WaitKey {
41            database: database.to_owned(),
42            role: routing,
43        };
44
45        let cfg = self.config.load_full();
46        let max_prepared_statements = cfg.server.max_prepared_statements;
47
48        let (
49            cluster_name,
50            connect_timeout,
51            idle_timeout,
52            max_lifetime,
53            checkout_timeout,
54            upstream_database,
55        ) = {
56            let Some((name, cluster, pool_config)) = cfg.find_pool(database) else {
57                return Err(ResolveError::UnknownDatabase {
58                    database: database.to_owned(),
59                }
60                .into());
61            };
62            (
63                name.to_owned(),
64                cluster.connect_timeout,
65                pool_config.idle_timeout,
66                pool_config.max_lifetime,
67                pool_config
68                    .checkout_timeout
69                    .unwrap_or(cfg.server.checkout_timeout),
70                pool_config.database_name(database).to_owned(),
71            )
72        };
73
74        let (upstream_user, user_params) = {
75            let user_config = cfg.find_user(database, user);
76            let upstream_user = user_config
77                .map_or(user, |u| u.upstream_name(user))
78                .to_owned();
79            let params = user_config
80                .map(|u| u.parameters.clone())
81                .unwrap_or_default();
82            (upstream_user, params)
83        };
84
85        let checkout_start = Instant::now();
86        let mut queue_start: Option<Instant> = None;
87
88        async {
89            loop {
90                let candidates = self.resolve_candidates(&cfg, database, user, routing);
91                if candidates.is_empty() {
92                    return Err(match routing {
93                        Routing::Primary => ResolveError::NoPrimary {
94                            cluster: cluster_name.clone(),
95                        },
96                        Routing::Replica => ResolveError::NoReplica {
97                            cluster: cluster_name.clone(),
98                        },
99                    }
100                    .into());
101                }
102
103                match self.try_candidates(&cfg, &candidates, idle_timeout, max_lifetime) {
104                    CandidateResult::GotIdle { key, conn } => {
105                        record_checkout_success(
106                            &key,
107                            checkout_start,
108                            queue_start,
109                            database,
110                            user,
111                            routing,
112                            true,
113                        );
114                        return Ok(ConnGuard {
115                            pools: Arc::clone(self),
116                            key,
117                            id: conn.id,
118                            conn: Some(conn.conn),
119                        });
120                    }
121                    CandidateResult::Claimed { key, id } => {
122                        let password =
123                            self.lookup_password(&key.node, &upstream_database, &upstream_user);
124                        match server::connect_server(
125                            &key.node,
126                            &upstream_database,
127                            &upstream_user,
128                            password.as_deref(),
129                            &user_params,
130                            max_prepared_statements,
131                            connect_timeout,
132                        )
133                        .await
134                        {
135                            Ok(conn) => {
136                                record_checkout_success(
137                                    &key,
138                                    checkout_start,
139                                    queue_start,
140                                    database,
141                                    user,
142                                    routing,
143                                    false,
144                                );
145                                return Ok(ConnGuard {
146                                    pools: Arc::clone(self),
147                                    key,
148                                    id,
149                                    conn: Some(conn),
150                                });
151                            }
152                            Err(e) => {
153                                self.discard_internal(&key, id);
154                                if candidates.len() == 1 {
155                                    return Err(e);
156                                }
157                            }
158                        }
159                    }
160                    CandidateResult::AllFull => {}
161                }
162
163                let rx = {
164                    let (tx, rx) = oneshot::channel();
165                    let mut inner = self.inner.lock();
166                    inner
167                        .waits
168                        .entry(wait_key.clone())
169                        .or_default()
170                        .waiters
171                        .push_back(Waiter {
172                            enqueued_at: Instant::now(),
173                            user: user.to_owned(),
174                            tx,
175                        });
176                    rx
177                };
178                client.set_waiting_for(Some(clients::WaitTarget {
179                    database: database.to_owned(),
180                    user: user.to_owned(),
181                    role: match routing {
182                        Routing::Primary => clients::WaitRole::Primary,
183                        Routing::Replica => clients::WaitRole::Replica,
184                    },
185                }));
186                client.set_state(clients::ClientState::Waiting);
187                queue_start.get_or_insert_with(Instant::now);
188
189                let remaining = checkout_timeout.saturating_sub(checkout_start.elapsed());
190                if remaining.is_zero() {
191                    client.set_waiting_for(None);
192                    client.set_state(clients::ClientState::Idle);
193                    anyhow::bail!(
194                        "checkout timed out after {checkout_timeout:?} for {database}/{user}",
195                    );
196                }
197                let wake = tokio::time::timeout(remaining, rx).await;
198                client.set_waiting_for(None);
199                client.set_state(clients::ClientState::Idle);
200
201                if wake.is_err() {
202                    anyhow::bail!(
203                        "checkout timed out after {checkout_timeout:?} for {database}/{user}",
204                    );
205                }
206                if self.shutting_down.load(Ordering::Acquire) {
207                    anyhow::bail!("halephant is shutting down");
208                }
209            }
210        }
211        .await
212        .inspect(|_| {
213            tracing::Span::current().record("otel.status_code", "OK");
214        })
215        .inspect_err(|e| {
216            let msg = e.to_string();
217            let error_type = if let Some(resolve_err) = e.downcast_ref::<ResolveError>() {
218                match resolve_err {
219                    ResolveError::UnknownDatabase { .. } => "unknown_database",
220                    ResolveError::NoPrimary { .. } => "no_primary",
221                    ResolveError::NoReplica { .. } => "no_replica",
222                }
223            } else if msg.contains("shutting down") {
224                "shutting_down"
225            } else if msg.contains("checkout timed out") {
226                "checkout_timeout"
227            } else {
228                "connect_failed"
229            };
230            o11y::metrics::record_checkout_error(database, user, error_type);
231            let span = tracing::Span::current();
232            span.record("otel.status_code", "ERROR");
233            span.record("otel.status_description", "checkout failed");
234        })
235    }
236
237    // -----------------------------------------------------------------------
238    // Internals
239    // -----------------------------------------------------------------------
240
241    fn resolve_candidates(
242        &self,
243        cfg: &crate::config::Config,
244        database: &str,
245        user: &str,
246        routing: Routing,
247    ) -> Vec<PoolKey> {
248        let Some((cluster_name, _, _)) = cfg.find_pool(database) else {
249            return Vec::new();
250        };
251        match routing {
252            Routing::Primary => {
253                let Some(primary) = self.topology.primary(cluster_name) else {
254                    return Vec::new();
255                };
256                vec![PoolKey {
257                    node: primary,
258                    database: database.to_owned(),
259                    user: user.to_owned(),
260                }]
261            }
262            Routing::Replica => self
263                .topology
264                .replicas(cluster_name)
265                .iter()
266                .map(|addr| PoolKey {
267                    node: addr.clone(),
268                    database: database.to_owned(),
269                    user: user.to_owned(),
270                })
271                .collect(),
272        }
273    }
274
275    fn try_candidates(
276        &self,
277        cfg: &crate::config::Config,
278        candidates: &[PoolKey],
279        idle_timeout: Duration,
280        max_lifetime: Duration,
281    ) -> CandidateResult {
282        let mut inner = self.inner.lock();
283
284        // Rank by active count (least-connections). Rotate for tie-break.
285        let mut ranked: Vec<&PoolKey> = candidates.iter().collect();
286        if ranked.len() > 1 {
287            let start = self.replica_rr.fetch_add(1, Ordering::Relaxed) % ranked.len();
288            ranked.rotate_left(start);
289            ranked.sort_by_key(|k| inner.pools.get(*k).map_or(0, |p| p.active.len()));
290        }
291
292        for key in ranked {
293            let limits = self.resolve_limits(cfg, key);
294
295            // Try to reuse an idle connection. Scoped so the mutable
296            // pool borrow drops before the node_total scan.
297            let idle = {
298                let pool = inner.pools.entry(key.clone()).or_default();
299                let idle = pool.pop_idle(idle_timeout, max_lifetime);
300                if let Some(ref c) = idle {
301                    pool.active.insert(c.id);
302                }
303                idle
304            };
305            if let Some(idle) = idle {
306                return CandidateResult::GotIdle {
307                    key: key.clone(),
308                    conn: Box::new(idle),
309                };
310            }
311
312            let user_total = inner.pools.get(key).map_or(0, super::types::Pool::total);
313            if !limits.user_has_capacity(user_total) {
314                continue;
315            }
316
317            let node_total = inner.node_total(&key.node, &key.database);
318            if !limits.pool_has_capacity(node_total) && !Self::evict_idle_for_node(&mut inner, key)
319            {
320                continue;
321            }
322
323            let id = ConnId::next();
324            inner
325                .pools
326                .entry(key.clone())
327                .or_default()
328                .active
329                .insert(id);
330            return CandidateResult::Claimed {
331                key: key.clone(),
332                id,
333            };
334        }
335
336        CandidateResult::AllFull
337    }
338
339    fn evict_idle_for_node(inner: &mut super::types::Inner, key: &PoolKey) -> bool {
340        let victim = inner
341            .pools
342            .iter()
343            .filter(|(k, p)| k.node == key.node && k.database == key.database && !p.idle.is_empty())
344            .min_by_key(|(_, p)| p.idle.front().map_or(Instant::now(), |i| i.idle_since))
345            .map(|(k, _)| k.clone());
346        if let Some(ref vk) = victim
347            && let Some(pool) = inner.pools.get_mut(vk)
348        {
349            pool.idle.pop_front();
350        }
351        victim.is_some()
352    }
353
354    pub(in crate::pool) fn discard_internal(&self, key: &PoolKey, id: ConnId) {
355        {
356            let mut inner = self.inner.lock();
357            if let Some(pool) = inner.pools.get_mut(key) {
358                pool.active.remove(&id);
359            }
360        }
361        self.wake_one_for_node(key);
362    }
363}
364
365fn record_checkout_success(
366    key: &PoolKey,
367    checkout_start: Instant,
368    queue_start: Option<Instant>,
369    database: &str,
370    user: &str,
371    routing: Routing,
372    reused: bool,
373) {
374    if let Some(queued_at) = queue_start {
375        o11y::metrics::record_wait_duration(queued_at.elapsed(), database, user, routing);
376    }
377    o11y::metrics::record_checkout(checkout_start, database, user, &key.node);
378    let span = tracing::Span::current();
379    span.record("pool.reused", reused);
380    span.record("pool.waited", queue_start.is_some());
381    debug!(database, user, node = %key.node, reused, "checkout complete");
382}