Skip to main content

halephant_core/pool/
checkout.rs

1//! Connection checkout — the hot path through the pool.
2
3use std::sync::Arc;
4use std::sync::atomic::Ordering;
5use std::time::{Duration, Instant};
6
7use tokio::sync::oneshot;
8use tracing::debug;
9
10use crate::clients;
11use crate::connections::server;
12use crate::errors::ResolveError;
13use crate::o11y;
14use crate::pool::PoolManager;
15use crate::pool::types::{CandidateResult, ConnGuard, ConnId, PoolKey, Routing, WaitKey, Waiter};
16
17impl PoolManager {
18    #[tracing::instrument(name = "pool.checkout", skip_all, err(Display), fields(
19        db.namespace = %database,
20        user = %user,
21        halephant.client.id = client.id().as_u64(),
22        read_only,
23        pool.reused,
24        pool.waited,
25        otel.status_code,
26        otel.status_description,
27    ))]
28    pub async fn checkout(
29        self: &Arc<Self>,
30        client: &clients::ClientGuard,
31        database: &str,
32        user: &str,
33        read_only: bool,
34    ) -> anyhow::Result<ConnGuard> {
35        let routing = if read_only {
36            Routing::Replica
37        } else {
38            Routing::Primary
39        };
40        let wait_key = WaitKey {
41            database: database.to_owned(),
42            role: routing,
43        };
44
45        let cfg = self.config.load_full();
46        let max_prepared_statements = cfg.server.max_prepared_statements;
47
48        let (
49            cluster_name,
50            connect_timeout,
51            idle_timeout,
52            max_lifetime,
53            checkout_timeout,
54            upstream_database,
55        ) = {
56            let Some((name, cluster, pool_config)) = cfg.find_pool(database) else {
57                return Err(ResolveError::UnknownDatabase {
58                    database: database.to_owned(),
59                }
60                .into());
61            };
62            (
63                name.to_owned(),
64                cluster.connect_timeout,
65                pool_config.idle_timeout,
66                pool_config.max_lifetime,
67                pool_config
68                    .checkout_timeout
69                    .unwrap_or(cfg.server.checkout_timeout),
70                pool_config.database_name(database).to_owned(),
71            )
72        };
73
74        let (upstream_user, user_params) = {
75            let user_config = cfg.find_user(database, user);
76            let upstream_user = user_config
77                .map_or(user, |u| u.upstream_name(user))
78                .to_owned();
79            let params = user_config
80                .map(|u| u.parameters.clone())
81                .unwrap_or_default();
82            (upstream_user, params)
83        };
84
85        let checkout_start = Instant::now();
86        let mut queue_start: Option<Instant> = None;
87
88        async {
89            loop {
90                // Build candidate lists: primary target first, then
91                // primary as fallback for read-only traffic.
92                let candidates = self.resolve_candidates(&cfg, database, user, routing);
93                let fallback = if routing == Routing::Replica {
94                    self.resolve_candidates(&cfg, database, user, Routing::Primary)
95                } else {
96                    Vec::new()
97                };
98                if candidates.is_empty() && fallback.is_empty() {
99                    return Err(match routing {
100                        Routing::Primary => ResolveError::NoPrimary {
101                            cluster: cluster_name.clone(),
102                        },
103                        Routing::Replica => ResolveError::NoReplica {
104                            cluster: cluster_name.clone(),
105                        },
106                    }
107                    .into());
108                }
109
110                // Try preferred candidates, then fallback.
111                let mut got = CandidateResult::AllFull;
112                for batch in [&candidates, &fallback] {
113                    if batch.is_empty() {
114                        continue;
115                    }
116                    got = self.try_candidates(&cfg, batch, idle_timeout, max_lifetime);
117                    if !matches!(got, CandidateResult::AllFull) {
118                        break;
119                    }
120                }
121
122                match got {
123                    CandidateResult::GotIdle { key, conn } => {
124                        record_checkout_success(
125                            &key,
126                            checkout_start,
127                            queue_start,
128                            database,
129                            user,
130                            routing,
131                            true,
132                        );
133                        return Ok(ConnGuard {
134                            pools: Arc::clone(self),
135                            key,
136                            id: conn.id,
137                            conn: Some(conn.conn),
138                        });
139                    }
140                    CandidateResult::Claimed { key, id } => {
141                        let password =
142                            self.lookup_password(&key.node, &upstream_database, &upstream_user);
143                        match server::connect_server(
144                            &key.node,
145                            &upstream_database,
146                            &upstream_user,
147                            password.as_deref(),
148                            &user_params,
149                            max_prepared_statements,
150                            connect_timeout,
151                        )
152                        .await
153                        {
154                            Ok(conn) => {
155                                record_checkout_success(
156                                    &key,
157                                    checkout_start,
158                                    queue_start,
159                                    database,
160                                    user,
161                                    routing,
162                                    false,
163                                );
164                                return Ok(ConnGuard {
165                                    pools: Arc::clone(self),
166                                    key,
167                                    id,
168                                    conn: Some(conn),
169                                });
170                            }
171                            Err(e) => {
172                                self.discard_internal(&key, id);
173                                let total = candidates.len() + fallback.len();
174                                if total == 1 {
175                                    return Err(e);
176                                }
177                            }
178                        }
179                    }
180                    CandidateResult::AllFull => {}
181                }
182
183                let rx = {
184                    let (tx, rx) = oneshot::channel();
185                    let mut inner = self.inner.lock();
186                    inner
187                        .waits
188                        .entry(wait_key.clone())
189                        .or_default()
190                        .waiters
191                        .push_back(Waiter {
192                            enqueued_at: Instant::now(),
193                            user: user.to_owned(),
194                            tx,
195                        });
196                    rx
197                };
198                client.set_waiting_for(Some(clients::WaitTarget {
199                    database: database.to_owned(),
200                    user: user.to_owned(),
201                    role: match routing {
202                        Routing::Primary => clients::WaitRole::Primary,
203                        Routing::Replica => clients::WaitRole::Replica,
204                    },
205                }));
206                client.set_state(clients::ClientState::Waiting);
207                queue_start.get_or_insert_with(Instant::now);
208
209                let remaining = checkout_timeout.saturating_sub(checkout_start.elapsed());
210                if remaining.is_zero() {
211                    client.set_waiting_for(None);
212                    client.set_state(clients::ClientState::Idle);
213                    anyhow::bail!(
214                        "checkout timed out after {checkout_timeout:?} for {database}/{user}",
215                    );
216                }
217                let wake = tokio::time::timeout(remaining, rx).await;
218                client.set_waiting_for(None);
219                client.set_state(clients::ClientState::Idle);
220
221                if wake.is_err() {
222                    anyhow::bail!(
223                        "checkout timed out after {checkout_timeout:?} for {database}/{user}",
224                    );
225                }
226                if self.shutting_down.load(Ordering::Acquire) {
227                    anyhow::bail!("halephant is shutting down");
228                }
229            }
230        }
231        .await
232        .inspect(|_| {
233            tracing::Span::current().record("otel.status_code", "OK");
234        })
235        .inspect_err(|e| {
236            let msg = e.to_string();
237            let error_type = if let Some(resolve_err) = e.downcast_ref::<ResolveError>() {
238                match resolve_err {
239                    ResolveError::UnknownDatabase { .. } => "unknown_database",
240                    ResolveError::NoPrimary { .. } => "no_primary",
241                    ResolveError::NoReplica { .. } => "no_replica",
242                }
243            } else if msg.contains("shutting down") {
244                "shutting_down"
245            } else if msg.contains("checkout timed out") {
246                "checkout_timeout"
247            } else {
248                "connect_failed"
249            };
250            o11y::metrics::record_checkout_error(database, user, error_type);
251            let span = tracing::Span::current();
252            span.record("otel.status_code", "ERROR");
253            span.record("otel.status_description", "checkout failed");
254        })
255    }
256
257    // -----------------------------------------------------------------------
258    // Internals
259    // -----------------------------------------------------------------------
260
261    fn resolve_candidates(
262        &self,
263        cfg: &crate::config::Config,
264        database: &str,
265        user: &str,
266        routing: Routing,
267    ) -> Vec<PoolKey> {
268        let Some((cluster_name, _, _)) = cfg.find_pool(database) else {
269            return Vec::new();
270        };
271        match routing {
272            Routing::Primary => {
273                let Some(primary) = self.topology.primary(cluster_name) else {
274                    return Vec::new();
275                };
276                vec![PoolKey {
277                    node: primary,
278                    database: database.to_owned(),
279                    user: user.to_owned(),
280                }]
281            }
282            Routing::Replica => self
283                .topology
284                .replicas(cluster_name)
285                .iter()
286                .map(|addr| PoolKey {
287                    node: addr.clone(),
288                    database: database.to_owned(),
289                    user: user.to_owned(),
290                })
291                .collect(),
292        }
293    }
294
295    fn try_candidates(
296        &self,
297        cfg: &crate::config::Config,
298        candidates: &[PoolKey],
299        idle_timeout: Duration,
300        max_lifetime: Duration,
301    ) -> CandidateResult {
302        let mut inner = self.inner.lock();
303
304        // Rank by active count (least-connections). Rotate for tie-break.
305        let mut ranked: Vec<&PoolKey> = candidates.iter().collect();
306        if ranked.len() > 1 {
307            let start = self.replica_rr.fetch_add(1, Ordering::Relaxed) % ranked.len();
308            ranked.rotate_left(start);
309            ranked.sort_by_key(|k| inner.pools.get(*k).map_or(0, |p| p.active.len()));
310        }
311
312        for key in ranked {
313            let limits = self.resolve_limits(cfg, key);
314
315            // Try to reuse an idle connection. Scoped so the mutable
316            // pool borrow drops before the node_total scan.
317            let idle = {
318                let pool = inner.pools.entry(key.clone()).or_default();
319                let idle = pool.pop_idle(idle_timeout, max_lifetime);
320                if let Some(ref c) = idle {
321                    pool.active.insert(c.id);
322                }
323                idle
324            };
325            if let Some(idle) = idle {
326                return CandidateResult::GotIdle {
327                    key: key.clone(),
328                    conn: Box::new(idle),
329                };
330            }
331
332            let user_total = inner.pools.get(key).map_or(0, super::types::Pool::total);
333            if !limits.user_has_capacity(user_total) {
334                continue;
335            }
336
337            let node_total = inner.node_total(&key.node, &key.database);
338            if !limits.pool_has_capacity(node_total) && !Self::evict_idle_for_node(&mut inner, key)
339            {
340                continue;
341            }
342
343            let id = ConnId::next();
344            inner
345                .pools
346                .entry(key.clone())
347                .or_default()
348                .active
349                .insert(id);
350            return CandidateResult::Claimed {
351                key: key.clone(),
352                id,
353            };
354        }
355
356        CandidateResult::AllFull
357    }
358
359    fn evict_idle_for_node(inner: &mut super::types::Inner, key: &PoolKey) -> bool {
360        let victim = inner
361            .pools
362            .iter()
363            .filter(|(k, p)| k.node == key.node && k.database == key.database && !p.idle.is_empty())
364            .min_by_key(|(_, p)| p.idle.front().map_or(Instant::now(), |i| i.idle_since))
365            .map(|(k, _)| k.clone());
366        if let Some(ref vk) = victim
367            && let Some(pool) = inner.pools.get_mut(vk)
368        {
369            pool.idle.pop_front();
370        }
371        victim.is_some()
372    }
373
374    pub(in crate::pool) fn discard_internal(&self, key: &PoolKey, id: ConnId) {
375        {
376            let mut inner = self.inner.lock();
377            if let Some(pool) = inner.pools.get_mut(key) {
378                pool.active.remove(&id);
379            }
380        }
381        self.wake_one_for_node(key);
382    }
383}
384
385fn record_checkout_success(
386    key: &PoolKey,
387    checkout_start: Instant,
388    queue_start: Option<Instant>,
389    database: &str,
390    user: &str,
391    routing: Routing,
392    reused: bool,
393) {
394    if let Some(queued_at) = queue_start {
395        o11y::metrics::record_wait_duration(queued_at.elapsed(), database, user, routing);
396    }
397    o11y::metrics::record_checkout(checkout_start, database, user, &key.node);
398    let span = tracing::Span::current();
399    span.record("pool.reused", reused);
400    span.record("pool.waited", queue_start.is_some());
401    debug!(database, user, node = %key.node, reused, "checkout complete");
402}