1use std::sync::Arc;
4use std::sync::atomic::Ordering;
5use std::time::{Duration, Instant};
6
7use tokio::sync::oneshot;
8use tracing::debug;
9
10use crate::clients;
11use crate::connections::server;
12use crate::errors::ResolveError;
13use crate::o11y;
14use crate::pool::PoolManager;
15use crate::pool::types::{CandidateResult, ConnGuard, ConnId, PoolKey, Routing, WaitKey, Waiter};
16
17impl PoolManager {
18 #[tracing::instrument(name = "pool.checkout", skip_all, err(Display), fields(
19 db.namespace = %database,
20 user = %user,
21 halephant.client.id = client.id().as_u64(),
22 read_only,
23 pool.reused,
24 pool.waited,
25 otel.status_code,
26 otel.status_description,
27 ))]
28 pub async fn checkout(
29 self: &Arc<Self>,
30 client: &clients::ClientGuard,
31 database: &str,
32 user: &str,
33 read_only: bool,
34 ) -> anyhow::Result<ConnGuard> {
35 let routing = if read_only {
36 Routing::Replica
37 } else {
38 Routing::Primary
39 };
40 let wait_key = WaitKey {
41 database: database.to_owned(),
42 role: routing,
43 };
44
45 let cfg = self.config.load_full();
46 let max_prepared_statements = cfg.server.max_prepared_statements;
47
48 let (
49 cluster_name,
50 connect_timeout,
51 idle_timeout,
52 max_lifetime,
53 checkout_timeout,
54 upstream_database,
55 ) = {
56 let Some((name, cluster, pool_config)) = cfg.find_pool(database) else {
57 return Err(ResolveError::UnknownDatabase {
58 database: database.to_owned(),
59 }
60 .into());
61 };
62 (
63 name.to_owned(),
64 cluster.connect_timeout,
65 pool_config.idle_timeout,
66 pool_config.max_lifetime,
67 pool_config
68 .checkout_timeout
69 .unwrap_or(cfg.server.checkout_timeout),
70 pool_config.database_name(database).to_owned(),
71 )
72 };
73
74 let (upstream_user, user_params) = {
75 let user_config = cfg.find_user(database, user);
76 let upstream_user = user_config
77 .map_or(user, |u| u.upstream_name(user))
78 .to_owned();
79 let params = user_config
80 .map(|u| u.parameters.clone())
81 .unwrap_or_default();
82 (upstream_user, params)
83 };
84
85 let checkout_start = Instant::now();
86 let mut queue_start: Option<Instant> = None;
87
88 async {
89 loop {
90 let candidates = self.resolve_candidates(&cfg, database, user, routing);
91 if candidates.is_empty() {
92 return Err(match routing {
93 Routing::Primary => ResolveError::NoPrimary {
94 cluster: cluster_name.clone(),
95 },
96 Routing::Replica => ResolveError::NoReplica {
97 cluster: cluster_name.clone(),
98 },
99 }
100 .into());
101 }
102
103 match self.try_candidates(&cfg, &candidates, idle_timeout, max_lifetime) {
104 CandidateResult::GotIdle { key, conn } => {
105 record_checkout_success(
106 &key,
107 checkout_start,
108 queue_start,
109 database,
110 user,
111 routing,
112 true,
113 );
114 return Ok(ConnGuard {
115 pools: Arc::clone(self),
116 key,
117 id: conn.id,
118 conn: Some(conn.conn),
119 });
120 }
121 CandidateResult::Claimed { key, id } => {
122 let password =
123 self.lookup_password(&key.node, &upstream_database, &upstream_user);
124 match server::connect_server(
125 &key.node,
126 &upstream_database,
127 &upstream_user,
128 password.as_deref(),
129 &user_params,
130 max_prepared_statements,
131 connect_timeout,
132 )
133 .await
134 {
135 Ok(conn) => {
136 record_checkout_success(
137 &key,
138 checkout_start,
139 queue_start,
140 database,
141 user,
142 routing,
143 false,
144 );
145 return Ok(ConnGuard {
146 pools: Arc::clone(self),
147 key,
148 id,
149 conn: Some(conn),
150 });
151 }
152 Err(e) => {
153 self.discard_internal(&key, id);
154 if candidates.len() == 1 {
155 return Err(e);
156 }
157 }
158 }
159 }
160 CandidateResult::AllFull => {}
161 }
162
163 let rx = {
164 let (tx, rx) = oneshot::channel();
165 let mut inner = self.inner.lock();
166 inner
167 .waits
168 .entry(wait_key.clone())
169 .or_default()
170 .waiters
171 .push_back(Waiter {
172 enqueued_at: Instant::now(),
173 user: user.to_owned(),
174 tx,
175 });
176 rx
177 };
178 client.set_waiting_for(Some(clients::WaitTarget {
179 database: database.to_owned(),
180 user: user.to_owned(),
181 role: match routing {
182 Routing::Primary => clients::WaitRole::Primary,
183 Routing::Replica => clients::WaitRole::Replica,
184 },
185 }));
186 client.set_state(clients::ClientState::Waiting);
187 queue_start.get_or_insert_with(Instant::now);
188
189 let remaining = checkout_timeout.saturating_sub(checkout_start.elapsed());
190 if remaining.is_zero() {
191 client.set_waiting_for(None);
192 client.set_state(clients::ClientState::Idle);
193 anyhow::bail!(
194 "checkout timed out after {checkout_timeout:?} for {database}/{user}",
195 );
196 }
197 let wake = tokio::time::timeout(remaining, rx).await;
198 client.set_waiting_for(None);
199 client.set_state(clients::ClientState::Idle);
200
201 if wake.is_err() {
202 anyhow::bail!(
203 "checkout timed out after {checkout_timeout:?} for {database}/{user}",
204 );
205 }
206 if self.shutting_down.load(Ordering::Acquire) {
207 anyhow::bail!("halephant is shutting down");
208 }
209 }
210 }
211 .await
212 .inspect(|_| {
213 tracing::Span::current().record("otel.status_code", "OK");
214 })
215 .inspect_err(|e| {
216 let msg = e.to_string();
217 let error_type = if let Some(resolve_err) = e.downcast_ref::<ResolveError>() {
218 match resolve_err {
219 ResolveError::UnknownDatabase { .. } => "unknown_database",
220 ResolveError::NoPrimary { .. } => "no_primary",
221 ResolveError::NoReplica { .. } => "no_replica",
222 }
223 } else if msg.contains("shutting down") {
224 "shutting_down"
225 } else if msg.contains("checkout timed out") {
226 "checkout_timeout"
227 } else {
228 "connect_failed"
229 };
230 o11y::metrics::record_checkout_error(database, user, error_type);
231 let span = tracing::Span::current();
232 span.record("otel.status_code", "ERROR");
233 span.record("otel.status_description", "checkout failed");
234 })
235 }
236
237 fn resolve_candidates(
242 &self,
243 cfg: &crate::config::Config,
244 database: &str,
245 user: &str,
246 routing: Routing,
247 ) -> Vec<PoolKey> {
248 let Some((cluster_name, _, _)) = cfg.find_pool(database) else {
249 return Vec::new();
250 };
251 match routing {
252 Routing::Primary => {
253 let Some(primary) = self.topology.primary(cluster_name) else {
254 return Vec::new();
255 };
256 vec![PoolKey {
257 node: primary,
258 database: database.to_owned(),
259 user: user.to_owned(),
260 }]
261 }
262 Routing::Replica => self
263 .topology
264 .replicas(cluster_name)
265 .iter()
266 .map(|addr| PoolKey {
267 node: addr.clone(),
268 database: database.to_owned(),
269 user: user.to_owned(),
270 })
271 .collect(),
272 }
273 }
274
275 fn try_candidates(
276 &self,
277 cfg: &crate::config::Config,
278 candidates: &[PoolKey],
279 idle_timeout: Duration,
280 max_lifetime: Duration,
281 ) -> CandidateResult {
282 let mut inner = self.inner.lock();
283
284 let mut ranked: Vec<&PoolKey> = candidates.iter().collect();
286 if ranked.len() > 1 {
287 let start = self.replica_rr.fetch_add(1, Ordering::Relaxed) % ranked.len();
288 ranked.rotate_left(start);
289 ranked.sort_by_key(|k| inner.pools.get(*k).map_or(0, |p| p.active.len()));
290 }
291
292 for key in ranked {
293 let limits = self.resolve_limits(cfg, key);
294
295 let idle = {
298 let pool = inner.pools.entry(key.clone()).or_default();
299 let idle = pool.pop_idle(idle_timeout, max_lifetime);
300 if let Some(ref c) = idle {
301 pool.active.insert(c.id);
302 }
303 idle
304 };
305 if let Some(idle) = idle {
306 return CandidateResult::GotIdle {
307 key: key.clone(),
308 conn: Box::new(idle),
309 };
310 }
311
312 let user_total = inner.pools.get(key).map_or(0, super::types::Pool::total);
313 if !limits.user_has_capacity(user_total) {
314 continue;
315 }
316
317 let node_total = inner.node_total(&key.node, &key.database);
318 if !limits.pool_has_capacity(node_total) && !Self::evict_idle_for_node(&mut inner, key)
319 {
320 continue;
321 }
322
323 let id = ConnId::next();
324 inner
325 .pools
326 .entry(key.clone())
327 .or_default()
328 .active
329 .insert(id);
330 return CandidateResult::Claimed {
331 key: key.clone(),
332 id,
333 };
334 }
335
336 CandidateResult::AllFull
337 }
338
339 fn evict_idle_for_node(inner: &mut super::types::Inner, key: &PoolKey) -> bool {
340 let victim = inner
341 .pools
342 .iter()
343 .filter(|(k, p)| k.node == key.node && k.database == key.database && !p.idle.is_empty())
344 .min_by_key(|(_, p)| p.idle.front().map_or(Instant::now(), |i| i.idle_since))
345 .map(|(k, _)| k.clone());
346 if let Some(ref vk) = victim
347 && let Some(pool) = inner.pools.get_mut(vk)
348 {
349 pool.idle.pop_front();
350 }
351 victim.is_some()
352 }
353
354 pub(in crate::pool) fn discard_internal(&self, key: &PoolKey, id: ConnId) {
355 {
356 let mut inner = self.inner.lock();
357 if let Some(pool) = inner.pools.get_mut(key) {
358 pool.active.remove(&id);
359 }
360 }
361 self.wake_one_for_node(key);
362 }
363}
364
365fn record_checkout_success(
366 key: &PoolKey,
367 checkout_start: Instant,
368 queue_start: Option<Instant>,
369 database: &str,
370 user: &str,
371 routing: Routing,
372 reused: bool,
373) {
374 if let Some(queued_at) = queue_start {
375 o11y::metrics::record_wait_duration(queued_at.elapsed(), database, user, routing);
376 }
377 o11y::metrics::record_checkout(checkout_start, database, user, &key.node);
378 let span = tracing::Span::current();
379 span.record("pool.reused", reused);
380 span.record("pool.waited", queue_start.is_some());
381 debug!(database, user, node = %key.node, reused, "checkout complete");
382}