1use std::sync::Arc;
4use std::sync::atomic::Ordering;
5use std::time::{Duration, Instant};
6
7use tokio::sync::oneshot;
8use tracing::debug;
9
10use crate::clients;
11use crate::connections::server;
12use crate::errors::ResolveError;
13use crate::o11y;
14use crate::pool::PoolManager;
15use crate::pool::types::{CandidateResult, ConnGuard, ConnId, PoolKey, Routing, WaitKey, Waiter};
16
17impl PoolManager {
18 #[tracing::instrument(name = "pool.checkout", skip_all, err(Display), fields(
19 db.namespace = %database,
20 user = %user,
21 halephant.client.id = client.id().as_u64(),
22 read_only,
23 pool.reused,
24 pool.waited,
25 otel.status_code,
26 otel.status_description,
27 ))]
28 pub async fn checkout(
29 self: &Arc<Self>,
30 client: &clients::ClientGuard,
31 database: &str,
32 user: &str,
33 read_only: bool,
34 ) -> anyhow::Result<ConnGuard> {
35 let routing = if read_only {
36 Routing::Replica
37 } else {
38 Routing::Primary
39 };
40 let wait_key = WaitKey {
41 database: database.to_owned(),
42 role: routing,
43 };
44
45 let cfg = self.config.load_full();
46 let max_prepared_statements = cfg.server.max_prepared_statements;
47
48 let (
49 cluster_name,
50 connect_timeout,
51 idle_timeout,
52 max_lifetime,
53 checkout_timeout,
54 upstream_database,
55 ) = {
56 let Some((name, cluster, pool_config)) = cfg.find_pool(database) else {
57 return Err(ResolveError::UnknownDatabase {
58 database: database.to_owned(),
59 }
60 .into());
61 };
62 (
63 name.to_owned(),
64 cluster.connect_timeout,
65 pool_config.idle_timeout,
66 pool_config.max_lifetime,
67 pool_config
68 .checkout_timeout
69 .unwrap_or(cfg.server.checkout_timeout),
70 pool_config.database_name(database).to_owned(),
71 )
72 };
73
74 let (upstream_user, user_params) = {
75 let user_config = cfg.find_user(database, user);
76 let upstream_user = user_config
77 .map_or(user, |u| u.upstream_name(user))
78 .to_owned();
79 let params = user_config
80 .map(|u| u.parameters.clone())
81 .unwrap_or_default();
82 (upstream_user, params)
83 };
84
85 let checkout_start = Instant::now();
86 let mut queue_start: Option<Instant> = None;
87
88 async {
89 loop {
90 let candidates = self.resolve_candidates(&cfg, database, user, routing);
93 let fallback = if routing == Routing::Replica {
94 self.resolve_candidates(&cfg, database, user, Routing::Primary)
95 } else {
96 Vec::new()
97 };
98 if candidates.is_empty() && fallback.is_empty() {
99 return Err(match routing {
100 Routing::Primary => ResolveError::NoPrimary {
101 cluster: cluster_name.clone(),
102 },
103 Routing::Replica => ResolveError::NoReplica {
104 cluster: cluster_name.clone(),
105 },
106 }
107 .into());
108 }
109
110 let mut got = CandidateResult::AllFull;
112 for batch in [&candidates, &fallback] {
113 if batch.is_empty() {
114 continue;
115 }
116 got = self.try_candidates(&cfg, batch, idle_timeout, max_lifetime);
117 if !matches!(got, CandidateResult::AllFull) {
118 break;
119 }
120 }
121
122 match got {
123 CandidateResult::GotIdle { key, conn } => {
124 record_checkout_success(
125 &key,
126 checkout_start,
127 queue_start,
128 database,
129 user,
130 routing,
131 true,
132 );
133 return Ok(ConnGuard {
134 pools: Arc::clone(self),
135 key,
136 id: conn.id,
137 conn: Some(conn.conn),
138 });
139 }
140 CandidateResult::Claimed { key, id } => {
141 let password =
142 self.lookup_password(&key.node, &upstream_database, &upstream_user);
143 match server::connect_server(
144 &key.node,
145 &upstream_database,
146 &upstream_user,
147 password.as_deref(),
148 &user_params,
149 max_prepared_statements,
150 connect_timeout,
151 )
152 .await
153 {
154 Ok(conn) => {
155 record_checkout_success(
156 &key,
157 checkout_start,
158 queue_start,
159 database,
160 user,
161 routing,
162 false,
163 );
164 return Ok(ConnGuard {
165 pools: Arc::clone(self),
166 key,
167 id,
168 conn: Some(conn),
169 });
170 }
171 Err(e) => {
172 self.discard_internal(&key, id);
173 let total = candidates.len() + fallback.len();
174 if total == 1 {
175 return Err(e);
176 }
177 }
178 }
179 }
180 CandidateResult::AllFull => {}
181 }
182
183 let rx = {
184 let (tx, rx) = oneshot::channel();
185 let mut inner = self.inner.lock();
186 inner
187 .waits
188 .entry(wait_key.clone())
189 .or_default()
190 .waiters
191 .push_back(Waiter {
192 enqueued_at: Instant::now(),
193 user: user.to_owned(),
194 tx,
195 });
196 rx
197 };
198 client.set_waiting_for(Some(clients::WaitTarget {
199 database: database.to_owned(),
200 user: user.to_owned(),
201 role: match routing {
202 Routing::Primary => clients::WaitRole::Primary,
203 Routing::Replica => clients::WaitRole::Replica,
204 },
205 }));
206 client.set_state(clients::ClientState::Waiting);
207 queue_start.get_or_insert_with(Instant::now);
208
209 let remaining = checkout_timeout.saturating_sub(checkout_start.elapsed());
210 if remaining.is_zero() {
211 client.set_waiting_for(None);
212 client.set_state(clients::ClientState::Idle);
213 anyhow::bail!(
214 "checkout timed out after {checkout_timeout:?} for {database}/{user}",
215 );
216 }
217 let wake = tokio::time::timeout(remaining, rx).await;
218 client.set_waiting_for(None);
219 client.set_state(clients::ClientState::Idle);
220
221 if wake.is_err() {
222 anyhow::bail!(
223 "checkout timed out after {checkout_timeout:?} for {database}/{user}",
224 );
225 }
226 if self.shutting_down.load(Ordering::Acquire) {
227 anyhow::bail!("halephant is shutting down");
228 }
229 }
230 }
231 .await
232 .inspect(|_| {
233 tracing::Span::current().record("otel.status_code", "OK");
234 })
235 .inspect_err(|e| {
236 let msg = e.to_string();
237 let error_type = if let Some(resolve_err) = e.downcast_ref::<ResolveError>() {
238 match resolve_err {
239 ResolveError::UnknownDatabase { .. } => "unknown_database",
240 ResolveError::NoPrimary { .. } => "no_primary",
241 ResolveError::NoReplica { .. } => "no_replica",
242 }
243 } else if msg.contains("shutting down") {
244 "shutting_down"
245 } else if msg.contains("checkout timed out") {
246 "checkout_timeout"
247 } else {
248 "connect_failed"
249 };
250 o11y::metrics::record_checkout_error(database, user, error_type);
251 let span = tracing::Span::current();
252 span.record("otel.status_code", "ERROR");
253 span.record("otel.status_description", "checkout failed");
254 })
255 }
256
257 fn resolve_candidates(
262 &self,
263 cfg: &crate::config::Config,
264 database: &str,
265 user: &str,
266 routing: Routing,
267 ) -> Vec<PoolKey> {
268 let Some((cluster_name, _, _)) = cfg.find_pool(database) else {
269 return Vec::new();
270 };
271 match routing {
272 Routing::Primary => {
273 let Some(primary) = self.topology.primary(cluster_name) else {
274 return Vec::new();
275 };
276 vec![PoolKey {
277 node: primary,
278 database: database.to_owned(),
279 user: user.to_owned(),
280 }]
281 }
282 Routing::Replica => self
283 .topology
284 .replicas(cluster_name)
285 .iter()
286 .map(|addr| PoolKey {
287 node: addr.clone(),
288 database: database.to_owned(),
289 user: user.to_owned(),
290 })
291 .collect(),
292 }
293 }
294
295 fn try_candidates(
296 &self,
297 cfg: &crate::config::Config,
298 candidates: &[PoolKey],
299 idle_timeout: Duration,
300 max_lifetime: Duration,
301 ) -> CandidateResult {
302 let mut inner = self.inner.lock();
303
304 let mut ranked: Vec<&PoolKey> = candidates.iter().collect();
306 if ranked.len() > 1 {
307 let start = self.replica_rr.fetch_add(1, Ordering::Relaxed) % ranked.len();
308 ranked.rotate_left(start);
309 ranked.sort_by_key(|k| inner.pools.get(*k).map_or(0, |p| p.active.len()));
310 }
311
312 for key in ranked {
313 let limits = self.resolve_limits(cfg, key);
314
315 let idle = {
318 let pool = inner.pools.entry(key.clone()).or_default();
319 let idle = pool.pop_idle(idle_timeout, max_lifetime);
320 if let Some(ref c) = idle {
321 pool.active.insert(c.id);
322 }
323 idle
324 };
325 if let Some(idle) = idle {
326 return CandidateResult::GotIdle {
327 key: key.clone(),
328 conn: Box::new(idle),
329 };
330 }
331
332 let user_total = inner.pools.get(key).map_or(0, super::types::Pool::total);
333 if !limits.user_has_capacity(user_total) {
334 continue;
335 }
336
337 let node_total = inner.node_total(&key.node, &key.database);
338 if !limits.pool_has_capacity(node_total) && !Self::evict_idle_for_node(&mut inner, key)
339 {
340 continue;
341 }
342
343 let id = ConnId::next();
344 inner
345 .pools
346 .entry(key.clone())
347 .or_default()
348 .active
349 .insert(id);
350 return CandidateResult::Claimed {
351 key: key.clone(),
352 id,
353 };
354 }
355
356 CandidateResult::AllFull
357 }
358
359 fn evict_idle_for_node(inner: &mut super::types::Inner, key: &PoolKey) -> bool {
360 let victim = inner
361 .pools
362 .iter()
363 .filter(|(k, p)| k.node == key.node && k.database == key.database && !p.idle.is_empty())
364 .min_by_key(|(_, p)| p.idle.front().map_or(Instant::now(), |i| i.idle_since))
365 .map(|(k, _)| k.clone());
366 if let Some(ref vk) = victim
367 && let Some(pool) = inner.pools.get_mut(vk)
368 {
369 pool.idle.pop_front();
370 }
371 victim.is_some()
372 }
373
374 pub(in crate::pool) fn discard_internal(&self, key: &PoolKey, id: ConnId) {
375 {
376 let mut inner = self.inner.lock();
377 if let Some(pool) = inner.pools.get_mut(key) {
378 pool.active.remove(&id);
379 }
380 }
381 self.wake_one_for_node(key);
382 }
383}
384
385fn record_checkout_success(
386 key: &PoolKey,
387 checkout_start: Instant,
388 queue_start: Option<Instant>,
389 database: &str,
390 user: &str,
391 routing: Routing,
392 reused: bool,
393) {
394 if let Some(queued_at) = queue_start {
395 o11y::metrics::record_wait_duration(queued_at.elapsed(), database, user, routing);
396 }
397 o11y::metrics::record_checkout(checkout_start, database, user, &key.node);
398 let span = tracing::Span::current();
399 span.record("pool.reused", reused);
400 span.record("pool.waited", queue_start.is_some());
401 debug!(database, user, node = %key.node, reused, "checkout complete");
402}