halephant_core/pool/refill.rs
1//! Background connection management: warm-up, shortfall refills,
2//! scavenger ticks, and the reset-and-return path taken by
3//! [`crate::pool::types::ConnGuard::checkin`].
4
5use std::sync::Arc;
6use std::time::Instant;
7
8use tracing::{Instrument, debug, warn};
9
10use crate::connections::reset::reset_connection;
11use crate::connections::server::{self, ServerConn};
12use crate::pool::PoolManager;
13use crate::pool::types::{ConnId, IdleConn, PoolKey};
14
15/// Maximum in-flight connection opens per call to
16/// [`PoolManager::spawn_connections`]. Because PostgreSQL's postmaster
17/// accepts and forks backends serially, slamming a single node with many
18/// simultaneous connects queues every SCRAM handshake behind the fork
19/// loop and inflates per-connection latency. A small cap keeps each
20/// connect close to its minimum wall time while still parallelizing
21/// across different nodes (each batch gets its own semaphore).
22const PER_NODE_REFILL_CONCURRENCY: usize = 8;
23
24impl PoolManager {
25 /// Open idle connections at startup based on `min_connections` config.
26 pub async fn warm_up(self: &Arc<Self>) {
27 let shortfalls = self.shortfalls();
28 if shortfalls.is_empty() {
29 debug!("warm-up: nothing to do");
30 return;
31 }
32
33 let mut tasks = tokio::task::JoinSet::new();
34 for (key, n) in shortfalls {
35 let this = Arc::clone(self);
36 tasks.spawn(async move { this.spawn_connections(&key, n).await });
37 }
38
39 let mut opened = 0u32;
40 while let Some(result) = tasks.join_next().await {
41 match result {
42 Ok(n) => opened += n,
43 Err(e) => warn!("warm-up task panicked: {e}"),
44 }
45 }
46 debug!(opened, "warm-up complete");
47 }
48
49 /// Remove idle connections that exceed `max_lifetime` or whose socket is
50 /// dead, then apply `idle_timeout` only to connections in excess of
51 /// `min_connections` for that pool entry.
52 ///
53 /// `max_lifetime` and dead-socket detection always apply because those
54 /// represent unhealthy connections. `idle_timeout` represents excess
55 /// capacity that can be released — so it must respect the floor set by
56 /// `min_connections`, otherwise warm-up connections decay away even when
57 /// the user explicitly asked for a minimum.
58 ///
59 /// Returns the number of connections removed across all pool entries.
60 /// Normally invoked from [`PoolManager::run_scavenger`]; exposed so tests
61 /// can drive it directly without spinning up the full background task.
62 pub fn scavenge_idle(&self) -> u32 {
63 let cfg = self.config.load_full();
64 let mut freed: Vec<(PoolKey, u32)> = Vec::new();
65 let mut removed = 0u32;
66
67 {
68 let mut inner = self.inner.lock();
69
70 for (key, pool) in &mut inner.pools {
71 let pool_config = cfg.find_pool(&key.database).map(|(_, _, p)| p);
72 let idle_timeout =
73 pool_config.map_or(std::time::Duration::from_secs(300), |p| p.idle_timeout);
74 let max_lifetime =
75 pool_config.map_or(std::time::Duration::from_secs(3600), |p| p.max_lifetime);
76
77 let before = pool.idle.len();
78
79 // Remove dead sockets and connections past max_lifetime
80 // unconditionally (regardless of min_connections floor).
81 pool.idle.retain(|idle| {
82 idle.conn.created_at.elapsed() <= max_lifetime
83 && crate::connections::sock::is_alive(idle.conn.framed.get_ref())
84 });
85
86 // Apply idle_timeout only to excess capacity above
87 // the min_connections floor.
88 let min_idle = self.min_for_key(&cfg, key) as usize;
89 let removable = pool.idle.len().saturating_sub(min_idle);
90 let mut dropped = 0usize;
91 pool.idle.retain(|idle| {
92 if dropped >= removable {
93 return true;
94 }
95 if idle.idle_since.elapsed() > idle_timeout {
96 dropped += 1;
97 return false;
98 }
99 true
100 });
101
102 let n = (before - pool.idle.len()) as u32;
103 if n > 0 {
104 freed.push((key.clone(), n));
105 }
106 removed += n;
107 }
108
109 inner.waits.retain(|_, q| !q.waiters.is_empty());
110 }
111
112 // Removing idle connections frees physical capacity — wake
113 // any waiter that was blocked on node-level limits.
114 for (key, n) in &freed {
115 for _ in 0..*n {
116 self.wake_one_for_node(key);
117 }
118 }
119
120 removed
121 }
122
123 /// Enumerate every configured pool key whose connection count is below
124 /// its `min_for_key` floor, paired with the deficit. Walks the
125 /// configured `(cluster, database, user)` tuples and resolves the
126 /// expected primary and replica nodes from current topology — so the
127 /// result includes brand-new replicas that have no pool entry yet.
128 fn shortfalls(&self) -> Vec<(PoolKey, u32)> {
129 let cfg = self.config.load_full();
130 let inner = self.inner.lock();
131 let mut result = Vec::new();
132
133 for (cluster_name, cluster) in &cfg.cluster {
134 let primary = self.topology.primary(cluster_name);
135 let replicas = self.topology.replicas(cluster_name);
136
137 for (db_name, pool_config) in &cluster.pool {
138 for (user_name, user_config) in &pool_config.user {
139 // Primary shortfall.
140 if user_config.min_connections.primary > 0
141 && let Some(ref primary_addr) = primary
142 {
143 let key = PoolKey {
144 node: primary_addr.clone(),
145 database: db_name.clone(),
146 user: user_name.clone(),
147 };
148 let total = inner.pools.get(&key).map_or(0, |p| p.total() as u32);
149 let want = user_config.min_connections.primary;
150 if total < want {
151 result.push((key, want - total));
152 }
153 }
154
155 // Replica shortfall — per replica node, not distributed.
156 if user_config.min_connections.replica > 0 {
157 for replica_addr in &replicas {
158 let key = PoolKey {
159 node: replica_addr.clone(),
160 database: db_name.clone(),
161 user: user_name.clone(),
162 };
163 let total = inner.pools.get(&key).map_or(0, |p| p.total() as u32);
164 let want = user_config.min_connections.replica;
165 if total < want {
166 result.push((key, want - total));
167 }
168 }
169 }
170 }
171 }
172 }
173
174 result
175 }
176
177 /// Open up to `n` new connections for the given pool key in parallel
178 /// and insert each successful one into the idle queue. Returns the
179 /// number opened. Per-connection failures are logged and counted as
180 /// zero so a single bad node does not abort the rest of the batch.
181 ///
182 /// Wrapped in a `pool.refill` span so the per-connection `pool.connect`
183 /// spans nest under it in OTel traces.
184 #[tracing::instrument(name = "pool.refill", skip_all, fields(
185 server.address = %key.node,
186 db.namespace = %key.database,
187 user = %key.user,
188 count = n,
189 opened = tracing::field::Empty,
190 otel.status_code,
191 ))]
192 async fn spawn_connections(self: &Arc<Self>, key: &PoolKey, n: u32) -> u32 {
193 if n == 0 {
194 return 0;
195 }
196
197 // Resolve the upstream identity once for the batch. Connection
198 // failures from a missing config or user are surfaced via warn,
199 // not by silently returning 0 — that would mask config drift.
200 let cfg = self.config.load_full();
201 let (upstream_database, upstream_user, params, max_prepared, connect_timeout) = {
202 let Some((_, cluster_config, pool_config)) = cfg.find_pool(&key.database) else {
203 warn!(
204 database = %key.database,
205 "refill: database not configured",
206 );
207 return 0;
208 };
209 let Some(user_config) = pool_config.user.get(&key.user) else {
210 warn!(
211 database = %key.database,
212 user = %key.user,
213 "refill: user not configured",
214 );
215 return 0;
216 };
217 (
218 pool_config.database_name(&key.database).to_owned(),
219 user_config.upstream_name(&key.user).to_owned(),
220 user_config.parameters.clone(),
221 cfg.server.max_prepared_statements,
222 cluster_config.connect_timeout,
223 )
224 };
225
226 let password = self.lookup_password(&key.node, &upstream_database, &upstream_user);
227
228 // Cap in-flight opens to the same upstream node. PostgreSQL's
229 // postmaster accepts and forks backends serially — slamming it
230 // with tons of simultaneous connects queues every SCRAM handshake
231 // behind the fork loop, so each individual `pool.connect` span
232 // stretches to hundreds of milliseconds. Capping the batch keeps
233 // the per-connect wall time close to the minimum and gets the
234 // earliest connections into the idle queue sooner. This semaphore
235 // is local to one `spawn_connections` call, so different nodes
236 // still run fully in parallel.
237 let permits = Arc::new(tokio::sync::Semaphore::new(PER_NODE_REFILL_CONCURRENCY));
238
239 let parent_span = tracing::Span::current();
240 let mut tasks = tokio::task::JoinSet::new();
241 for _ in 0..n {
242 let this = Arc::clone(self);
243 let key = key.clone();
244 let upstream_database = upstream_database.clone();
245 let upstream_user = upstream_user.clone();
246 let password = password.clone();
247 let params = params.clone();
248 let permits = Arc::clone(&permits);
249
250 tasks.spawn(
251 async move {
252 // Acquire before connecting so permits throttle the
253 // actual handshake work, not just scheduling.
254 let _permit = permits
255 .acquire_owned()
256 .await
257 .expect("refill semaphore is never closed");
258 match server::connect_server(
259 &key.node,
260 &upstream_database,
261 &upstream_user,
262 password.as_deref(),
263 ¶ms,
264 max_prepared,
265 connect_timeout,
266 )
267 .await
268 {
269 Ok(conn) => {
270 {
271 let mut inner = this.inner.lock();
272 let pool = inner.pools.entry(key.clone()).or_default();
273 pool.idle.push_back(IdleConn {
274 id: ConnId::next(),
275 conn,
276 idle_since: Instant::now(),
277 });
278 }
279 // New idle connection — wake a waiter if
280 // anyone is queued for this role.
281 this.wake_one_for_node(&key);
282 true
283 }
284 Err(e) => {
285 warn!(
286 database = key.database,
287 user = key.user,
288 node = key.node,
289 %e,
290 "refill connection failed",
291 );
292 false
293 }
294 }
295 }
296 .instrument(parent_span.clone()),
297 );
298 }
299
300 let mut opened = 0u32;
301 while let Some(result) = tasks.join_next().await {
302 match result {
303 Ok(true) => opened += 1,
304 Ok(false) => {}
305 Err(e) => warn!(%e, "refill task panicked"),
306 }
307 }
308 let span = tracing::Span::current();
309 span.record("opened", opened);
310 // Status is OK as long as at least one connection made it into the
311 // pool. A batch where every task failed is recorded as error.
312 if opened > 0 || n == 0 {
313 span.record("otel.status_code", "OK");
314 } else {
315 span.record("otel.status_code", "ERROR");
316 }
317 opened
318 }
319
320 /// Run the idle scavenger and refill loop at a fixed interval.
321 ///
322 /// Each tick prunes expired idle connections (preserving the
323 /// `min_connections` floor), then refills any pool entry below its
324 /// floor by fanning every `spawn_connections` call into a
325 /// `JoinSet` and awaiting them concurrently — matching
326 /// [`PoolManager::warm_up`]. Running them in series here would
327 /// let a single slow node blow most of the 30-second budget and
328 /// leave unrelated pools under their `min_connections` floor.
329 /// The per-tick state is logged at debug level so the loop is
330 /// visible without needing trace-level filters.
331 pub async fn run_scavenger(self: &Arc<Self>) {
332 let interval = std::time::Duration::from_secs(30);
333 loop {
334 tokio::time::sleep(interval).await;
335 let removed = self.scavenge_idle();
336 let shortfalls = self.shortfalls();
337 debug!(removed, refills = shortfalls.len(), "scavenger tick");
338 let mut tasks = tokio::task::JoinSet::new();
339 for (key, n) in shortfalls {
340 let this = Arc::clone(self);
341 tasks.spawn(async move { this.spawn_connections(&key, n).await });
342 }
343 while let Some(result) = tasks.join_next().await {
344 if let Err(e) = result {
345 warn!(%e, "scavenger refill task panicked");
346 }
347 }
348 }
349 }
350
351 pub(in crate::pool) async fn reset_and_return(
352 self: &Arc<Self>,
353 key: PoolKey,
354 id: ConnId,
355 mut conn: ServerConn,
356 ) {
357 match reset_connection(&mut conn).await {
358 Ok(()) => {
359 let returned = {
360 let mut inner = self.inner.lock();
361 if let Some(pool) = inner.pools.get_mut(&key) {
362 pool.resetting.remove(&id);
363 pool.idle.push_back(IdleConn {
364 id,
365 conn,
366 idle_since: Instant::now(),
367 });
368 true
369 } else {
370 false
371 }
372 };
373 if returned {
374 self.wake_one_for_node(&key);
375 }
376 }
377 Err(e) => {
378 debug!(
379 node = key.node,
380 database = key.database,
381 user = key.user,
382 %e,
383 "reset failed, discarding connection"
384 );
385 {
386 let mut inner = self.inner.lock();
387 if let Some(pool) = inner.pools.get_mut(&key) {
388 pool.resetting.remove(&id);
389 }
390 }
391 self.wake_one_for_node(&key);
392 }
393 }
394 }
395}