halephant_core/pool/refill.rs
1//! Background connection management: warm-up, shortfall refills,
2//! scavenger ticks, and the reset-and-return path taken by
3//! [`crate::pool::types::ConnGuard::checkin`].
4
5use std::sync::Arc;
6use std::time::Instant;
7
8use tracing::{Instrument, debug, trace, warn};
9
10use crate::connections::reset::reset_connection;
11use crate::connections::server::{self, ServerConn};
12use crate::pool::PoolManager;
13use crate::pool::types::{IdleConn, Pool, PoolKey};
14
15/// Maximum in-flight connection opens per call to
16/// [`PoolManager::spawn_connections`]. Because PostgreSQL's postmaster
17/// accepts and forks backends serially, slamming a single node with many
18/// simultaneous connects queues every SCRAM handshake behind the fork
19/// loop and inflates per-connection latency. A small cap keeps each
20/// connect close to its minimum wall time while still parallelizing
21/// across different nodes (each batch gets its own semaphore).
22const PER_NODE_REFILL_CONCURRENCY: usize = 8;
23
24impl PoolManager {
25 /// Open idle connections at startup based on `min_connections` config.
26 pub async fn warm_up(self: &Arc<Self>) {
27 let shortfalls = self.shortfalls();
28 if shortfalls.is_empty() {
29 debug!("warm-up: nothing to do");
30 return;
31 }
32
33 let mut tasks = tokio::task::JoinSet::new();
34 for (key, n) in shortfalls {
35 let this = Arc::clone(self);
36 tasks.spawn(async move { this.spawn_connections(&key, n).await });
37 }
38
39 let mut opened = 0u32;
40 while let Some(result) = tasks.join_next().await {
41 match result {
42 Ok(n) => opened += n,
43 Err(e) => warn!("warm-up task panicked: {e}"),
44 }
45 }
46 debug!(opened, "warm-up complete");
47 }
48
49 /// Remove idle connections that exceed `max_lifetime` or whose socket is
50 /// dead, then apply `idle_timeout` only to connections in excess of
51 /// `min_connections` for that pool entry.
52 ///
53 /// `max_lifetime` and dead-socket detection always apply because those
54 /// represent unhealthy connections. `idle_timeout` represents excess
55 /// capacity that can be released — so it must respect the floor set by
56 /// `min_connections`, otherwise warm-up connections decay away even when
57 /// the user explicitly asked for a minimum.
58 ///
59 /// Returns the number of connections removed across all pool entries.
60 /// Normally invoked from [`PoolManager::run_scavenger`]; exposed so tests
61 /// can drive it directly without spinning up the full background task.
62 pub fn scavenge_idle(&self) -> u32 {
63 let cfg = self.config.load_full();
64 let mut inner = self.inner.lock();
65 let mut removed = 0u32;
66
67 for (key, pool) in &mut inner.pools {
68 let pool_config = cfg.find_pool(&key.database).map(|(_, _, p)| p);
69 let idle_timeout =
70 pool_config.map_or(std::time::Duration::from_secs(300), |p| p.idle_timeout);
71 let max_lifetime =
72 pool_config.map_or(std::time::Duration::from_secs(3600), |p| p.max_lifetime);
73
74 let before = pool.idle.len();
75
76 // Step 1: always drop dead and over-lifetime connections.
77 pool.idle.retain(|idle| {
78 if idle.conn.created_at.elapsed() > max_lifetime {
79 return false;
80 }
81 crate::connections::sock::is_alive(idle.conn.framed.get_ref())
82 });
83
84 // Step 2: apply idle_timeout, but stop once the pool is at the
85 // floor. The idle queue is FIFO, so the oldest expired connections
86 // are dropped first and the newest survive.
87 let min_idle = self.min_for_key(&cfg, key) as usize;
88 let removable = pool.idle.len().saturating_sub(min_idle);
89 let mut dropped = 0usize;
90 pool.idle.retain(|idle| {
91 if dropped >= removable {
92 return true;
93 }
94 if idle.idle_since.elapsed() > idle_timeout {
95 dropped += 1;
96 return false;
97 }
98 true
99 });
100
101 removed += (before - pool.idle.len()) as u32;
102 }
103
104 // Prune empty wait-queue entries so short-lived
105 // `(database, user, role)` combinations that contended once
106 // and never came back don't accumulate in the map. This runs
107 // while the inner lock is already held for the scavenge pass.
108 inner.waits.retain(|_, q| !q.waiters.is_empty());
109
110 removed
111 }
112
113 /// Enumerate every configured pool key whose connection count is below
114 /// its `min_for_key` floor, paired with the deficit. Walks the
115 /// configured `(cluster, database, user)` tuples and resolves the
116 /// expected primary and replica nodes from current topology — so the
117 /// result includes brand-new replicas that have no pool entry yet.
118 fn shortfalls(&self) -> Vec<(PoolKey, u32)> {
119 let cfg = self.config.load_full();
120 let inner = self.inner.lock();
121 let mut result = Vec::new();
122
123 for (cluster_name, cluster) in &cfg.cluster {
124 let primary = self.topology.primary(cluster_name);
125 let replicas = self.topology.replicas(cluster_name);
126
127 for (db_name, pool_config) in &cluster.pool {
128 for (user_name, user_config) in &pool_config.user {
129 // Primary shortfall.
130 if user_config.min_connections.primary > 0
131 && let Some(ref primary_addr) = primary
132 {
133 let key = PoolKey {
134 node: primary_addr.clone(),
135 database: db_name.clone(),
136 user: user_name.clone(),
137 };
138 let total = inner.pools.get(&key).map_or(0, Pool::total);
139 let want = user_config.min_connections.primary;
140 if total < want {
141 result.push((key, want - total));
142 }
143 }
144
145 // Replica shortfall — per replica node, not distributed.
146 if user_config.min_connections.replica > 0 {
147 for replica_addr in &replicas {
148 let key = PoolKey {
149 node: replica_addr.clone(),
150 database: db_name.clone(),
151 user: user_name.clone(),
152 };
153 let total = inner.pools.get(&key).map_or(0, Pool::total);
154 let want = user_config.min_connections.replica;
155 if total < want {
156 result.push((key, want - total));
157 }
158 }
159 }
160 }
161 }
162 }
163
164 result
165 }
166
167 /// Open up to `n` new connections for the given pool key in parallel
168 /// and insert each successful one into the idle queue. Returns the
169 /// number opened. Per-connection failures are logged and counted as
170 /// zero so a single bad node does not abort the rest of the batch.
171 ///
172 /// Wrapped in a `pool.refill` span so the per-connection `pool.connect`
173 /// spans nest under it in OTel traces.
174 #[tracing::instrument(name = "pool.refill", skip_all, fields(
175 server.address = %key.node,
176 db.namespace = %key.database,
177 user = %key.user,
178 count = n,
179 opened = tracing::field::Empty,
180 otel.status_code,
181 ))]
182 async fn spawn_connections(self: &Arc<Self>, key: &PoolKey, n: u32) -> u32 {
183 if n == 0 {
184 return 0;
185 }
186
187 // Resolve the upstream identity once for the batch. Connection
188 // failures from a missing config or user are surfaced via warn,
189 // not by silently returning 0 — that would mask config drift.
190 let cfg = self.config.load_full();
191 let (upstream_database, upstream_user, params, max_prepared, connect_timeout) = {
192 let Some((_, cluster_config, pool_config)) = cfg.find_pool(&key.database) else {
193 warn!(
194 database = %key.database,
195 "refill: database not configured",
196 );
197 return 0;
198 };
199 let Some(user_config) = pool_config.user.get(&key.user) else {
200 warn!(
201 database = %key.database,
202 user = %key.user,
203 "refill: user not configured",
204 );
205 return 0;
206 };
207 (
208 pool_config.database_name(&key.database).to_owned(),
209 user_config.upstream_name(&key.user).to_owned(),
210 user_config.parameters.clone(),
211 cfg.server.max_prepared_statements,
212 cluster_config.connect_timeout,
213 )
214 };
215
216 let password = self.lookup_password(&key.node, &upstream_database, &upstream_user);
217
218 // Cap in-flight opens to the same upstream node. PostgreSQL's
219 // postmaster accepts and forks backends serially — slamming it
220 // with tons of simultaneous connects queues every SCRAM handshake
221 // behind the fork loop, so each individual `pool.connect` span
222 // stretches to hundreds of milliseconds. Capping the batch keeps
223 // the per-connect wall time close to the minimum and gets the
224 // earliest connections into the idle queue sooner. This semaphore
225 // is local to one `spawn_connections` call, so different nodes
226 // still run fully in parallel.
227 let permits = Arc::new(tokio::sync::Semaphore::new(PER_NODE_REFILL_CONCURRENCY));
228
229 let parent_span = tracing::Span::current();
230 let mut tasks = tokio::task::JoinSet::new();
231 for _ in 0..n {
232 let this = Arc::clone(self);
233 let key = key.clone();
234 let upstream_database = upstream_database.clone();
235 let upstream_user = upstream_user.clone();
236 let password = password.clone();
237 let params = params.clone();
238 let permits = Arc::clone(&permits);
239
240 tasks.spawn(
241 async move {
242 // Acquire before connecting so permits throttle the
243 // actual handshake work, not just scheduling.
244 let _permit = permits
245 .acquire_owned()
246 .await
247 .expect("refill semaphore is never closed");
248 match server::connect_server(
249 &key.node,
250 &upstream_database,
251 &upstream_user,
252 password.as_deref(),
253 ¶ms,
254 max_prepared,
255 connect_timeout,
256 )
257 .await
258 {
259 Ok(conn) => {
260 {
261 let mut inner = this.inner.lock();
262 let pool = inner.pools.entry(key.clone()).or_default();
263 pool.idle.push_back(IdleConn {
264 conn,
265 idle_since: Instant::now(),
266 });
267 }
268 // New idle connection — wake a waiter if
269 // anyone is queued for this role.
270 this.wake_one_for_node(&key);
271 true
272 }
273 Err(e) => {
274 warn!(
275 database = key.database,
276 user = key.user,
277 node = key.node,
278 %e,
279 "refill connection failed",
280 );
281 false
282 }
283 }
284 }
285 .instrument(parent_span.clone()),
286 );
287 }
288
289 let mut opened = 0u32;
290 while let Some(result) = tasks.join_next().await {
291 match result {
292 Ok(true) => opened += 1,
293 Ok(false) => {}
294 Err(e) => warn!(%e, "refill task panicked"),
295 }
296 }
297 let span = tracing::Span::current();
298 span.record("opened", opened);
299 // Status is OK as long as at least one connection made it into the
300 // pool. A batch where every task failed is recorded as error.
301 if opened > 0 || n == 0 {
302 span.record("otel.status_code", "OK");
303 } else {
304 span.record("otel.status_code", "ERROR");
305 }
306 opened
307 }
308
309 /// Run the idle scavenger and refill loop at a fixed interval.
310 ///
311 /// Each tick prunes expired idle connections (preserving the
312 /// `min_connections` floor), then refills any pool entry below its
313 /// floor by fanning every `spawn_connections` call into a
314 /// `JoinSet` and awaiting them concurrently — matching
315 /// [`PoolManager::warm_up`]. Running them in series here would
316 /// let a single slow node blow most of the 30-second budget and
317 /// leave unrelated pools under their `min_connections` floor.
318 /// The per-tick state is logged at debug level so the loop is
319 /// visible without needing trace-level filters.
320 pub async fn run_scavenger(self: &Arc<Self>) {
321 let interval = std::time::Duration::from_secs(30);
322 loop {
323 tokio::time::sleep(interval).await;
324 let removed = self.scavenge_idle();
325 let shortfalls = self.shortfalls();
326 debug!(removed, refills = shortfalls.len(), "scavenger tick");
327 let mut tasks = tokio::task::JoinSet::new();
328 for (key, n) in shortfalls {
329 let this = Arc::clone(self);
330 tasks.spawn(async move { this.spawn_connections(&key, n).await });
331 }
332 while let Some(result) = tasks.join_next().await {
333 if let Err(e) = result {
334 warn!(%e, "scavenger refill task panicked");
335 }
336 }
337 }
338 }
339
340 /// Reset a connection in the background and return it to the idle pool.
341 pub(in crate::pool) async fn reset_and_return(
342 self: &Arc<Self>,
343 key: PoolKey,
344 mut conn: ServerConn,
345 ) {
346 match reset_connection(&mut conn).await {
347 Ok(()) => {
348 let returned = {
349 let mut inner = self.inner.lock();
350 if let Some(pool) = inner.pools.get_mut(&key) {
351 pool.resetting = pool.resetting.saturating_sub(1);
352 pool.idle.push_back(IdleConn {
353 conn,
354 idle_since: Instant::now(),
355 });
356 trace!(
357 node = key.node,
358 database = key.database,
359 user = key.user,
360 idle = pool.idle.len(),
361 "connection reset and returned to pool"
362 );
363 true
364 } else {
365 // Pool entry gone — connection dropped.
366 false
367 }
368 };
369 if returned {
370 self.wake_one_for_node(&key);
371 }
372 }
373 Err(e) => {
374 debug!(
375 node = key.node,
376 database = key.database,
377 user = key.user,
378 %e,
379 "reset failed, discarding connection"
380 );
381 {
382 let mut inner = self.inner.lock();
383 if let Some(pool) = inner.pools.get_mut(&key) {
384 pool.resetting = pool.resetting.saturating_sub(1);
385 }
386 }
387 // The reset failed so the connection is gone — that
388 // frees a slot on the node, wake a waiter so it can
389 // try to grow the pool.
390 self.wake_one_for_node(&key);
391 }
392 }
393 }
394}