halephant 0.6.0

A proxy and pooler for Postgres
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
#![cfg_attr(test, allow(clippy::unwrap_used))]
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;

use anyhow::Context;
use arc_swap::ArcSwap;
use clap::Parser;
use opentelemetry::trace::TracerProvider;
use opentelemetry_otlp::WithExportConfig;
use tokio::net::{TcpListener, TcpStream};
use tokio::task::JoinSet;
use tracing::{error, info, warn};
use tracing_subscriber::Layer as _;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;

use halephant_core::auth::Authenticator;
use halephant_core::clients;
use halephant_core::config::Config;
use halephant_core::listener::ListenerManager;
use halephant_core::pool::PoolManager;
use halephant_core::proxy;
use halephant_core::topology::TopologyManager;

// ---------------------------------------------------------------------------
// CLI
// ---------------------------------------------------------------------------

#[derive(Parser)]
#[command(
    name = "halephant",
    version,
    about = "PostgreSQL connection pooler and proxy"
)]
struct Cli {
    /// Path to configuration file.
    #[arg(short, long, default_value = "halephant.toml")]
    config: PathBuf,

    /// Validate configuration and exit.
    #[arg(long)]
    check: bool,
}

// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------

fn main() -> anyhow::Result<()> {
    let cli = Cli::parse();

    let config = Config::load(&cli.config)
        .with_context(|| format!("failed to load config from {}", cli.config.display()))?;

    if cli.check {
        #[allow(clippy::print_stdout)]
        {
            println!("configuration is valid");
        }
        return Ok(());
    }

    // `main` stays synchronous so the config is in hand before the
    // runtime exists: `server.workers` sizes the worker pool, and a
    // `#[tokio::main]` runtime would already be running by the time the
    // file is read. Validation (`--check`) never needs a runtime at all.
    let runtime = build_runtime(config.server.workers).context("failed to build tokio runtime")?;
    runtime.block_on(run(config, &cli.config))
}

/// Builds the multi-threaded runtime that hosts the proxy, sized by
/// `server.workers`.
///
/// A `workers` value of `0` — the default, documented as auto-detecting the
/// CPU count — leaves `worker_threads` unset so the builder applies its own
/// default of one worker per available core. Any other value pins the pool to
/// exactly that many worker threads.
///
/// The runtime is built once, before anything is spawned, which is why
/// `server.workers` is restart-required rather than hot-reloadable.
fn build_runtime(workers: usize) -> std::io::Result<tokio::runtime::Runtime> {
    let mut builder = tokio::runtime::Builder::new_multi_thread();
    if workers > 0 {
        builder.worker_threads(workers);
    }
    builder.enable_all().build()
}

/// Serves the proxy on the runtime built by [`build_runtime`]: installs
/// telemetry, warms the pools, accepts clients until SIGINT or SIGTERM, then
/// drains. `config_path` is the file SIGHUP re-reads.
async fn run(config: Config, config_path: &Path) -> anyhow::Result<()> {
    let env_filter = tracing_subscriber::EnvFilter::try_from_default_env()
        .unwrap_or_else(|_| format!("halephant={}", config.logging.level).into());
    let fmt_layer = tracing_subscriber::fmt::layer().with_filter(env_filter);

    // Shared target filter applied to every OTel exporter layer. Inherits
    // the default level from `logging.level` so operators control OTel
    // volume from the same knob as stdout, and silences noisy upstream
    // dependencies to prevent the OTel exporter (which uses tonic/h2/
    // hyper internally) from logging its own shipping activity back into
    // itself.
    let otel_default_level: tracing::Level =
        config.logging.level.parse().unwrap_or(tracing::Level::INFO);
    let otel_target_filter = || {
        tracing_subscriber::filter::Targets::new()
            .with_default(otel_default_level)
            .with_target("opentelemetry", tracing::level_filters::LevelFilter::OFF)
            .with_target("tonic", tracing::level_filters::LevelFilter::OFF)
            .with_target("h2", tracing::level_filters::LevelFilter::OFF)
            .with_target("hyper", tracing::level_filters::LevelFilter::OFF)
    };

    let mut trace_provider: Option<opentelemetry_sdk::trace::SdkTracerProvider> = None;
    let mut meter_provider: Option<opentelemetry_sdk::metrics::SdkMeterProvider> = None;
    let mut logger_provider: Option<opentelemetry_sdk::logs::SdkLoggerProvider> = None;
    let mut otel_trace_layer = None;
    let mut otel_log_layer = None;
    if let Some(ref endpoint) = config.otel.endpoint {
        let resource = opentelemetry_sdk::Resource::builder()
            .with_service_name(config.otel.service_name.clone())
            .build();

        // Traces.
        let span_exporter = opentelemetry_otlp::SpanExporter::builder()
            .with_tonic()
            .with_endpoint(endpoint)
            .build()
            .context("failed to build OTLP span exporter")?;
        let tp = opentelemetry_sdk::trace::SdkTracerProvider::builder()
            .with_batch_exporter(span_exporter)
            .with_resource(resource.clone())
            .build();
        let tracer = tp.tracer("halephant");
        trace_provider = Some(tp);

        // Metrics.
        let metric_exporter = opentelemetry_otlp::MetricExporter::builder()
            .with_tonic()
            .with_endpoint(endpoint)
            .build()
            .context("failed to build OTLP metric exporter")?;
        let metric_reader = opentelemetry_sdk::metrics::PeriodicReader::builder(metric_exporter)
            .with_interval(std::time::Duration::from_secs(15))
            .build();
        let mp = opentelemetry_sdk::metrics::SdkMeterProvider::builder()
            .with_reader(metric_reader)
            .with_resource(resource.clone())
            .build();
        opentelemetry::global::set_meter_provider(mp.clone());
        meter_provider = Some(mp);

        // Logs. Every tracing event becomes an OTel `LogRecord` with a
        // populated `SeverityText` / `SeverityNumber`, so HyperDX and other
        // OTel-native log backends show proper levels instead of
        // "undefined". The bridge attaches the current trace context to
        // each record, so jumping from a trace to its logs works.
        let log_exporter = opentelemetry_otlp::LogExporter::builder()
            .with_tonic()
            .with_endpoint(endpoint)
            .build()
            .context("failed to build OTLP log exporter")?;
        let lp = opentelemetry_sdk::logs::SdkLoggerProvider::builder()
            .with_batch_exporter(log_exporter)
            .with_resource(resource)
            .build();
        otel_log_layer = Some(
            opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(&lp)
                .with_filter(otel_target_filter()),
        );
        logger_provider = Some(lp);

        otel_trace_layer = Some(
            tracing_opentelemetry::layer()
                .with_tracer(tracer)
                .with_level(true)
                .with_filter(otel_target_filter()),
        );
    }

    tracing_subscriber::Registry::default()
        .with(otel_trace_layer)
        .with(otel_log_layer)
        .with(fmt_layer)
        .init();

    // Wrap the loaded config in an `ArcSwap` so a future SIGHUP reload
    // can swap a new snapshot in atomically without touching any of
    // the long-lived handles (pools, topology, admin, listeners, …).
    let config = Arc::new(ArcSwap::from_pointee(config));

    let pgpass = {
        use std::path::PathBuf;
        let cfg = config.load();
        let path = cfg.server.pgpass.clone().unwrap_or_else(|| {
            std::env::var("PGPASSFILE").map_or_else(
                |_| {
                    std::env::var("HOME")
                        .map_or_default(PathBuf::from)
                        .join(".pgpass")
                },
                PathBuf::from,
            )
        });
        Arc::new(halephant_core::auth::pgpass::Pgpass::load(&path))
    };

    let topology = Arc::new(TopologyManager::new(
        Arc::clone(&config),
        Arc::clone(&pgpass),
    ));

    // Run initial topology discovery before accepting connections so warm-up
    // sees accurate primary/replica roles. The periodic refresh loop starts
    // below.
    info!("discovering cluster topology...");
    let _ = topology.refresh().await;

    let pools = Arc::new(PoolManager::new(
        Arc::clone(&config),
        Arc::clone(&topology),
        Arc::clone(&pgpass),
    ));
    pools.warm_up().await;

    // Tracks every accepted client from accept through cleanup. Shared
    // with the admin HTTP API and the client-state metric gauge.
    let clients = Arc::new(clients::ClientRegistry::new());

    // Register pool metrics via the global meter. When OTel is disabled the
    // global meter returns no-op instruments (zero overhead).
    let _metrics = halephant_core::o11y::gauges::register_pool_gauges(
        &pools,
        Arc::clone(&config),
        Arc::clone(&topology),
        &clients,
    );

    // Start background topology refresh loop (drains pools on role changes).
    let topo_loop = Arc::clone(&topology);
    let pools_for_topo = Arc::clone(&pools);
    tokio::spawn(async move { topo_loop.run_loop(&pools_for_topo).await });

    // Start background idle connection scavenger.
    let pools_for_scavenger = Arc::clone(&pools);
    tokio::spawn(async move { pools_for_scavenger.run_scavenger().await });

    // Start admin HTTP API if configured.
    if let Some(addr) = config.load().admin.listen.clone() {
        let admin_config = Arc::clone(&config);
        let admin_pools = Arc::clone(&pools);
        let admin_topo = Arc::clone(&topology);
        let admin_clients = Arc::clone(&clients);
        tokio::spawn(async move {
            if let Err(e) = halephant_core::admin::serve(
                &addr,
                admin_config,
                admin_pools,
                admin_topo,
                admin_clients,
            )
            .await
            {
                error!(%e, "admin API failed");
            }
        });
    }

    let listeners = Arc::new(ListenerManager::new(
        Arc::clone(&config),
        Arc::clone(&topology),
        Arc::clone(&pgpass),
    ));
    let auth = Arc::new(Authenticator::new());

    // Snapshot the startup-time config for listen addresses. These
    // socket bindings happen once and cannot be rebound while the
    // process is running, so `server.listen` is in the restart-required
    // set. Other fields like `shutdown_timeout` are read from the live
    // `ArcSwap` at their point of use so hot-reloads take effect.
    let startup_cfg = config.load_full();
    // Spawn one accept task per configured listen address and funnel
    // accepted sockets through a shared channel. Lets the tokio
    // scheduler race listeners fairly — an MPSC recv inside the main
    // `select!` avoids hand-rolled polling over a runtime-sized
    // `Vec<TcpListener>` (which would bias toward whichever listener
    // is iterated first). Buffer is deliberately small: if the main
    // loop falls behind, applying backpressure to the accept side is
    // preferable to queueing sockets that would be force-closed on
    // shutdown anyway.
    let (accept_tx, mut accept_rx) = tokio::sync::mpsc::channel::<(TcpStream, SocketAddr)>(32);
    for addr in &startup_cfg.server.listen {
        let listener = TcpListener::bind(addr)
            .await
            .with_context(|| format!("failed to bind to {addr}"))?;
        info!(%addr, "listening");
        let tx = accept_tx.clone();
        let addr = addr.clone();
        tokio::spawn(async move {
            loop {
                match listener.accept().await {
                    Ok((stream, client_addr)) => {
                        if tx.send((stream, client_addr)).await.is_err() {
                            // Main loop dropped its receiver — shutting down.
                            return;
                        }
                    }
                    Err(e) => {
                        // Accept failures are rare but recoverable (for
                        // example, fd exhaustion under load). Log and
                        // back off briefly so a sustained failure
                        // doesn't busy-spin the task.
                        warn!(%addr, %e, "accept failed");
                        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
                    }
                }
            }
        });
    }
    // Drop the main thread's handle so the channel closes once every
    // spawned accept task exits (which happens when the listener is
    // dropped at process shutdown via task cancellation).
    drop(accept_tx);
    info!(clusters = startup_cfg.cluster.len(), "halephant ready",);
    drop(startup_cfg);

    let mut connections = JoinSet::new();

    // Hot-reload counter. Incremented once per SIGHUP attempt with an
    // `outcome` attribute so dashboards can alert on parse_failed /
    // restart_required spikes without breaking down the success series.
    let reload_counter = opentelemetry::global::meter("halephant")
        .u64_counter("halephant.config.reloads")
        .with_description("SIGHUP config reload attempts, grouped by outcome")
        .build();

    // Install SIGTERM handler so `docker stop` and similar trigger a
    // graceful drain instead of waiting out the grace period and being
    // SIGKILL'd (exit code 137). SIGINT still works for interactive users.
    let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
        .context("failed to install SIGTERM handler")?;

    // Install SIGHUP handler for live config reload. Operators send
    // SIGHUP after editing the config file on disk; the select branch
    // below re-reads, validates, and atomically swaps it in.
    let mut sighup = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup())
        .context("failed to install SIGHUP handler")?;

    loop {
        tokio::select! {
            Some((client_stream, client_addr)) = accept_rx.recv() => {
                client_stream.set_nodelay(true)?;
                // Snapshot the current config for the whole client
                // session. A concurrent hot-reload doesn't retarget an
                // already-connected client; their resolved database
                // mode, `otel.query_text`, and authentication settings
                // stay consistent for the duration of the session.
                // New connections accepted after the swap pick up the
                // new snapshot immediately.
                let session_cfg = config.load_full();
                let pools = Arc::clone(&pools);
                let listeners = Arc::clone(&listeners);
                let auth = Arc::clone(&auth);
                let clients = Arc::clone(&clients);
                connections.spawn(async move {
                    info!(%client_addr, "accepted");
                    if let Err(e) = proxy::frontend::forward(
                        client_stream,
                        client_addr,
                        session_cfg.as_ref(),
                        &pools,
                        &listeners,
                        &auth,
                        &clients,
                    )
                    .await
                    {
                        error!(%client_addr, %e, "connection error");
                    }
                    info!(%client_addr, "closed");
                });
            }
            _ = tokio::signal::ctrl_c() => {
                info!("received SIGINT, shutting down");
                break;
            }
            _ = sigterm.recv() => {
                info!("received SIGTERM, shutting down");
                break;
            }
            _ = sighup.recv() => {
                info!("received SIGHUP, reloading config");
                use halephant_core::config::reload::{reload_config, ReloadOutcome};
                use opentelemetry::KeyValue;
                match reload_config(config_path, &config).await {
                    ReloadOutcome::Success => {
                        reload_counter.add(1, &[KeyValue::new("outcome", "success")]);
                        info!("config reloaded");
                        // Side effects after the atomic swap, spawned
                        // as a background task so the accept loop is
                        // not blocked while `topology.refresh()` waits
                        // on slow or unreachable nodes (each probe can
                        // take up to `timeout`). The swap
                        // itself is already visible to every component
                        // — these steps just bring the running state
                        // in line with the new config.
                        //
                        // Order matters: refresh topology first so
                        // `warm_up` sees the updated cluster node list
                        // when deciding which primaries and replicas
                        // to open idle connections against.
                        let topo = Arc::clone(&topology);
                        let pools_for_reload = Arc::clone(&pools);
                        tokio::spawn(async move {
                            let changed = topo.refresh().await;
                            for node in &changed {
                                pools_for_reload.drain_node(node);
                            }
                            pools_for_reload.warm_up().await;
                        });
                    }
                    ReloadOutcome::RestartRequired(field) => {
                        reload_counter.add(
                            1,
                            &[KeyValue::new("outcome", "restart_required")],
                        );
                        warn!(
                            field,
                            "config reload rejected: change to this field requires a process restart"
                        );
                    }
                    ReloadOutcome::ParseFailed(e) => {
                        reload_counter.add(
                            1,
                            &[KeyValue::new("outcome", "parse_failed")],
                        );
                        error!(%e, "config reload failed");
                    }
                }
            }
        }

        // Reap finished tasks without blocking.
        while connections.try_join_next().is_some() {}
    }

    // Graceful shutdown.
    // 1. Stop accepting (listeners dropped above).
    // 2. Wake every queued waiter with a shutting-down error so tasks
    //    blocked in `checkout()` return immediately instead of waiting
    //    out the full `checkout_timeout`. Without this, waiters would
    //    sit until their individual timeouts expire or the drain
    //    deadline aborts them.
    pools.shutdown();
    // 3. Drain in-flight client connections with a timeout. Read the
    //    timeout from the current config so a hot-reloaded value takes
    //    effect on shutdown — makes `server.shutdown_timeout`
    //    hot-reloadable even though `server.listen` is not.
    let shutdown_timeout = config.load().server.shutdown_timeout;
    let active = connections.len();
    if active > 0 {
        info!(active, timeout = ?shutdown_timeout, "shutting down, draining connections...");
        let deadline = tokio::time::sleep(shutdown_timeout);
        tokio::pin!(deadline);

        loop {
            tokio::select! {
                result = connections.join_next() => {
                    if result.is_none() {
                        break; // all done
                    }
                    let remaining = connections.len();
                    if remaining > 0 {
                        info!(remaining, "draining...");
                    }
                }
                () = &mut deadline => {
                    let remaining = connections.len();
                    warn!(remaining, "shutdown timeout reached, dropping remaining connections");
                    connections.abort_all();
                    break;
                }
            }
        }
    }

    // 3. Close all idle pool connections.
    pools.drain_all();

    // 4. Flush pending OTel data before exiting.
    if let Some(provider) = trace_provider
        && let Err(e) = provider.shutdown()
    {
        warn!("failed to flush OpenTelemetry spans: {e}");
    }
    if let Some(provider) = meter_provider
        && let Err(e) = provider.shutdown()
    {
        warn!("failed to flush OpenTelemetry metrics: {e}");
    }
    if let Some(provider) = logger_provider
        && let Err(e) = provider.shutdown()
    {
        warn!("failed to flush OpenTelemetry logs: {e}");
    }

    info!("halephant stopped");
    Ok(())
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::build_runtime;

    /// Worker count the builder picks on its own, which is what
    /// `server.workers = 0` must preserve. Reading it from a plain builder
    /// keeps the assertion honest if tokio changes how it auto-detects.
    fn builder_default_workers() -> usize {
        tokio::runtime::Builder::new_multi_thread()
            .enable_all()
            .build()
            .unwrap()
            .metrics()
            .num_workers()
    }

    #[test]
    fn zero_workers_keeps_the_builder_default() {
        let runtime = build_runtime(0).unwrap();
        assert_eq!(runtime.metrics().num_workers(), builder_default_workers());
    }

    #[test]
    fn explicit_worker_count_sizes_the_runtime() {
        let runtime = build_runtime(3).unwrap();
        assert_eq!(runtime.metrics().num_workers(), 3);
    }

    /// A single worker is a legal setting, not a synonym for auto-detect.
    #[test]
    fn one_worker_pins_a_single_thread() {
        let runtime = build_runtime(1).unwrap();
        assert_eq!(runtime.metrics().num_workers(), 1);
    }

    /// The runtime carries the IO and time drivers the proxy needs, so
    /// `enable_all` has to survive both branches of the worker mapping.
    #[test]
    fn runtime_enables_io_and_time_drivers() {
        for workers in [0, 2] {
            let runtime = build_runtime(workers).unwrap();
            runtime.block_on(async {
                tokio::time::sleep(std::time::Duration::from_millis(1)).await;
                tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
            });
        }
    }
}