#![cfg_attr(test, allow(clippy::unwrap_used))]
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::Context;
use arc_swap::ArcSwap;
use clap::Parser;
use opentelemetry::trace::TracerProvider;
use opentelemetry_otlp::WithExportConfig;
use tokio::net::{TcpListener, TcpStream};
use tokio::task::JoinSet;
use tracing::{error, info, warn};
use tracing_subscriber::Layer as _;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use halephant_core::auth::Authenticator;
use halephant_core::clients;
use halephant_core::config::Config;
use halephant_core::listener::ListenerManager;
use halephant_core::pool::PoolManager;
use halephant_core::proxy;
use halephant_core::topology::TopologyManager;
#[derive(Parser)]
#[command(
name = "halephant",
version,
about = "PostgreSQL connection pooler and proxy"
)]
struct Cli {
#[arg(short, long, default_value = "halephant.toml")]
config: PathBuf,
#[arg(long)]
check: bool,
}
fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
let config = Config::load(&cli.config)
.with_context(|| format!("failed to load config from {}", cli.config.display()))?;
if cli.check {
#[allow(clippy::print_stdout)]
{
println!("configuration is valid");
}
return Ok(());
}
let runtime = build_runtime(config.server.workers).context("failed to build tokio runtime")?;
runtime.block_on(run(config, &cli.config))
}
fn build_runtime(workers: usize) -> std::io::Result<tokio::runtime::Runtime> {
let mut builder = tokio::runtime::Builder::new_multi_thread();
if workers > 0 {
builder.worker_threads(workers);
}
builder.enable_all().build()
}
async fn run(config: Config, config_path: &Path) -> anyhow::Result<()> {
let env_filter = tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| format!("halephant={}", config.logging.level).into());
let fmt_layer = tracing_subscriber::fmt::layer().with_filter(env_filter);
let otel_default_level: tracing::Level =
config.logging.level.parse().unwrap_or(tracing::Level::INFO);
let otel_target_filter = || {
tracing_subscriber::filter::Targets::new()
.with_default(otel_default_level)
.with_target("opentelemetry", tracing::level_filters::LevelFilter::OFF)
.with_target("tonic", tracing::level_filters::LevelFilter::OFF)
.with_target("h2", tracing::level_filters::LevelFilter::OFF)
.with_target("hyper", tracing::level_filters::LevelFilter::OFF)
};
let mut trace_provider: Option<opentelemetry_sdk::trace::SdkTracerProvider> = None;
let mut meter_provider: Option<opentelemetry_sdk::metrics::SdkMeterProvider> = None;
let mut logger_provider: Option<opentelemetry_sdk::logs::SdkLoggerProvider> = None;
let mut otel_trace_layer = None;
let mut otel_log_layer = None;
if let Some(ref endpoint) = config.otel.endpoint {
let resource = opentelemetry_sdk::Resource::builder()
.with_service_name(config.otel.service_name.clone())
.build();
let span_exporter = opentelemetry_otlp::SpanExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.build()
.context("failed to build OTLP span exporter")?;
let tp = opentelemetry_sdk::trace::SdkTracerProvider::builder()
.with_batch_exporter(span_exporter)
.with_resource(resource.clone())
.build();
let tracer = tp.tracer("halephant");
trace_provider = Some(tp);
let metric_exporter = opentelemetry_otlp::MetricExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.build()
.context("failed to build OTLP metric exporter")?;
let metric_reader = opentelemetry_sdk::metrics::PeriodicReader::builder(metric_exporter)
.with_interval(std::time::Duration::from_secs(15))
.build();
let mp = opentelemetry_sdk::metrics::SdkMeterProvider::builder()
.with_reader(metric_reader)
.with_resource(resource.clone())
.build();
opentelemetry::global::set_meter_provider(mp.clone());
meter_provider = Some(mp);
let log_exporter = opentelemetry_otlp::LogExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.build()
.context("failed to build OTLP log exporter")?;
let lp = opentelemetry_sdk::logs::SdkLoggerProvider::builder()
.with_batch_exporter(log_exporter)
.with_resource(resource)
.build();
otel_log_layer = Some(
opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge::new(&lp)
.with_filter(otel_target_filter()),
);
logger_provider = Some(lp);
otel_trace_layer = Some(
tracing_opentelemetry::layer()
.with_tracer(tracer)
.with_level(true)
.with_filter(otel_target_filter()),
);
}
tracing_subscriber::Registry::default()
.with(otel_trace_layer)
.with(otel_log_layer)
.with(fmt_layer)
.init();
let config = Arc::new(ArcSwap::from_pointee(config));
let pgpass = {
use std::path::PathBuf;
let cfg = config.load();
let path = cfg.server.pgpass.clone().unwrap_or_else(|| {
std::env::var("PGPASSFILE").map_or_else(
|_| {
std::env::var("HOME")
.map_or_default(PathBuf::from)
.join(".pgpass")
},
PathBuf::from,
)
});
Arc::new(halephant_core::auth::pgpass::Pgpass::load(&path))
};
let topology = Arc::new(TopologyManager::new(
Arc::clone(&config),
Arc::clone(&pgpass),
));
info!("discovering cluster topology...");
let _ = topology.refresh().await;
let pools = Arc::new(PoolManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::clone(&pgpass),
));
pools.warm_up().await;
let clients = Arc::new(clients::ClientRegistry::new());
let _metrics = halephant_core::o11y::gauges::register_pool_gauges(
&pools,
Arc::clone(&config),
Arc::clone(&topology),
&clients,
);
let topo_loop = Arc::clone(&topology);
let pools_for_topo = Arc::clone(&pools);
tokio::spawn(async move { topo_loop.run_loop(&pools_for_topo).await });
let pools_for_scavenger = Arc::clone(&pools);
tokio::spawn(async move { pools_for_scavenger.run_scavenger().await });
if let Some(addr) = config.load().admin.listen.clone() {
let admin_config = Arc::clone(&config);
let admin_pools = Arc::clone(&pools);
let admin_topo = Arc::clone(&topology);
let admin_clients = Arc::clone(&clients);
tokio::spawn(async move {
if let Err(e) = halephant_core::admin::serve(
&addr,
admin_config,
admin_pools,
admin_topo,
admin_clients,
)
.await
{
error!(%e, "admin API failed");
}
});
}
let listeners = Arc::new(ListenerManager::new(
Arc::clone(&config),
Arc::clone(&topology),
Arc::clone(&pgpass),
));
let auth = Arc::new(Authenticator::new());
let startup_cfg = config.load_full();
let (accept_tx, mut accept_rx) = tokio::sync::mpsc::channel::<(TcpStream, SocketAddr)>(32);
for addr in &startup_cfg.server.listen {
let listener = TcpListener::bind(addr)
.await
.with_context(|| format!("failed to bind to {addr}"))?;
info!(%addr, "listening");
let tx = accept_tx.clone();
let addr = addr.clone();
tokio::spawn(async move {
loop {
match listener.accept().await {
Ok((stream, client_addr)) => {
if tx.send((stream, client_addr)).await.is_err() {
return;
}
}
Err(e) => {
warn!(%addr, %e, "accept failed");
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
}
}
}
});
}
drop(accept_tx);
info!(clusters = startup_cfg.cluster.len(), "halephant ready",);
drop(startup_cfg);
let mut connections = JoinSet::new();
let reload_counter = opentelemetry::global::meter("halephant")
.u64_counter("halephant.config.reloads")
.with_description("SIGHUP config reload attempts, grouped by outcome")
.build();
let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.context("failed to install SIGTERM handler")?;
let mut sighup = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::hangup())
.context("failed to install SIGHUP handler")?;
loop {
tokio::select! {
Some((client_stream, client_addr)) = accept_rx.recv() => {
client_stream.set_nodelay(true)?;
let session_cfg = config.load_full();
let pools = Arc::clone(&pools);
let listeners = Arc::clone(&listeners);
let auth = Arc::clone(&auth);
let clients = Arc::clone(&clients);
connections.spawn(async move {
info!(%client_addr, "accepted");
if let Err(e) = proxy::frontend::forward(
client_stream,
client_addr,
session_cfg.as_ref(),
&pools,
&listeners,
&auth,
&clients,
)
.await
{
error!(%client_addr, %e, "connection error");
}
info!(%client_addr, "closed");
});
}
_ = tokio::signal::ctrl_c() => {
info!("received SIGINT, shutting down");
break;
}
_ = sigterm.recv() => {
info!("received SIGTERM, shutting down");
break;
}
_ = sighup.recv() => {
info!("received SIGHUP, reloading config");
use halephant_core::config::reload::{reload_config, ReloadOutcome};
use opentelemetry::KeyValue;
match reload_config(config_path, &config).await {
ReloadOutcome::Success => {
reload_counter.add(1, &[KeyValue::new("outcome", "success")]);
info!("config reloaded");
let topo = Arc::clone(&topology);
let pools_for_reload = Arc::clone(&pools);
tokio::spawn(async move {
let changed = topo.refresh().await;
for node in &changed {
pools_for_reload.drain_node(node);
}
pools_for_reload.warm_up().await;
});
}
ReloadOutcome::RestartRequired(field) => {
reload_counter.add(
1,
&[KeyValue::new("outcome", "restart_required")],
);
warn!(
field,
"config reload rejected: change to this field requires a process restart"
);
}
ReloadOutcome::ParseFailed(e) => {
reload_counter.add(
1,
&[KeyValue::new("outcome", "parse_failed")],
);
error!(%e, "config reload failed");
}
}
}
}
while connections.try_join_next().is_some() {}
}
pools.shutdown();
let shutdown_timeout = config.load().server.shutdown_timeout;
let active = connections.len();
if active > 0 {
info!(active, timeout = ?shutdown_timeout, "shutting down, draining connections...");
let deadline = tokio::time::sleep(shutdown_timeout);
tokio::pin!(deadline);
loop {
tokio::select! {
result = connections.join_next() => {
if result.is_none() {
break; }
let remaining = connections.len();
if remaining > 0 {
info!(remaining, "draining...");
}
}
() = &mut deadline => {
let remaining = connections.len();
warn!(remaining, "shutdown timeout reached, dropping remaining connections");
connections.abort_all();
break;
}
}
}
}
pools.drain_all();
if let Some(provider) = trace_provider
&& let Err(e) = provider.shutdown()
{
warn!("failed to flush OpenTelemetry spans: {e}");
}
if let Some(provider) = meter_provider
&& let Err(e) = provider.shutdown()
{
warn!("failed to flush OpenTelemetry metrics: {e}");
}
if let Some(provider) = logger_provider
&& let Err(e) = provider.shutdown()
{
warn!("failed to flush OpenTelemetry logs: {e}");
}
info!("halephant stopped");
Ok(())
}
#[cfg(test)]
mod tests {
use super::build_runtime;
fn builder_default_workers() -> usize {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap()
.metrics()
.num_workers()
}
#[test]
fn zero_workers_keeps_the_builder_default() {
let runtime = build_runtime(0).unwrap();
assert_eq!(runtime.metrics().num_workers(), builder_default_workers());
}
#[test]
fn explicit_worker_count_sizes_the_runtime() {
let runtime = build_runtime(3).unwrap();
assert_eq!(runtime.metrics().num_workers(), 3);
}
#[test]
fn one_worker_pins_a_single_thread() {
let runtime = build_runtime(1).unwrap();
assert_eq!(runtime.metrics().num_workers(), 1);
}
#[test]
fn runtime_enables_io_and_time_drivers() {
for workers in [0, 2] {
let runtime = build_runtime(workers).unwrap();
runtime.block_on(async {
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
});
}
}
}