use std::{
borrow::Cow,
fmt,
future::Future,
time::{Duration, Instant},
};
use sqlx::{
PgPool, Postgres, Transaction,
migrate::{MigrateDatabase, MigrateError, Migrator},
pool::PoolConnection,
postgres::PgPoolOptions,
};
use thiserror::Error;
use crate::{
db::MigrationFailureKind,
result::{ClassifyError, ErrorDescriptor, ErrorKind},
secret::ExposeSecret,
};
use super::{
config::{
DEFAULT_POSTGRES_ADVISORY_LOCK_TIMEOUT, DEFAULT_POSTGRES_MIGRATION_TIMEOUT,
PostgresBootstrapConfig, PostgresPoolConfig, PostgresStartupRetryConfig,
},
diagnostics::{
PostgresBootstrapError, PostgresOperation, PostgresPoolError, diagnose_migration_error,
diagnose_migration_lock_timeout, diagnose_operation_timeout, diagnose_pool_error,
diagnose_sqlx_error,
},
identity::{database_name, schema_maintenance_advisory_lock_key},
};
pub async fn connect_pool(
database_url: &str,
config: PostgresPoolConfig,
) -> Result<PgPool, PostgresPoolError> {
Ok(pool_options(config).connect(database_url).await?)
}
pub fn connect_lazy_pool(
database_url: &str,
config: PostgresPoolConfig,
) -> Result<PgPool, PostgresPoolError> {
Ok(pool_options(config).connect_lazy(database_url)?)
}
#[must_use]
pub fn pool_options(config: PostgresPoolConfig) -> PgPoolOptions {
let mut options = PgPoolOptions::new()
.max_connections(config.max_connections)
.min_connections(config.min_connections)
.acquire_timeout(config.acquire_timeout)
.test_before_acquire(config.test_before_acquire);
if let Some(idle_timeout) = config.idle_timeout {
options = options.idle_timeout(idle_timeout);
}
if let Some(max_lifetime) = config.max_lifetime {
options = options.max_lifetime(max_lifetime);
}
options
}
pub async fn bootstrap_database(
config: PostgresBootstrapConfig,
migrator: Option<&Migrator>,
) -> Result<PgPool, PostgresBootstrapError> {
let migrator = if config.migrate_on_startup {
Some(migrator.ok_or(PostgresBootstrapError::MissingMigrator)?)
} else {
None
};
let database_url = config.database_url.expose_secret();
if config.create_on_startup {
tracing::info!(
database = %display_database_name(database_url),
"ensuring PostgreSQL database exists"
);
ensure_database_exists_with_retry(database_url, config.startup_retry)
.await
.map_err(|source| PostgresBootstrapError::EnsureDatabase {
diagnosis: diagnose_sqlx_error(
PostgresOperation::EnsureDatabase,
database_url,
&source,
),
source: Box::new(source),
})?;
}
let pool = connect_pool_with_retry(database_url, config.pool_config, config.startup_retry)
.await
.map_err(|source| PostgresBootstrapError::Connect {
diagnosis: diagnose_pool_error(PostgresOperation::Connect, database_url, &source),
source: Box::new(source),
})?;
if let Some(migrator) = migrator {
tracing::info!(
database = %display_database_name(database_url),
"running PostgreSQL database migrations"
);
if let Err(error) = run_migrations_for_database(
&pool,
migrator,
database_name(database_url).as_deref(),
config.migration_lock_key,
config.migration_lock_timeout,
config.migration_timeout,
)
.await
{
return Err(bootstrap_migration_run_error(database_url, error));
}
}
Ok(pool)
}
pub async fn ensure_database_exists(database_url: &str) -> Result<(), sqlx::Error> {
if Postgres::database_exists(database_url).await? {
return Ok(());
}
match Postgres::create_database(database_url).await {
Ok(()) => Ok(()),
Err(error) => match Postgres::database_exists(database_url).await {
Ok(true) => Ok(()),
Ok(false) | Err(_) => Err(error),
},
}
}
pub async fn ensure_database_exists_with_retry(
database_url: &str,
retry: PostgresStartupRetryConfig,
) -> Result<(), sqlx::Error> {
retry_async(retry, "postgres_ensure_database_failed", || {
ensure_database_exists(database_url)
})
.await
}
pub async fn run_migrations(
pool: &PgPool,
migrator: &Migrator,
) -> Result<(), PostgresMigrationRunError> {
run_migrations_with_timeouts(
pool,
migrator,
None,
DEFAULT_POSTGRES_ADVISORY_LOCK_TIMEOUT,
DEFAULT_POSTGRES_MIGRATION_TIMEOUT,
)
.await
}
#[derive(Error)]
#[non_exhaustive]
pub enum PostgresMigrationRunError {
#[error("failed to acquire PostgreSQL schema-maintenance lock for migrations")]
Lock(#[source] sqlx::Error),
#[error("timed out waiting for PostgreSQL schema-maintenance lock after {timeout:?}")]
LockTimeout {
timeout: Duration,
},
#[error("failed to run PostgreSQL migrations")]
Migration {
kind: MigrationFailureKind,
#[source]
source: Box<MigrateError>,
},
#[error("timed out while running PostgreSQL migrations after {timeout:?}")]
MigrationTimeout {
timeout: Duration,
},
}
impl fmt::Debug for PostgresMigrationRunError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("PostgresMigrationRunError")
.field("error_code", &self.error_code())
.field("source", &"[REDACTED]")
.finish()
}
}
impl ClassifyError for PostgresMigrationRunError {
fn error_kind(&self) -> ErrorKind {
match self {
Self::LockTimeout { .. } | Self::MigrationTimeout { .. } => ErrorKind::Timeout,
Self::Lock(_) | Self::Migration { .. } => ErrorKind::Dependency,
}
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(match self {
Self::Lock(_) => "postgres_migration_lock_failed",
Self::LockTimeout { .. } => "postgres_migration_lock_timeout",
Self::Migration { .. } => "postgres_migration_failed",
Self::MigrationTimeout { .. } => "postgres_migration_timeout",
})
}
fn error_descriptor(&self) -> ErrorDescriptor {
ErrorDescriptor::new(self.error_kind(), self.error_code(), self.to_string()).with_retryable(
matches!(
self,
Self::Lock(_) | Self::LockTimeout { .. } | Self::MigrationTimeout { .. }
),
)
}
}
impl From<PostgresMigrationRunError> for PostgresBootstrapError {
fn from(error: PostgresMigrationRunError) -> Self {
bootstrap_migration_run_error("", error)
}
}
fn bootstrap_migration_run_error(
database_url: &str,
error: PostgresMigrationRunError,
) -> PostgresBootstrapError {
match error {
PostgresMigrationRunError::Lock(source) => PostgresBootstrapError::MigrationLock {
diagnosis: diagnose_sqlx_error(PostgresOperation::MigrationLock, database_url, &source),
source: Box::new(source),
},
PostgresMigrationRunError::LockTimeout { timeout } => {
PostgresBootstrapError::MigrationLockTimeout {
diagnosis: diagnose_migration_lock_timeout(database_url, timeout),
timeout,
}
}
PostgresMigrationRunError::Migration { kind, source } => {
PostgresBootstrapError::Migration {
diagnosis: diagnose_migration_error(database_url, &source),
kind,
source,
}
}
PostgresMigrationRunError::MigrationTimeout { timeout } => {
PostgresBootstrapError::MigrationTimeout {
diagnosis: diagnose_operation_timeout(
PostgresOperation::Migrate,
database_url,
timeout,
),
timeout,
}
}
}
}
pub async fn run_migrations_with_advisory_lock(
pool: &PgPool,
migrator: &Migrator,
lock_key: i64,
) -> Result<(), PostgresMigrationRunError> {
run_migrations_with_advisory_lock_timeout(
pool,
migrator,
lock_key,
DEFAULT_POSTGRES_ADVISORY_LOCK_TIMEOUT,
)
.await
}
pub async fn run_migrations_with_advisory_lock_timeout(
pool: &PgPool,
migrator: &Migrator,
lock_key: i64,
lock_timeout: Duration,
) -> Result<(), PostgresMigrationRunError> {
run_migrations_with_timeouts(
pool,
migrator,
Some(lock_key),
lock_timeout,
DEFAULT_POSTGRES_MIGRATION_TIMEOUT,
)
.await
}
pub async fn run_migrations_with_timeouts(
pool: &PgPool,
migrator: &Migrator,
additional_lock_key: Option<i64>,
lock_timeout: Duration,
migration_timeout: Duration,
) -> Result<(), PostgresMigrationRunError> {
run_migrations_for_database(
pool,
migrator,
None,
additional_lock_key,
lock_timeout,
migration_timeout,
)
.await
}
async fn run_migrations_for_database(
pool: &PgPool,
migrator: &Migrator,
known_database_name: Option<&str>,
additional_lock_key: Option<i64>,
lock_timeout: Duration,
migration_timeout: Duration,
) -> Result<(), PostgresMigrationRunError> {
if migration_timeout.is_zero() || Instant::now().checked_add(migration_timeout).is_none() {
return Err(PostgresMigrationRunError::MigrationTimeout {
timeout: migration_timeout,
});
}
let mut lock_guard = acquire_schema_maintenance_session_lock(
pool,
known_database_name,
additional_lock_key,
lock_timeout,
)
.await
.map_err(map_migration_lock_error)?;
let migration_result = tokio::time::timeout(
migration_timeout,
migrator.run(&mut **lock_guard.connection()),
)
.await;
match migration_result {
Ok(Ok(())) => lock_guard
.release_with_timeout(lock_timeout)
.await
.map_err(PostgresMigrationRunError::Lock),
Ok(Err(source)) => Err(PostgresMigrationRunError::Migration {
kind: MigrationFailureKind::from_error(&source),
source: Box::new(source),
}),
Err(_) => Err(PostgresMigrationRunError::MigrationTimeout {
timeout: migration_timeout,
}),
}
}
fn map_migration_lock_error(error: SessionAdvisoryLockAcquireError) -> PostgresMigrationRunError {
match error {
SessionAdvisoryLockAcquireError::Sqlx(source) => PostgresMigrationRunError::Lock(source),
SessionAdvisoryLockAcquireError::Timeout { timeout } => {
PostgresMigrationRunError::LockTimeout { timeout }
}
}
}
#[derive(Debug)]
pub(super) enum SessionAdvisoryLockAcquireError {
Sqlx(sqlx::Error),
Timeout { timeout: Duration },
}
pub(super) struct SessionAdvisoryLockGuard {
connection: PoolConnection<Postgres>,
lock_keys: Vec<i64>,
armed: bool,
}
impl SessionAdvisoryLockGuard {
fn new(connection: PoolConnection<Postgres>) -> Self {
Self {
connection,
lock_keys: Vec::with_capacity(2),
armed: false,
}
}
pub(super) fn connection(&mut self) -> &mut PoolConnection<Postgres> {
&mut self.connection
}
pub(super) async fn release_with_timeout(
mut self,
timeout: Duration,
) -> Result<(), sqlx::Error> {
let started = Instant::now();
while let Some(lock_key) = self.lock_keys.pop() {
let remaining = timeout
.checked_sub(started.elapsed())
.filter(|remaining| !remaining.is_zero())
.ok_or(sqlx::Error::PoolTimedOut)?;
let result: Result<bool, sqlx::Error> = tokio::time::timeout(
remaining,
sqlx::query_scalar("SELECT pg_advisory_unlock($1)")
.bind(lock_key)
.fetch_one(&mut *self.connection),
)
.await
.map_err(|_| sqlx::Error::PoolTimedOut)?;
match result {
Ok(true) => {}
Ok(false) => {
return Err(sqlx::Error::Protocol(
"PostgreSQL advisory lock ownership was lost before release".to_owned(),
));
}
Err(error) => return Err(error),
}
}
self.armed = false;
Ok(())
}
}
impl Drop for SessionAdvisoryLockGuard {
fn drop(&mut self) {
if self.armed {
self.connection.close_on_drop();
}
}
}
pub(super) async fn acquire_schema_maintenance_session_lock(
pool: &PgPool,
known_database_name: Option<&str>,
additional_lock_key: Option<i64>,
timeout: Duration,
) -> Result<SessionAdvisoryLockGuard, SessionAdvisoryLockAcquireError> {
if timeout.is_zero() || Instant::now().checked_add(timeout).is_none() {
return Err(SessionAdvisoryLockAcquireError::Timeout { timeout });
}
let started = Instant::now();
let remaining = remaining_timeout(started, timeout)?;
let connection = tokio::time::timeout(remaining, pool.acquire())
.await
.map_err(|_| SessionAdvisoryLockAcquireError::Timeout { timeout })?
.map_err(SessionAdvisoryLockAcquireError::Sqlx)?;
let mut guard = SessionAdvisoryLockGuard::new(connection);
let resolved_database_name = if let Some(name) = known_database_name {
name.to_owned()
} else {
let remaining = remaining_timeout(started, timeout)?;
guard.armed = true;
let name: String = tokio::time::timeout(
remaining,
sqlx::query_scalar("SELECT current_database()").fetch_one(&mut **guard.connection()),
)
.await
.map_err(|_| SessionAdvisoryLockAcquireError::Timeout { timeout })?
.map_err(SessionAdvisoryLockAcquireError::Sqlx)?;
guard.armed = false;
name
};
let lock_keys = schema_maintenance_lock_keys(&resolved_database_name, additional_lock_key);
acquire_locks_on_guard(guard, &lock_keys, started, timeout).await
}
pub(super) fn schema_maintenance_lock_keys(
database_name: &str,
additional_lock_key: Option<i64>,
) -> Vec<i64> {
let maintenance_key = schema_maintenance_advisory_lock_key(database_name);
let mut lock_keys = Vec::with_capacity(2);
lock_keys.push(maintenance_key);
if let Some(additional_key) = additional_lock_key
&& additional_key != maintenance_key
{
lock_keys.push(additional_key);
}
lock_keys
}
async fn acquire_locks_on_guard(
mut guard: SessionAdvisoryLockGuard,
lock_keys: &[i64],
started: Instant,
timeout: Duration,
) -> Result<SessionAdvisoryLockGuard, SessionAdvisoryLockAcquireError> {
for &lock_key in lock_keys {
if guard.lock_keys.contains(&lock_key) {
continue;
}
loop {
let remaining = remaining_timeout(started, timeout)?;
guard.armed = true;
let acquired = tokio::time::timeout(
remaining,
sqlx::query_scalar("SELECT pg_try_advisory_lock($1)")
.bind(lock_key)
.fetch_one(&mut **guard.connection()),
)
.await
.map_err(|_| SessionAdvisoryLockAcquireError::Timeout { timeout })?
.map_err(SessionAdvisoryLockAcquireError::Sqlx)?;
if acquired {
guard.lock_keys.push(lock_key);
break;
}
guard.armed = !guard.lock_keys.is_empty();
let remaining = remaining_timeout(started, timeout)?;
tokio::time::sleep(remaining.min(Duration::from_millis(50))).await;
}
}
Ok(guard)
}
fn remaining_timeout(
started: Instant,
timeout: Duration,
) -> Result<Duration, SessionAdvisoryLockAcquireError> {
timeout
.checked_sub(started.elapsed())
.filter(|remaining| !remaining.is_zero())
.ok_or(SessionAdvisoryLockAcquireError::Timeout { timeout })
}
pub async fn ping_pool(pool: &PgPool) -> Result<(), sqlx::Error> {
sqlx::query("SELECT 1").execute(pool).await?;
Ok(())
}
pub async fn connect_pool_with_retry(
database_url: &str,
config: PostgresPoolConfig,
retry: PostgresStartupRetryConfig,
) -> Result<PgPool, PostgresPoolError> {
retry_async(retry, "postgres_connect_failed", || {
connect_pool(database_url, config)
})
.await
}
pub async fn check_pool_health(pool: &PgPool) -> PostgresHealth {
let started = Instant::now();
match ping_pool(pool).await {
Ok(()) => PostgresHealth {
status: PostgresHealthStatus::Ready,
latency: started.elapsed(),
},
Err(_) => PostgresHealth {
status: PostgresHealthStatus::Unavailable,
latency: started.elapsed(),
},
}
}
pub(super) const POSTGRES_HEALTH_ERROR: &str = "database health check failed";
fn display_database_name(database_url: &str) -> Cow<'_, str> {
database_name(database_url)
.map(Cow::Owned)
.unwrap_or(Cow::Borrowed("<unknown>"))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum PostgresHealthStatus {
Ready,
Unavailable,
}
#[derive(Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct PostgresHealth {
pub(super) status: PostgresHealthStatus,
pub(super) latency: Duration,
}
impl fmt::Debug for PostgresHealth {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("PostgresHealth")
.field("status", &self.status)
.field("latency", &self.latency)
.field("has_error", &!self.ready())
.finish()
}
}
impl PostgresHealth {
#[must_use]
pub const fn status(&self) -> PostgresHealthStatus {
self.status
}
#[must_use]
pub const fn latency(&self) -> Duration {
self.latency
}
#[must_use]
pub fn error(&self) -> Option<&str> {
(!self.ready()).then_some(POSTGRES_HEALTH_ERROR)
}
#[must_use]
pub const fn ready(&self) -> bool {
matches!(self.status, PostgresHealthStatus::Ready)
}
}
async fn retry_async<T, E, F, Fut>(
retry: PostgresStartupRetryConfig,
error_code: &'static str,
mut operation: F,
) -> Result<T, E>
where
F: FnMut() -> Fut,
Fut: Future<Output = Result<T, E>>,
{
let max_attempts = retry.max_attempts.max(1);
let mut attempt = 1_u32;
let mut delay = if retry.initial_delay.is_zero() {
PostgresStartupRetryConfig::default().initial_delay
} else {
retry.initial_delay
};
let max_delay = retry.max_delay.max(delay);
loop {
match operation().await {
Ok(value) => return Ok(value),
Err(error) if attempt >= max_attempts => return Err(error),
Err(_) => {
tracing::warn!(
attempt,
max_attempts,
retry_after_ms = delay.as_millis(),
error_code,
"PostgreSQL startup operation failed; retrying"
);
tokio::time::sleep(delay).await;
attempt = attempt.saturating_add(1);
delay = delay.checked_mul(2).unwrap_or(Duration::MAX).min(max_delay);
}
}
}
}
pub type PostgresTransaction<'a> = Transaction<'a, Postgres>;
pub async fn begin_transaction(pool: &PgPool) -> Result<PostgresTransaction<'_>, sqlx::Error> {
pool.begin().await
}
pub async fn commit_transaction(transaction: PostgresTransaction<'_>) -> Result<(), sqlx::Error> {
transaction.commit().await
}
pub async fn rollback_transaction(transaction: PostgresTransaction<'_>) -> Result<(), sqlx::Error> {
transaction.rollback().await
}
pub async fn acquire_advisory_transaction_lock(
transaction: &mut PostgresTransaction<'_>,
key: i64,
) -> Result<(), sqlx::Error> {
sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(key)
.execute(&mut **transaction)
.await?;
Ok(())
}
pub async fn acquire_shared_advisory_transaction_lock(
transaction: &mut PostgresTransaction<'_>,
key: i64,
) -> Result<(), sqlx::Error> {
sqlx::query("SELECT pg_advisory_xact_lock_shared($1)")
.bind(key)
.execute(&mut **transaction)
.await?;
Ok(())
}
pub async fn try_acquire_advisory_transaction_lock(
transaction: &mut PostgresTransaction<'_>,
key: i64,
) -> Result<bool, sqlx::Error> {
sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)")
.bind(key)
.fetch_one(&mut **transaction)
.await
}