use std::{
sync::{Condvar, Mutex, TryLockError},
time::Duration,
};
use platform::{
force_shutdown,
postmaster_process_is_running as platform_postmaster_process_is_running,
prepare_process_exit_failsafe,
request_shutdown,
};
use postgresql_embedded::Settings;
use crate::{CleanupMode, error::BootstrapResult};
mod cleanup;
mod pidfile;
mod platform;
use pidfile::read_postmaster_process_from_dir;
#[cfg(any(doc, test, feature = "cluster-unit-tests", feature = "dev-worker"))]
pub use pidfile::{
PostmasterPid,
PostmasterProcess,
postmaster_process_is_running,
process_is_running,
read_postmaster_pid,
read_postmaster_process,
};
struct RegisteredShutdownState {
settings: Settings,
shutdown_timeout: Duration,
cleanup_mode: CleanupMode,
_exit_failsafe: platform::ProcessExitFailsafe,
}
enum ShutdownRegistrationState {
Empty,
Registering,
Registered(Box<RegisteredShutdownState>),
}
static SHUTDOWN_STATE: Mutex<ShutdownRegistrationState> =
Mutex::new(ShutdownRegistrationState::Empty);
static SHUTDOWN_STATE_CHANGED: Condvar = Condvar::new();
const POLL_INTERVAL: Duration = Duration::from_millis(50);
const POST_SIGKILL_GRACE: Duration = Duration::from_millis(100);
pub(super) fn register_shutdown_hook(
settings: Settings,
shutdown_timeout: Duration,
cleanup_mode: CleanupMode,
) -> BootstrapResult<()> {
if !reserve_shutdown_registration() {
return Ok(());
}
let state = match build_shutdown_state(settings, shutdown_timeout, cleanup_mode) {
Ok(state) => state,
Err(err) => {
rollback_shutdown_registration();
return Err(err);
}
};
commit_shutdown_registration(state);
log_registration_success();
Ok(())
}
fn reserve_shutdown_registration() -> bool {
let mut guard = SHUTDOWN_STATE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
loop {
match &*guard {
ShutdownRegistrationState::Empty => {
*guard = ShutdownRegistrationState::Registering;
return true;
}
ShutdownRegistrationState::Registered(_) => {
log_already_registered();
return false;
}
ShutdownRegistrationState::Registering => {
guard = SHUTDOWN_STATE_CHANGED
.wait(guard)
.unwrap_or_else(std::sync::PoisonError::into_inner);
}
}
}
}
fn build_shutdown_state(
settings: Settings,
shutdown_timeout: Duration,
cleanup_mode: CleanupMode,
) -> BootstrapResult<RegisteredShutdownState> {
let postmaster_process = read_postmaster_process_from_dir(&settings.data_dir)?;
register_atexit()?;
let exit_failsafe = prepare_process_exit_failsafe(postmaster_process);
Ok(RegisteredShutdownState {
settings,
shutdown_timeout,
cleanup_mode,
_exit_failsafe: exit_failsafe,
})
}
fn commit_shutdown_registration(state: RegisteredShutdownState) {
let mut guard = SHUTDOWN_STATE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*guard = ShutdownRegistrationState::Registered(Box::new(state));
SHUTDOWN_STATE_CHANGED.notify_all();
}
fn rollback_shutdown_registration() {
let mut guard = SHUTDOWN_STATE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*guard = ShutdownRegistrationState::Empty;
SHUTDOWN_STATE_CHANGED.notify_all();
}
#[cfg(all(test, any(unix, feature = "cluster-unit-tests")))]
fn reset_shutdown_registration_for_tests() {
let mut guard = SHUTDOWN_STATE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*guard = ShutdownRegistrationState::Empty;
SHUTDOWN_STATE_CHANGED.notify_all();
}
fn log_already_registered() {
tracing::debug!(
target: crate::observability::LOG_TARGET,
"shutdown hook already registered; skipping duplicate registration"
);
}
fn log_registration_success() {
tracing::debug!(
target: crate::observability::LOG_TARGET,
"registered atexit shutdown hook for PostgreSQL postmaster"
);
}
fn register_atexit() -> BootstrapResult<()> {
let rc = unsafe { libc::atexit(shutdown_callback) };
if rc != 0 {
return Err(color_eyre::eyre::eyre!("libc::atexit registration failed (rc={rc})").into());
}
Ok(())
}
extern "C" fn shutdown_callback() {
let Some(work) = shutdown_work() else {
return;
};
let process = match read_postmaster_process_from_dir(&work.settings.data_dir) {
Ok(Some(process)) => process,
Ok(None) => {
cleanup::best_effort_cleanup(&work);
return;
}
Err(err) => {
tracing::warn!(
target: crate::observability::LOG_TARGET,
%err,
"failed to read postmaster process during shutdown callback"
);
return;
}
};
if !postmaster_process_is_running_for_shutdown(process) {
cleanup::best_effort_cleanup(&work);
return;
}
stop_postmaster(process, &work);
cleanup::best_effort_cleanup(&work);
}
fn shutdown_work() -> Option<ShutdownWork> {
let guard = match SHUTDOWN_STATE.try_lock() {
Ok(guard) => guard,
Err(TryLockError::Poisoned(poisoned)) => {
tracing::warn!(
target: crate::observability::LOG_TARGET,
"shutdown state mutex was poisoned; continuing shutdown cleanup"
);
poisoned.into_inner()
}
Err(TryLockError::WouldBlock) => {
return None;
}
};
let ShutdownRegistrationState::Registered(state) = &*guard else {
return None;
};
Some(ShutdownWork {
settings: state.settings.clone(),
shutdown_timeout: state.shutdown_timeout,
cleanup_mode: state.cleanup_mode,
})
}
struct ShutdownWork {
settings: Settings,
shutdown_timeout: Duration,
cleanup_mode: CleanupMode,
}
fn stop_postmaster(process: platform::PostmasterProcess, state: &ShutdownWork) {
request_shutdown(process);
if wait_for_exit(process, state.shutdown_timeout) {
return;
}
tracing::warn!(
target: crate::observability::LOG_TARGET,
timeout_ms = state.shutdown_timeout.as_millis(),
"postmaster did not exit before shutdown-hook timeout; escalating to forceful termination"
);
force_shutdown(process);
std::thread::sleep(POST_SIGKILL_GRACE);
}
fn wait_for_exit(process: platform::PostmasterProcess, timeout: Duration) -> bool {
let deadline = std::time::Instant::now() + timeout;
loop {
if !postmaster_process_is_running_for_shutdown(process) {
return true;
}
if std::time::Instant::now() >= deadline {
return false;
}
std::thread::sleep(POLL_INTERVAL);
}
}
fn postmaster_process_is_running_for_shutdown(process: platform::PostmasterProcess) -> bool {
platform_postmaster_process_is_running(process).unwrap_or_else(|err| {
tracing::warn!(
target: crate::observability::LOG_TARGET,
%err,
"failed to probe postmaster process during shutdown"
);
true
})
}
#[cfg(all(test, feature = "loom-tests"))]
mod loom_tests;
#[cfg(all(test, feature = "cluster-unit-tests"))]
mod tests;
#[cfg(all(test, unix))]
#[path = "exit_hook_tests.rs"]
mod exit_hook_tests;