use crate::resource::{HealthState, MIN_HEALTH_INTERVAL, Resource};
use crate::resource_lifecycle::{
admit_health_tasks, run_initial_health_checks, shutdown_resources,
};
use crate::runtime_state::{
RuntimeConfig, RuntimeContextGuard, RuntimeInner, drain_root_scope, install_runtime,
stop_cancel_watcher, teardown_runtime,
};
use crate::runtime_test_support::{RuntimeController, RuntimeSchedule};
use crate::tls::CertStore;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
pub use tokio::runtime::Handle as TokioHandle;
pub(crate) use crate::runtime_state::{
cancel_channel, check_cancel, has_runtime, runtime_context, try_current_runtime,
};
pub use crate::runtime_state::{
block_on, is_shutting_down, on_cancel, request_shutdown, tokio_handle,
};
impl std::fmt::Debug for RuntimeBuilder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RuntimeBuilder")
.field("worker_threads", &self.config.worker_threads)
.field("shutdown_timeout", &self.config.shutdown_timeout)
.field("keepalive_timeout", &self.config.keepalive_timeout)
.field("tracing_enabled", &self.config.tracing_enabled)
.field("metrics_enabled", &self.config.metrics_enabled)
.field("health_interval", &self.config.health_interval)
.field("connection_limit", &self.config.connection_limit)
.field("resource_count", &self.resources.len())
.field("has_tls", &self.has_manual_tls())
.finish_non_exhaustive()
}
}
pub struct RuntimeBuilder {
config: RuntimeConfig,
test_schedule: Option<Arc<RuntimeSchedule>>,
resources: Vec<Box<dyn Resource>>,
tls_cert_path: Option<Box<std::path::Path>>,
tls_key_path: Option<Box<std::path::Path>>,
tls_cert_store: Option<CertStore>,
#[cfg(feature = "acme")]
acme_config: Option<crate::acme::AcmeConfig>,
#[cfg(feature = "dns01")]
dns01_setup: Option<crate::dns01::Dns01Setup>,
#[cfg(feature = "otel")]
otel_endpoint: Option<Box<str>>,
}
impl RuntimeBuilder {
fn new() -> Self {
Self {
config: RuntimeConfig::default(),
test_schedule: None,
resources: Vec::new(),
tls_cert_path: None,
tls_key_path: None,
tls_cert_store: None,
#[cfg(feature = "acme")]
acme_config: None,
#[cfg(feature = "dns01")]
dns01_setup: None,
#[cfg(feature = "otel")]
otel_endpoint: None,
}
}
pub fn worker_threads(mut self, n: usize) -> Self {
self.config.worker_threads = n;
self
}
pub fn shutdown_timeout(mut self, timeout: Duration) -> Self {
const MIN: Duration = Duration::from_millis(100);
self.config.shutdown_timeout =
crate::time::clamp_duration(timeout, MIN, "shutdown_timeout");
self
}
pub fn keepalive_timeout(mut self, timeout: Duration) -> Self {
const MIN: Duration = Duration::from_millis(100);
self.config.keepalive_timeout =
crate::time::clamp_duration(timeout, MIN, "keepalive_timeout");
self
}
pub fn health_interval(mut self, interval: Duration) -> Self {
self.config.health_interval = interval.max(MIN_HEALTH_INTERVAL);
self
}
pub fn connection_limit(mut self, n: usize) -> Self {
self.config.connection_limit = Some(n);
self
}
#[doc(hidden)]
pub fn with_test_schedule(mut self, controller: &RuntimeController) -> Self {
self.test_schedule = Some(controller.schedule());
self
}
pub fn resource(mut self, r: impl Resource) -> Self {
self.resources.push(Box::new(r));
self
}
pub fn with_tracing(mut self) -> Self {
self.config.tracing_enabled = true;
self
}
pub fn with_metrics(mut self) -> Self {
self.config.metrics_enabled = true;
self
}
#[cfg(feature = "profiling")]
pub fn with_profiling(mut self) -> Self {
self.config.profiling_enabled = true;
self
}
#[cfg(feature = "otel")]
pub fn otel_endpoint(mut self, url: &str) -> Self {
self.otel_endpoint = Some(Box::from(url));
self
}
pub fn tls_cert(mut self, path: &std::path::Path) -> Self {
self.tls_cert_path = Some(Box::from(path));
self
}
pub fn tls_key(mut self, path: &std::path::Path) -> Self {
self.tls_key_path = Some(Box::from(path));
self
}
pub fn tls_resolver(mut self, store: CertStore) -> Self {
self.tls_cert_store = Some(store);
self
}
#[cfg(feature = "acme")]
pub fn tls_auto(mut self, config: crate::acme::AcmeConfig) -> Self {
self.acme_config = Some(config);
self
}
#[cfg(feature = "dns01")]
pub fn tls_auto_dns01(
mut self,
acme: crate::dns01::AcmeDns01,
api_token: Box<str>,
domain: Box<str>,
) -> Self {
self.dns01_setup = Some(crate::dns01::Dns01Setup {
acme,
api_token,
domain,
});
self
}
pub fn run<F, T>(self, f: F) -> Result<T, crate::RuntimeError>
where
F: FnOnce() -> T,
{
reject_nested_runtime()?;
self.validate_tls_options()?;
if self.config.worker_threads == 0 {
return Err(crate::RuntimeError::InvalidArgument(
"worker_threads must be at least 1".into(),
));
}
if self.config.connection_limit == Some(0) {
return Err(crate::RuntimeError::InvalidArgument(
"connection_limit must be at least 1".into(),
));
}
let mut config = self.config;
let (tls_cfg, store) = crate::tls::resolve_tls(
self.tls_cert_store,
self.tls_cert_path.map(std::path::PathBuf::from),
self.tls_key_path.map(std::path::PathBuf::from),
)?;
config.tls_config = tls_cfg;
config.cert_store = store;
#[cfg(feature = "acme")]
let acme_state = match self.acme_config {
Some(acme_cfg) => {
let (tls_cfg, state) = acme_cfg.build()?;
config.tls_config = Some(tls_cfg);
Some(state)
}
None => None,
};
run_inner_impl(
config,
self.test_schedule,
self.resources.into(),
f,
#[cfg(feature = "acme")]
acme_state,
#[cfg(feature = "dns01")]
self.dns01_setup,
#[cfg(feature = "otel")]
self.otel_endpoint,
)
}
fn has_manual_tls(&self) -> bool {
self.tls_cert_path.is_some() || self.tls_key_path.is_some() || self.tls_cert_store.is_some()
}
fn validate_tls_options(&self) -> Result<(), crate::RuntimeError> {
let has_manual = self.has_manual_tls();
#[cfg(feature = "acme")]
let has_acme = self.acme_config.is_some();
#[cfg(not(feature = "acme"))]
let has_acme = false;
#[cfg(feature = "dns01")]
let has_dns01 = self.dns01_setup.is_some();
#[cfg(not(feature = "dns01"))]
let has_dns01 = false;
match (has_acme, has_dns01, has_manual) {
(true, true, _) => Err(crate::RuntimeError::Tls(
"tls_auto and tls_auto_dns01 are mutually exclusive".into(),
)),
(true, _, true) => Err(crate::RuntimeError::Tls(
"tls_auto and tls_cert/tls_key are mutually exclusive".into(),
)),
(_, true, true) => Err(crate::RuntimeError::Tls(
"tls_auto_dns01 and tls_cert/tls_key are mutually exclusive".into(),
)),
_ => Ok(()),
}
}
}
pub fn builder() -> RuntimeBuilder {
RuntimeBuilder::new()
}
pub fn test<F, T>(f: F) -> Result<T, crate::RuntimeError>
where
F: FnOnce() -> T,
{
let mut builder = RuntimeBuilder::new();
builder.config = test_runtime_config();
builder.run(f)
}
fn test_runtime_config() -> RuntimeConfig {
RuntimeConfig {
worker_threads: tokio_default_worker_threads(),
keepalive_timeout: Duration::from_millis(100),
shutdown_timeout: Duration::from_secs(1),
..RuntimeConfig::default()
}
}
#[doc(hidden)]
pub fn __test_async<F, Fut, T>(f: F) -> Result<T, crate::RuntimeError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
try_test_async(f)
}
fn try_test_async<F, Fut, T>(f: F) -> Result<T, crate::RuntimeError>
where
F: FnOnce() -> Fut,
Fut: std::future::Future<Output = T>,
{
reject_nested_runtime()?;
let config = test_runtime_config();
let tokio_rt = build_executor(config.worker_threads)?;
let (inner, context) =
establish_runtime(Some(tokio_rt.handle().clone()), config, None, None, None);
let scoped = run_scoped(&tokio_rt, &inner, f);
let drain = close_and_drain(&inner, &tokio_rt);
finish_runtime(&inner, tokio_rt, context, drain, None, scoped)
}
fn tokio_default_worker_threads() -> usize {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
}
fn build_executor(worker_threads: usize) -> Result<tokio::runtime::Runtime, crate::RuntimeError> {
let executor = tokio::runtime::Builder::new_multi_thread()
.worker_threads(worker_threads)
.enable_all()
.build()?;
Ok(executor)
}
pub(crate) fn establish_runtime(
tokio_handle: Option<TokioHandle>,
config: RuntimeConfig,
test_schedule: Option<Arc<RuntimeSchedule>>,
metrics_handle: Option<metrics_exporter_prometheus::PrometheusHandle>,
health_state: Option<HealthState>,
) -> (Arc<RuntimeInner>, RuntimeContextGuard) {
let mut inner = RuntimeInner::with_config_and_schedule(config, test_schedule);
inner.tokio_handle = tokio_handle;
inner.metrics_handle = metrics_handle;
inner.health_state = health_state;
let inner = Arc::new(inner);
inner.publish_to_test_schedule();
let context = install_runtime(Arc::clone(&inner));
(inner, context)
}
fn run_scoped<B, Fut>(
tokio_rt: &tokio::runtime::Runtime,
inner: &Arc<RuntimeInner>,
body: B,
) -> ScopedOutcome<Fut::Output>
where
B: FnOnce() -> Fut,
Fut: std::future::Future,
{
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
tokio_rt.block_on(crate::runtime_state::scope_runtime(
Arc::clone(inner),
body(),
))
}))
}
type ScopedOutcome<T> = Result<T, Box<dyn std::any::Any + Send>>;
fn close_and_drain(
inner: &RuntimeInner,
tokio_rt: &tokio::runtime::Runtime,
) -> Option<crate::RuntimeError> {
inner.close_scope();
drain_root_scope(inner, tokio_rt.handle())
}
fn runtime_failure(
inner: &RuntimeInner,
resource_failure: Option<crate::RuntimeError>,
drain: Option<crate::RuntimeError>,
) -> Option<crate::RuntimeError> {
let panic = select_runtime_panic(inner.take_internal_panic(), resource_failure);
match (panic, drain) {
(Some(panicked), Some(displaced)) => {
tracing::warn!(%displaced, "drain timeout displaced by a recorded runtime panic");
Some(panicked)
}
(panicked, drain) => panicked.or(drain),
}
}
fn select_runtime_panic(
internal: Option<crate::RuntimeError>,
resource: Option<crate::RuntimeError>,
) -> Option<crate::RuntimeError> {
match (internal, resource) {
(Some(primary), Some(displaced)) => {
tracing::warn!(%displaced, "resource panic displaced by an internal runtime panic");
Some(primary)
}
(panic, None) | (None, panic) => panic,
}
}
pub fn run<F, T>(f: F) -> Result<T, crate::RuntimeError>
where
F: FnOnce() -> T,
{
reject_nested_runtime()?;
run_inner_impl(
RuntimeConfig::default(),
None,
Vec::new().into(),
f,
#[cfg(feature = "acme")]
None,
#[cfg(feature = "dns01")]
None,
#[cfg(feature = "otel")]
None,
)
}
fn run_inner_impl<F, T>(
config: RuntimeConfig,
test_schedule: Option<Arc<RuntimeSchedule>>,
resources: Arc<[Box<dyn Resource>]>,
f: F,
#[cfg(feature = "acme")] acme_state: Option<crate::acme::AcmeState<std::io::Error>>,
#[cfg(feature = "dns01")] dns01_setup: Option<crate::dns01::Dns01Setup>,
#[cfg(feature = "otel")] otel_endpoint: Option<Box<str>>,
) -> Result<T, crate::RuntimeError>
where
F: FnOnce() -> T,
{
let metrics_handle = install_metrics(config.metrics_enabled)?;
let tokio_rt = build_executor(config.worker_threads)?;
#[cfg(feature = "dns01")]
let (config, dns01_renewal) = provision_dns01(&tokio_rt, config, dns01_setup)?;
#[cfg(feature = "otel")]
if let Some(endpoint) = otel_endpoint {
crate::http::otel::init_exporter(&endpoint)?;
}
let health_state = build_health_state(&resources);
let health_interval = config.health_interval;
let (inner, context) = establish_runtime(
Some(tokio_rt.handle().clone()),
config,
test_schedule,
metrics_handle,
health_state.clone(),
);
let runtime_scope = || async {
if let Some(ref hs) = health_state {
run_initial_health_checks(&resources, hs).await;
}
admit_owned_subsystems(
&inner,
&resources,
&health_state,
health_interval,
#[cfg(feature = "acme")]
acme_state,
#[cfg(feature = "dns01")]
dns01_renewal,
);
f()
};
let scoped = run_scoped(&tokio_rt, &inner, runtime_scope);
let drain = close_and_drain(&inner, &tokio_rt);
let resource_failure = shutdown_runtime_services(&inner, &tokio_rt, &resources);
finish_runtime(&inner, tokio_rt, context, drain, resource_failure, scoped)
}
fn shutdown_runtime_services(
inner: &RuntimeInner,
tokio_rt: &tokio::runtime::Runtime,
resources: &[Box<dyn Resource>],
) -> Option<crate::RuntimeError> {
stop_cancel_watcher(inner, tokio_rt.handle());
let resource_failure = shutdown_resources(resources);
#[cfg(feature = "otel")]
crate::http::otel::shutdown_exporter();
resource_failure
}
#[cfg(feature = "dns01")]
fn provision_dns01(
tokio_rt: &tokio::runtime::Runtime,
config: RuntimeConfig,
setup: Option<crate::dns01::Dns01Setup>,
) -> Result<(RuntimeConfig, Option<Dns01Renewal>), crate::RuntimeError> {
let setup = match setup {
Some(setup) => setup,
None => return Ok((config, None)),
};
let state = tokio_rt.block_on(crate::dns01::init_dns01(setup))?;
let mut config = config;
config.tls_config = Some(state.tls_config);
config.cert_store = Some(state.store.clone());
Ok((
config,
Some(Dns01Renewal {
acme: state.acme,
provider: state.provider,
store: state.store,
}),
))
}
#[cfg(feature = "dns01")]
struct Dns01Renewal {
acme: crate::dns01::AcmeDns01,
provider: crate::dns01::CloudflareProvider,
store: CertStore,
}
fn build_health_state(resources: &[Box<dyn Resource>]) -> Option<HealthState> {
match resources.is_empty() {
true => None,
false => Some(
resources
.iter()
.map(|r| (Box::from(r.name()), AtomicBool::new(true)))
.collect(),
),
}
}
fn admit_owned_subsystems(
inner: &Arc<RuntimeInner>,
resources: &Arc<[Box<dyn Resource>]>,
health_state: &Option<HealthState>,
health_interval: Duration,
#[cfg(feature = "acme")] acme_state: Option<crate::acme::AcmeState<std::io::Error>>,
#[cfg(feature = "dns01")] dns01_renewal: Option<Dns01Renewal>,
) {
drop(admit_signal_watcher(inner));
#[cfg(feature = "acme")]
if let Some(state) = acme_state {
drop(crate::task::admit_signalled_subsystem_on(
inner,
"acme renewal",
move |signals| crate::acme::acme_renewal_loop(state, signals),
));
}
#[cfg(feature = "dns01")]
if let Some(renewal) = dns01_renewal {
drop(crate::task::admit_signalled_subsystem_on(
inner,
"dns01 renewal",
move |signals| {
crate::dns01::dns01_renewal_loop(
renewal.acme,
renewal.provider,
renewal.store,
signals,
)
},
));
}
admit_health_tasks(inner, resources, health_state, health_interval);
}
pub(crate) fn admit_signal_watcher(inner: &Arc<RuntimeInner>) -> Result<(), crate::RuntimeError> {
let requested = Arc::clone(inner);
crate::task::admit_signalled_subsystem_on(inner, "signal watcher", move |signals| {
crate::signals::signal_watcher_loop(
crate::signals::SignalSources::register(),
crate::signals::ShutdownRequest::Runtime(requested),
signals,
)
})
}
fn finish_runtime<T>(
inner: &RuntimeInner,
tokio_rt: tokio::runtime::Runtime,
runtime_guard: RuntimeContextGuard,
drain: Option<crate::RuntimeError>,
resource_failure: Option<crate::RuntimeError>,
scoped: ScopedOutcome<T>,
) -> Result<T, crate::RuntimeError> {
teardown_runtime(inner);
drop(runtime_guard);
tokio_rt.shutdown_timeout(inner.config.shutdown_timeout);
match (scoped, runtime_failure(inner, resource_failure, drain)) {
(Ok(value), None) => Ok(value),
(Ok(_), Some(error)) => Err(error),
(Err(payload), failure) => resume_past_failure(failure, payload),
}
}
fn resume_past_failure(
failure: Option<crate::RuntimeError>,
payload: Box<dyn std::any::Any + Send>,
) -> ! {
if let Some(error) = failure {
tracing::error!(%error, "runtime failure displaced by an unwinding closure");
}
std::panic::resume_unwind(payload)
}
fn reject_nested_runtime() -> Result<(), crate::RuntimeError> {
match (has_runtime(), tokio::runtime::Handle::try_current().is_ok()) {
(false, false) => Ok(()),
_ => Err(crate::RuntimeError::InvalidArgument(
"nested runtime creation is not supported".into(),
)),
}
}
fn init_prometheus_recorder() -> Result<metrics_exporter_prometheus::PrometheusHandle, Box<str>> {
let recorder = metrics_exporter_prometheus::PrometheusBuilder::new().build_recorder();
let handle = recorder.handle();
match metrics::set_global_recorder(recorder) {
Ok(()) => Ok(handle),
Err(error) => {
Err(format!("global metrics recorder is already installed: {error}").into_boxed_str())
}
}
}
fn shared_metrics_handle()
-> Result<Option<metrics_exporter_prometheus::PrometheusHandle>, crate::RuntimeError> {
static HANDLE: std::sync::OnceLock<
Result<metrics_exporter_prometheus::PrometheusHandle, Box<str>>,
> = std::sync::OnceLock::new();
match HANDLE.get_or_init(init_prometheus_recorder) {
Ok(handle) => Ok(Some(handle.clone())),
Err(reason) => Err(crate::RuntimeError::Config(reason.clone())),
}
}
fn install_metrics(
enabled: bool,
) -> Result<Option<metrics_exporter_prometheus::PrometheusHandle>, crate::RuntimeError> {
match enabled {
false => Ok(None),
true => shared_metrics_handle(),
}
}