use crate::resource::{HealthState, Resource};
use std::ops::ControlFlow;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
pub(crate) fn shutdown_resources(resources: &[Box<dyn Resource>]) -> Option<crate::RuntimeError> {
let panic = std::sync::Mutex::new(None);
std::thread::scope(|scope| spawn_shutdown_tasks(scope, resources, &panic));
crate::runtime_state::recover_poisoned(panic.into_inner())
}
fn spawn_shutdown_tasks<'scope, 'env>(
scope: &'scope std::thread::Scope<'scope, 'env>,
resources: &'env [Box<dyn Resource>],
panic: &'env std::sync::Mutex<Option<crate::RuntimeError>>,
) {
for resource in resources.iter() {
let resource = resource.as_ref();
scope.spawn(move || shutdown_one(resource, panic));
}
}
fn shutdown_one(resource: &dyn Resource, panic: &std::sync::Mutex<Option<crate::RuntimeError>>) {
let mut name = None;
let outcome = crate::task::catch_panic(|| {
name = Some(resource.name());
resource.shutdown()
});
let name = name.unwrap_or("<unnamed resource>");
match outcome {
Ok(Ok(())) => {}
Ok(Err(error)) => {
tracing::error!(resource = name, %error, "resource shutdown failed");
}
Err(error) => record_shutdown_panic(name, error, panic),
}
}
fn record_shutdown_panic(
resource: &str,
error: crate::RuntimeError,
panic: &std::sync::Mutex<Option<crate::RuntimeError>>,
) {
let mut first = crate::runtime_state::recover_poisoned(panic.lock());
match first.as_ref() {
Some(_) => tracing::warn!(resource, %error, "further resource shutdown panic"),
None => {
tracing::error!(resource, %error, "resource shutdown panicked");
*first = Some(error);
}
}
}
fn log_health_result(name: &str, result: &Result<(), crate::RuntimeError>) {
if let Err(e) = result {
tracing::warn!(resource = name, error = %e, "health check failed");
}
}
pub(crate) fn admit_health_tasks(
runtime: &Arc<crate::runtime_state::RuntimeInner>,
resources: &Arc<[Box<dyn Resource>]>,
health_state: &Option<HealthState>,
interval: Duration,
) {
let hs = match health_state {
Some(hs) => hs,
None => return,
};
for (idx, resource) in resources.iter().enumerate() {
drop(crate::task::admit_signalled_subsystem_on(
runtime,
resource.name(),
|signals| {
run_health_task(
Arc::clone(resources),
Arc::clone(hs),
signals,
interval,
idx,
)
},
));
}
}
pub(crate) async fn run_initial_health_checks(
resources: &Arc<[Box<dyn Resource>]>,
health_state: &HealthState,
) {
let context = crate::runtime_state::try_current_runtime();
let mut join_set = tokio::task::JoinSet::new();
let mut probes = std::collections::HashMap::with_capacity(resources.len());
for idx in 0..resources.len() {
let handle = spawn_initial_health_check(
&mut join_set,
context.clone(),
Arc::clone(resources),
Arc::clone(health_state),
idx,
);
probes.insert(handle.id(), idx);
}
while let Some(joined) = join_set.join_next_with_id().await {
report_probe_join(health_state, &probes, joined);
}
}
fn report_probe_join(
health_state: &HealthState,
probes: &std::collections::HashMap<tokio::task::Id, usize>,
joined: Result<(tokio::task::Id, ()), tokio::task::JoinError>,
) {
let error = match joined {
Ok(_) => return,
Err(error) => error,
};
match probes.get(&error.id()).copied() {
Some(idx) => record_probe_failure(health_state, idx, &probe_join_error(error)),
None => tracing::error!(%error, "initial health check task failed to join"),
}
}
fn probe_join_error(error: tokio::task::JoinError) -> crate::RuntimeError {
match error.try_into_panic() {
Ok(payload) => crate::task::panic_to_error(payload),
Err(_) => crate::RuntimeError::Cancelled,
}
}
fn spawn_initial_health_check(
join_set: &mut tokio::task::JoinSet<()>,
context: Option<Arc<crate::runtime_state::RuntimeInner>>,
resources: Arc<[Box<dyn Resource>]>,
health_state: HealthState,
idx: usize,
) -> tokio::task::AbortHandle {
join_set.spawn_blocking(move || {
let guard = context.map(crate::runtime_state::install_runtime);
run_initial_probe(resources.as_ref(), &health_state, idx);
drop(guard);
})
}
fn run_initial_probe(resources: &[Box<dyn Resource>], health_state: &HealthState, idx: usize) {
match crate::task::catch_panic(|| update_resource_health(resources, health_state, idx)) {
Ok(()) => {}
Err(error) => record_probe_failure(health_state, idx, &error),
}
}
fn record_probe_failure(health_state: &HealthState, idx: usize, error: &crate::RuntimeError) {
match health_state.get(idx) {
Some((name, healthy)) => {
healthy.store(false, Ordering::Release);
tracing::error!(resource = %name, %error, "initial health check did not report");
}
None => tracing::error!(idx, %error, "initial health check did not report"),
}
}
async fn run_health_task(
resources: Arc<[Box<dyn Resource>]>,
health_state: HealthState,
signals: crate::runtime_state::LifecycleSignals,
interval: Duration,
idx: usize,
) {
while let ControlFlow::Continue(()) = signals.tick(interval).await {
crate::task::block_in_place(|| {
update_resource_health(resources.as_ref(), &health_state, idx)
});
}
}
fn update_resource_health(resources: &[Box<dyn Resource>], health_state: &HealthState, idx: usize) {
match (resources.get(idx), health_state.get(idx)) {
(Some(resource), Some((_, healthy))) => probe_resource(resource.as_ref(), healthy),
_ => tracing::error!(
idx,
resources = resources.len(),
health_state = health_state.len(),
"resource registry and health state disagree on length"
),
}
}
fn probe_resource(resource: &dyn Resource, healthy: &AtomicBool) {
let result = resource.health_check();
log_health_result(resource.name(), &result);
healthy.store(result.is_ok(), Ordering::Release);
}