use std::time::Duration;
pub async fn run_eggserve_services_until_shutdown<F>(
mut shutdown_future: std::pin::Pin<&mut F>,
mut grpc_handle: tokio::task::JoinHandle<Result<(), tonic::transport::Error>>,
http_control: eggserve_server::ServerControl,
mut http_completion: eggserve_server::ServerCompletion,
shutdown_sender: tokio::sync::broadcast::Sender<()>,
drain_timeout: Duration,
) -> ServiceShutdownOutcome
where
F: std::future::Future<Output = ()>,
{
let mut requested = false;
let mut grpc_result = None;
let mut http_result = None;
tokio::select! {
biased;
_ = &mut shutdown_future => requested = true,
result = &mut grpc_handle => grpc_result = Some(classify_result(result)),
result = http_completion.wait() => {
http_result = Some(match result {
Ok(_) => ServiceResult::Clean,
Err(error) => ServiceResult::ServiceError(error.to_string()),
});
}
}
let _ = shutdown_sender.send(());
http_control.shutdown();
let drain_result = tokio::time::timeout(drain_timeout, async {
while grpc_result.is_none() || http_result.is_none() {
tokio::select! {
result = &mut grpc_handle, if grpc_result.is_none() => {
grpc_result = Some(classify_result(result));
}
result = http_completion.wait(), if http_result.is_none() => {
http_result = Some(match result {
Ok(_) => ServiceResult::Clean,
Err(error) => ServiceResult::ServiceError(error.to_string()),
});
}
}
}
})
.await;
let forced = drain_result.is_err();
if forced {
grpc_handle.abort();
if grpc_result.is_none() {
grpc_result = Some(classify_result(grpc_handle.await));
}
if http_result.is_none() {
http_result = Some(match http_completion.wait().await {
Ok(_) => ServiceResult::Clean,
Err(error) => ServiceResult::ServiceError(error.to_string()),
});
}
}
ServiceShutdownOutcome {
requested,
forced,
grpc_result: grpc_result.unwrap_or_else(|| ServiceResult::Cancelled("not observed".into())),
http_result: http_result.unwrap_or_else(|| ServiceResult::Cancelled("not observed".into())),
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ServiceResult {
Clean,
ServiceError(String),
Panic(String),
Cancelled(String),
}
#[derive(Debug)]
pub struct ServiceShutdownOutcome {
pub requested: bool,
pub forced: bool,
pub grpc_result: ServiceResult,
pub http_result: ServiceResult,
}
impl ServiceShutdownOutcome {
pub fn is_clean_requested_shutdown(&self) -> bool {
self.requested
&& !self.forced
&& self.grpc_result == ServiceResult::Clean
&& self.http_result == ServiceResult::Clean
}
pub fn ensure_clean_requested_shutdown(&self) -> Result<(), String> {
if self.is_clean_requested_shutdown() {
return Ok(());
}
Err(format!(
"service shutdown was not clean: requested={}, forced={}, grpc={:?}, http={:?}",
self.requested, self.forced, self.grpc_result, self.http_result,
))
}
}
fn classify_result(
result: Result<Result<(), impl std::fmt::Display>, tokio::task::JoinError>,
) -> ServiceResult {
match result {
Ok(Ok(())) => ServiceResult::Clean,
Ok(Err(e)) => ServiceResult::ServiceError(e.to_string()),
Err(e) if e.is_panic() => ServiceResult::Panic(format!("{e}")),
Err(e) => ServiceResult::Cancelled(format!("{e}")),
}
}
#[cfg(test)]
pub async fn run_services_until_shutdown<F>(
mut shutdown_future: std::pin::Pin<&mut F>,
grpc_handle: tokio::task::JoinHandle<Result<(), tonic::transport::Error>>,
http_handle: tokio::task::JoinHandle<Result<(), std::io::Error>>,
shutdown_sender: tokio::sync::broadcast::Sender<()>,
drain_timeout: Duration,
) -> ServiceShutdownOutcome
where
F: std::future::Future<Output = ()>,
{
let mut requested = false;
let mut grpc_result: Option<ServiceResult> = None;
let mut http_result: Option<ServiceResult> = None;
let grpc_abort = grpc_handle.abort_handle();
let http_abort = http_handle.abort_handle();
#[derive(Clone, Copy)]
enum ServiceKind {
Grpc,
Http,
}
struct ServiceCompletion {
kind: ServiceKind,
result: ServiceResult,
}
let mut tasks = tokio::task::JoinSet::<ServiceCompletion>::new();
tasks.spawn(async move {
ServiceCompletion {
kind: ServiceKind::Grpc,
result: classify_result(grpc_handle.await),
}
});
tasks.spawn(async move {
ServiceCompletion {
kind: ServiceKind::Http,
result: classify_result(http_handle.await),
}
});
let record = |completion: ServiceCompletion,
grpc_result: &mut Option<ServiceResult>,
http_result: &mut Option<ServiceResult>,
phase: &str| {
if completion.result != ServiceResult::Clean {
let name = match completion.kind {
ServiceKind::Grpc => "gRPC",
ServiceKind::Http => "HTTP",
};
tracing::error!("{name} service {phase}: {:?}", completion.result);
}
match completion.kind {
ServiceKind::Grpc => *grpc_result = Some(completion.result),
ServiceKind::Http => *http_result = Some(completion.result),
}
};
tokio::select! {
biased;
_ = &mut shutdown_future => {
tracing::info!("Shutdown signal received");
requested = true;
}
completion = tasks.join_next() => {
if let Some(Ok(completion)) = completion {
record(completion, &mut grpc_result, &mut http_result, "completed");
}
}
}
let _ = shutdown_sender.send(());
let drain_result = tokio::time::timeout(drain_timeout, async {
while let Some(joined) = tasks.join_next().await {
if let Ok(completion) = joined {
record(
completion,
&mut grpc_result,
&mut http_result,
"during drain",
);
}
}
})
.await;
let forced = drain_result.is_err();
if forced {
tracing::warn!(
"Graceful shutdown timed out after {}s, aborting remaining tasks",
drain_timeout.as_secs()
);
grpc_abort.abort();
http_abort.abort();
while let Some(joined) = tasks.join_next().await {
if let Ok(completion) = joined {
record(
completion,
&mut grpc_result,
&mut http_result,
"after abort",
);
}
}
}
ServiceShutdownOutcome {
requested,
forced,
grpc_result: grpc_result
.unwrap_or_else(|| ServiceResult::Cancelled("not observed".to_owned())),
http_result: http_result
.unwrap_or_else(|| ServiceResult::Cancelled("not observed".to_owned())),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
type GrpcHandle = tokio::task::JoinHandle<Result<(), tonic::transport::Error>>;
type HttpHandle = tokio::task::JoinHandle<Result<(), std::io::Error>>;
#[tokio::test]
async fn requested_clean_shutdown() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = tokio::spawn({
let mut rx = tx.subscribe();
async move {
let _ = rx.recv().await;
Ok(())
}
});
let http: HttpHandle = tokio::spawn({
let mut rx = tx.subscribe();
async move {
let _ = rx.recv().await;
Ok(())
}
});
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
assert!(outcome.requested);
assert!(!outcome.forced);
assert!(outcome.is_clean_requested_shutdown());
assert!(
outcome.ensure_clean_requested_shutdown().is_ok(),
"production decision method must return Ok for clean requested shutdown"
);
assert_eq!(outcome.grpc_result, ServiceResult::Clean);
assert_eq!(outcome.http_result, ServiceResult::Clean);
}
#[tokio::test]
async fn one_service_completes_sibling_times_out() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc_await_count = Arc::new(AtomicUsize::new(0));
let http_drop_count = Arc::new(AtomicUsize::new(0));
let grpc: GrpcHandle = {
let count = grpc_await_count.clone();
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
count.fetch_add(1, Ordering::SeqCst);
Ok(())
})
};
let http: HttpHandle = {
let drop_count = http_drop_count.clone();
tokio::spawn(async move {
struct DropProbe(Arc<AtomicUsize>);
impl Drop for DropProbe {
fn drop(&mut self) {
self.0.fetch_add(1, Ordering::SeqCst);
}
}
let _probe = DropProbe(drop_count);
std::future::pending::<Result<(), std::io::Error>>().await
})
};
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_millis(100)).await;
assert!(outcome.requested);
assert!(outcome.forced);
assert_eq!(outcome.grpc_result, ServiceResult::Clean);
assert_eq!(
grpc_await_count.load(Ordering::SeqCst),
1,
"gRPC future should have completed exactly once"
);
assert!(matches!(outcome.http_result, ServiceResult::Cancelled(_)));
assert_eq!(
http_drop_count.load(Ordering::SeqCst),
1,
"the underlying HTTP future must be dropped before the helper returns"
);
let err = outcome
.ensure_clean_requested_shutdown()
.expect_err("forced shutdown must fail production check");
assert!(
err.contains("forced=true"),
"diagnostic should mention forced: {err}"
);
}
#[tokio::test]
async fn drain_time_service_panic() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
panic!("grpc drain panic");
})
};
let http: HttpHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
assert!(outcome.requested);
assert!(!outcome.forced);
assert!(!outcome.is_clean_requested_shutdown());
assert!(matches!(outcome.grpc_result, ServiceResult::Panic(_)));
assert_eq!(outcome.http_result, ServiceResult::Clean);
let err = outcome
.ensure_clean_requested_shutdown()
.expect_err("panic must fail production check");
assert!(
err.contains("grpc"),
"diagnostic should mention grpc: {err}"
);
assert!(
err.contains("grpc drain panic"),
"diagnostic should retain original panic detail: {err}"
);
}
#[tokio::test]
async fn drain_time_service_error() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
let http: HttpHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Err(std::io::Error::other("http service error"))
})
};
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
assert!(outcome.requested);
assert!(!outcome.forced);
assert!(!outcome.is_clean_requested_shutdown());
assert_eq!(outcome.grpc_result, ServiceResult::Clean);
assert!(matches!(
outcome.http_result,
ServiceResult::ServiceError(_)
));
let err = outcome
.ensure_clean_requested_shutdown()
.expect_err("drain-time error must fail production check");
assert!(
err.contains("http"),
"diagnostic should mention http: {err}"
);
assert!(
err.contains("http service error"),
"diagnostic should retain original error detail: {err}"
);
}
#[tokio::test]
async fn unexpected_grpc_exit() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = tokio::spawn(async { Ok(()) });
let http: HttpHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
tokio::time::sleep(Duration::from_millis(50)).await;
let (_signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
let signal = async move {
let _ = signal_rx.await;
};
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
assert!(!outcome.requested);
assert!(!outcome.forced);
assert!(!outcome.is_clean_requested_shutdown());
let err = outcome
.ensure_clean_requested_shutdown()
.expect_err("unexpected clean service exit must fail production check");
assert!(
err.contains("requested=false"),
"diagnostic should mention requested=false: {err}"
);
}
#[tokio::test]
async fn unexpected_http_error() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
let http: HttpHandle = tokio::spawn(async { Err(std::io::Error::other("http boom")) });
tokio::time::sleep(Duration::from_millis(50)).await;
let (_signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
let signal = async move {
let _ = signal_rx.await;
};
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
assert!(!outcome.requested);
assert!(!outcome.forced);
assert!(!outcome.is_clean_requested_shutdown());
let err = outcome
.ensure_clean_requested_shutdown()
.expect_err("unexpected service error must fail production check");
assert!(
err.contains("http"),
"diagnostic should mention http: {err}"
);
}
#[tokio::test]
async fn both_refuse_to_drain() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = tokio::spawn(async {
std::future::pending::<Result<(), tonic::transport::Error>>().await
});
let http: HttpHandle =
tokio::spawn(async { std::future::pending::<Result<(), std::io::Error>>().await });
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let start = std::time::Instant::now();
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_millis(100)).await;
let elapsed = start.elapsed();
assert!(outcome.requested);
assert!(outcome.forced);
assert!(!outcome.is_clean_requested_shutdown());
assert!(
elapsed < Duration::from_secs(2),
"test should complete quickly, took {elapsed:?}"
);
let err = outcome
.ensure_clean_requested_shutdown()
.expect_err("forced abort must fail production check");
assert!(
err.contains("forced=true"),
"diagnostic should mention forced: {err}"
);
}
#[tokio::test]
async fn no_pre_signal_lifetime_timeout() {
let drain_timeout = Duration::from_millis(100);
{
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let (_signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
let grpc: GrpcHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
let http: HttpHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
let mut signal = Box::pin(async move {
let _ = signal_rx.await;
});
let helper_fut =
run_services_until_shutdown(signal.as_mut(), grpc, http, tx, drain_timeout);
let phase_a_result = tokio::time::timeout(drain_timeout * 2, helper_fut).await;
assert!(
phase_a_result.is_err(),
"helper must remain pending for at least 2× drain timeout when no terminal event has occurred"
);
}
{
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let (signal_tx, signal_rx) = tokio::sync::oneshot::channel::<()>();
let grpc: GrpcHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
let http: HttpHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
let mut signal = Box::pin(async move {
let _ = signal_rx.await;
});
let _ = signal_tx.send(());
let helper_fut =
run_services_until_shutdown(signal.as_mut(), grpc, http, tx, drain_timeout);
let outcome = tokio::time::timeout(Duration::from_secs(5), helper_fut)
.await
.expect("helper should complete within 5s after signal trigger");
assert!(outcome.requested);
assert!(!outcome.forced);
assert!(outcome.is_clean_requested_shutdown());
assert!(
outcome.ensure_clean_requested_shutdown().is_ok(),
"production decision method must return Ok for clean requested shutdown"
);
}
}
#[tokio::test]
async fn grpc_service_completion_is_observed_once() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let poll_count = Arc::new(AtomicUsize::new(0));
let grpc: GrpcHandle = {
let count = poll_count.clone();
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
count.fetch_add(1, Ordering::SeqCst);
Ok(())
})
};
let http: HttpHandle =
tokio::spawn(async { std::future::pending::<Result<(), std::io::Error>>().await });
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_millis(100)).await;
assert!(outcome.forced);
assert_eq!(
poll_count.load(Ordering::SeqCst),
1,
"gRPC future should have been polled exactly once"
);
}
#[tokio::test]
async fn drain_panic_preserved_in_outcome() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
panic!("deliberate grpc panic");
})
};
let http: HttpHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
assert!(outcome.requested);
assert!(!outcome.forced);
assert!(!outcome.is_clean_requested_shutdown());
match &outcome.grpc_result {
ServiceResult::Panic(msg) => {
assert!(
msg.contains("deliberate grpc panic"),
"panic message should be preserved"
);
}
other => panic!("expected Panic, got {other:?}"),
}
}
#[tokio::test]
async fn http_aborted_while_grpc_clean() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
Ok(())
})
};
let http: HttpHandle =
tokio::spawn(async { std::future::pending::<Result<(), std::io::Error>>().await });
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_millis(100)).await;
assert!(outcome.requested);
assert!(outcome.forced);
assert_eq!(outcome.grpc_result, ServiceResult::Clean);
assert!(matches!(
outcome.http_result,
ServiceResult::Cancelled(_) | ServiceResult::Panic(_)
));
}
#[tokio::test]
async fn both_complete_during_drain() {
let (tx, _) = tokio::sync::broadcast::channel::<()>(1);
let grpc: GrpcHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
tokio::time::sleep(Duration::from_millis(10)).await;
Ok(())
})
};
let http: HttpHandle = {
let mut rx = tx.subscribe();
tokio::spawn(async move {
let _ = rx.recv().await;
tokio::time::sleep(Duration::from_millis(20)).await;
Ok(())
})
};
tokio::time::sleep(Duration::from_millis(50)).await;
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome =
run_services_until_shutdown(signal, grpc, http, tx, Duration::from_secs(2)).await;
assert!(outcome.requested);
assert!(!outcome.forced);
assert!(outcome.is_clean_requested_shutdown());
}
}
#[cfg(test)]
mod eggserve_tests {
use super::*;
#[tokio::test]
async fn eggserve_runtime_drains_after_requested_shutdown() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let server = eggserve_server::Server::builder()
.from_listener(listener)
.build()
.unwrap();
let handle = server
.start_with_service(eggserve_server::service_fn(|_request| async {
Ok(eggserve_primitives::Response::builder()
.status(eggserve_primitives::StatusCode::OK)
.body(eggserve_primitives::ResponseBody::Empty)
.unwrap())
}))
.await
.unwrap();
let (control, completion) = handle.into_parts();
let (shutdown_tx, _) = tokio::sync::broadcast::channel(1);
let grpc = tokio::spawn({
let mut shutdown_rx = shutdown_tx.subscribe();
async move {
let _ = shutdown_rx.recv().await;
Ok::<(), tonic::transport::Error>(())
}
});
let signal = std::future::ready(());
tokio::pin!(signal);
let outcome = run_eggserve_services_until_shutdown(
signal,
grpc,
control,
completion,
shutdown_tx,
Duration::from_secs(2),
)
.await;
assert!(outcome.is_clean_requested_shutdown(), "{outcome:?}");
}
}