use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::time::{Duration, Instant};
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use axum::{Json, Router};
use serde_json::{json, Value};
use crate::sql::Pool;
pub type CheckFn =
Arc<dyn Fn() -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> + Send + Sync>;
pub struct HealthRouter {
pool: Pool,
extra_checks: Vec<(String, Option<String>, CheckFn)>,
per_check_timeout: Duration,
include_db_probe: bool,
show_errors: bool,
}
#[derive(Clone)]
struct HealthState {
pool: Pool,
extra_checks: Arc<Vec<(String, Option<String>, CheckFn)>>,
per_check_timeout: Duration,
include_db_probe: bool,
show_errors: bool,
}
impl HealthRouter {
#[must_use]
pub fn new(pool: impl Into<Pool>) -> Self {
Self {
pool: pool.into(),
extra_checks: Vec::new(),
per_check_timeout: Duration::from_secs(5),
include_db_probe: true,
show_errors: false,
}
}
#[must_use]
pub fn show_errors(mut self) -> Self {
self.show_errors = true;
self
}
#[must_use]
pub fn timeout(mut self, t: Duration) -> Self {
self.per_check_timeout = t;
self
}
#[must_use]
pub fn skip_db_probe(mut self) -> Self {
self.include_db_probe = false;
self
}
#[must_use]
pub fn check<F, Fut>(mut self, name: &str, check: F) -> Self
where
F: Fn() -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<(), String>> + Send + 'static,
{
let boxed: CheckFn = Arc::new(move || Box::pin(check()));
self.extra_checks.push((name.to_owned(), None, boxed));
self
}
fn check_with_target<F, Fut>(mut self, name: &str, target: String, check: F) -> Self
where
F: Fn() -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<(), String>> + Send + 'static,
{
let boxed: CheckFn = Arc::new(move || Box::pin(check()));
self.extra_checks
.push((name.to_owned(), Some(target), boxed));
self
}
#[must_use]
pub fn tcp_probe(self, name: &str, addr: impl Into<String>) -> Self {
let addr = addr.into();
let target = addr.clone();
self.check_with_target(name, target, move || {
let addr = addr.clone();
async move {
tokio::net::TcpStream::connect(&addr)
.await
.map(|_| ())
.map_err(|e| format!("{addr}: {e}"))
}
})
}
#[cfg(feature = "cache")]
#[must_use]
pub fn cache_probe(self, name: &str, cache: crate::cache::BoxedCache) -> Self {
let probe_name = name.to_owned();
self.check(name, move || {
let cache = cache.clone();
let probe_name = probe_name.clone();
async move {
let value = uuid::Uuid::new_v4().simple().to_string();
let key = format!("rustango:health:{probe_name}:{value}");
cache
.set(&key, &value, Some(Duration::from_secs(10)))
.await
.map_err(|e| format!("set: {e}"))?;
let got = cache
.get(&key)
.await
.map_err(|e| format!("get: {e}"))?
.ok_or_else(|| "set succeeded but get returned None".to_owned())?;
if got != value {
return Err(format!(
"read-after-write mismatch: wrote {value}, read {got}"
));
}
let _ = cache.delete(&key).await;
Ok(())
}
})
}
#[cfg(feature = "http-client")]
#[must_use]
pub fn http_probe(
self,
name: &str,
client: crate::http_client::HttpClient,
url: impl Into<String>,
) -> Self {
let url = url.into();
let label = probe_label(&url);
self.check(name, move || {
let client = client.clone();
let url = url.clone();
let label = label.clone();
async move {
let resp = client
.get(url.as_str())
.send()
.await
.map_err(|e| format!("{label}: {e}"))?;
let status = resp.status();
if status.is_success() {
Ok(())
} else {
Err(format!("{label}: status {status}"))
}
}
})
}
#[must_use]
pub fn into_router(self) -> Router {
let state = HealthState {
pool: self.pool,
extra_checks: Arc::new(self.extra_checks),
per_check_timeout: self.per_check_timeout,
include_db_probe: self.include_db_probe,
show_errors: self.show_errors,
};
Router::new()
.route(PROBE_PATHS[0], get(handle_live))
.route(PROBE_PATHS[1], get(handle_ready))
.with_state(state)
}
}
pub(crate) const PROBE_PATHS: [&str; 2] = ["/health", "/ready"];
pub(crate) fn is_probe_path(path: &str) -> bool {
PROBE_PATHS.contains(&path)
}
#[cfg(feature = "http-client")]
fn probe_label(url: &str) -> String {
match reqwest::Url::parse(url) {
Ok(u) => {
let port = u.port().map(|p| format!(":{p}")).unwrap_or_default();
format!(
"{}://{}{port}{}",
u.scheme(),
u.host_str().unwrap_or(""),
u.path()
)
}
Err(_) => "<invalid url>".to_owned(),
}
}
#[must_use]
pub fn health_router(pool: impl Into<Pool>) -> Router {
HealthRouter::new(pool).into_router()
}
async fn handle_live() -> Response {
(StatusCode::OK, Json(json!({"status": "ok"}))).into_response()
}
async fn handle_ready(State(state): State<HealthState>) -> Response {
let mut checks = serde_json::Map::new();
let mut all_ok = true;
if state.include_db_probe {
let pool = state.pool.clone();
let outcome = run_with_timeout(state.per_check_timeout, None, async move {
crate::sql::raw_execute_pool(&pool, "SELECT 1", Vec::new())
.await
.map(|_| ())
.map_err(|e| e.to_string())
})
.await;
if record(&mut checks, "database", outcome, state.show_errors) {
all_ok = false;
}
}
for (name, target, check) in state.extra_checks.iter() {
let outcome = run_with_timeout(state.per_check_timeout, target.as_deref(), check()).await;
if record(&mut checks, name, outcome, state.show_errors) {
all_ok = false;
}
}
let body = json!({
"status": if all_ok { "ok" } else { "error" },
"checks": Value::Object(checks),
});
let status = if all_ok {
StatusCode::OK
} else {
StatusCode::SERVICE_UNAVAILABLE
};
(status, Json(body)).into_response()
}
async fn run_with_timeout<F>(
timeout: Duration,
target: Option<&str>,
fut: F,
) -> (Result<(), String>, Duration)
where
F: Future<Output = Result<(), String>>,
{
let start = Instant::now();
let outcome = match tokio::time::timeout(timeout, fut).await {
Ok(r) => r,
Err(_) => Err(match target {
Some(t) => format!("{t}: timed out after {}ms", timeout.as_millis()),
None => format!("timed out after {}ms", timeout.as_millis()),
}),
};
(outcome, start.elapsed())
}
fn record(
checks: &mut serde_json::Map<String, Value>,
name: &str,
(outcome, elapsed): (Result<(), String>, Duration),
show_errors: bool,
) -> bool {
let latency_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX);
let (status_str, error_field) = match outcome {
Ok(()) => ("ok", None),
Err(e) => {
tracing::warn!(target: "rustango::error", check = name, error = %e, "readiness check failed");
("error", show_errors.then(|| truncate(&e, 200)))
}
};
let mut entry = serde_json::Map::new();
entry.insert("status".into(), Value::String(status_str.into()));
entry.insert("latency_ms".into(), Value::Number(latency_ms.into()));
if let Some(err) = error_field {
entry.insert("error".into(), Value::String(err));
}
checks.insert(name.to_owned(), Value::Object(entry));
matches!(status_str, "error")
}
fn truncate(s: &str, max: usize) -> String {
if s.chars().count() <= max {
s.to_owned()
} else {
let mut out: String = s.chars().take(max).collect();
out.push('…');
out
}
}
#[cfg(test)]
mod tests {
use super::*;
use axum::body::Body;
use axum::http::Request;
use tower::ServiceExt;
fn lazy_pool() -> Pool {
#[cfg(feature = "postgres")]
{
return Pool::Postgres(
crate::sql::sqlx::postgres::PgPoolOptions::new()
.max_connections(1)
.connect_lazy("postgres://localhost:1/none")
.unwrap(),
);
}
#[cfg(all(feature = "sqlite", not(feature = "postgres")))]
{
return Pool::Sqlite(
crate::sql::sqlx::sqlite::SqlitePoolOptions::new()
.max_connections(1)
.connect_lazy("sqlite::memory:")
.unwrap(),
);
}
#[cfg(all(feature = "mysql", not(feature = "postgres"), not(feature = "sqlite")))]
{
return Pool::Mysql(
crate::sql::sqlx::mysql::MySqlPoolOptions::new()
.max_connections(1)
.connect_lazy("mysql://root@localhost:1/none")
.unwrap(),
);
}
}
async fn body_json(resp: Response) -> Value {
let bytes = axum::body::to_bytes(resp.into_body(), 1 << 16)
.await
.unwrap();
serde_json::from_slice(&bytes).unwrap()
}
#[tokio::test]
async fn health_endpoint_always_returns_ok() {
let app = HealthRouter::new(lazy_pool()).skip_db_probe().into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/health")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), 200);
let v = body_json(resp).await;
assert_eq!(v["status"], "ok");
}
#[tokio::test]
async fn ready_passes_with_only_passing_extra_checks() {
let app = HealthRouter::new(lazy_pool())
.skip_db_probe()
.show_errors()
.check("always_ok", || async { Ok(()) })
.into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), 200);
let v = body_json(resp).await;
assert_eq!(v["status"], "ok");
assert_eq!(v["checks"]["always_ok"]["status"], "ok");
assert!(v["checks"]["always_ok"]["latency_ms"].is_number());
}
#[tokio::test]
async fn ready_returns_503_when_one_check_fails() {
let app = HealthRouter::new(lazy_pool())
.skip_db_probe()
.show_errors()
.check("ok", || async { Ok(()) })
.check("broken", || async { Err("nope".into()) })
.into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), 503);
let v = body_json(resp).await;
assert_eq!(v["status"], "error");
assert_eq!(v["checks"]["broken"]["status"], "error");
assert_eq!(v["checks"]["broken"]["error"], "nope");
assert_eq!(v["checks"]["ok"]["status"], "ok");
}
#[tokio::test]
async fn slow_check_is_killed_by_timeout() {
let app = HealthRouter::new(lazy_pool())
.skip_db_probe()
.show_errors()
.timeout(Duration::from_millis(50))
.check("slow", || async {
tokio::time::sleep(Duration::from_secs(5)).await;
Ok(())
})
.into_router();
let start = std::time::Instant::now();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let elapsed = start.elapsed();
assert_eq!(resp.status(), 503);
assert!(
elapsed < Duration::from_millis(500),
"timeout didn't fire — took {elapsed:?}"
);
let v = body_json(resp).await;
assert_eq!(v["checks"]["slow"]["status"], "error");
assert!(v["checks"]["slow"]["error"]
.as_str()
.unwrap()
.contains("timed out"));
}
#[tokio::test]
async fn skip_db_probe_omits_database_check() {
let app = HealthRouter::new(lazy_pool()).skip_db_probe().into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let v = body_json(resp).await;
assert!(v["checks"].get("database").is_none());
}
#[tokio::test]
async fn long_error_message_is_truncated() {
let huge = "x".repeat(500);
let app = HealthRouter::new(lazy_pool())
.skip_db_probe()
.show_errors()
.check("verbose", move || {
let huge = huge.clone();
async move { Err(huge) }
})
.into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let v = body_json(resp).await;
let err = v["checks"]["verbose"]["error"].as_str().unwrap();
assert!(
err.chars().count() <= 201,
"got {} chars",
err.chars().count()
);
assert!(err.ends_with('…'));
}
#[tokio::test]
async fn ready_withholds_errors_and_targets_by_default() {
let app = HealthRouter::new(lazy_pool())
.tcp_probe("redis", "127.0.0.1:1")
.check("db2", || async {
Err("password authentication failed for user app".into())
})
.timeout(Duration::from_millis(500))
.into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let v = body_json(resp).await;
assert_eq!(v["status"], "error");
for name in ["redis", "db2"] {
assert_eq!(v["checks"][name]["status"], "error", "{v}");
assert!(v["checks"][name].get("error").is_none(), "{v}");
}
let text = v.to_string();
assert!(
!text.contains("127.0.0.1") && !text.contains("password"),
"{text}"
);
}
#[tokio::test]
async fn tcp_probe_failure_reports_addr_in_error() {
let app = HealthRouter::new(lazy_pool())
.skip_db_probe()
.show_errors()
.tcp_probe("nothing-listening", "127.0.0.1:1")
.timeout(Duration::from_millis(500))
.into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), 503);
let v = body_json(resp).await;
assert_eq!(v["checks"]["nothing-listening"]["status"], "error");
let err = v["checks"]["nothing-listening"]["error"].as_str().unwrap();
assert!(err.contains("127.0.0.1:1"), "expected addr in error: {err}");
}
#[cfg(feature = "http-client")]
#[tokio::test]
async fn http_probe_error_hides_url_secrets() {
let client = crate::http_client::HttpClient::builder()
.retry_policy(crate::http_client::RetryPolicy::off())
.build()
.unwrap();
let app = HealthRouter::new(lazy_pool())
.skip_db_probe()
.show_errors()
.http_probe(
"upstream",
client,
"http://admin:PASSWD@127.0.0.1:1/ping?token=SECRETTOKEN",
)
.timeout(Duration::from_secs(5))
.into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
let v = body_json(resp).await;
let err = v["checks"]["upstream"]["error"].as_str().unwrap();
assert!(err.contains("http://127.0.0.1:1/ping"), "{err}");
assert!(
!err.contains("PASSWD") && !err.contains("SECRETTOKEN"),
"{err}"
);
}
#[cfg(feature = "cache")]
#[tokio::test]
async fn cache_probe_succeeds_with_in_memory_cache() {
use crate::cache::{BoxedCache, InMemoryCache};
let cache: BoxedCache = Arc::new(InMemoryCache::new());
let app = HealthRouter::new(lazy_pool())
.skip_db_probe()
.show_errors()
.cache_probe("memory", cache)
.into_router();
let resp = app
.oneshot(
Request::builder()
.uri("/ready")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), 200);
let v = body_json(resp).await;
assert_eq!(v["checks"]["memory"]["status"], "ok");
}
#[cfg(feature = "cache")]
struct Yielding(crate::cache::InMemoryCache);
#[cfg(feature = "cache")]
#[async_trait::async_trait]
impl crate::cache::Cache for Yielding {
async fn get(&self, key: &str) -> Result<Option<String>, crate::cache::CacheError> {
tokio::task::yield_now().await;
self.0.get(key).await
}
async fn set(
&self,
key: &str,
value: &str,
ttl: Option<Duration>,
) -> Result<(), crate::cache::CacheError> {
self.0.set(key, value, ttl).await
}
async fn delete(&self, key: &str) -> Result<(), crate::cache::CacheError> {
self.0.delete(key).await
}
async fn exists(&self, key: &str) -> Result<bool, crate::cache::CacheError> {
self.0.exists(key).await
}
async fn clear(&self) -> Result<(), crate::cache::CacheError> {
self.0.clear().await
}
}
#[cfg(feature = "cache")]
#[tokio::test]
async fn concurrent_cache_probes_do_not_collide() {
let cache: crate::cache::BoxedCache =
Arc::new(Yielding(crate::cache::InMemoryCache::new()));
let app = HealthRouter::new(lazy_pool())
.skip_db_probe()
.show_errors()
.cache_probe("shared", cache)
.into_router();
let ready = || {
app.clone()
.oneshot(Request::get("/ready").body(Body::empty()).unwrap())
};
let (a, b, c) = tokio::join!(ready(), ready(), ready());
for resp in [a, b, c] {
let resp = resp.unwrap();
assert_eq!(resp.status(), 200, "{}", body_json(resp).await);
}
}
#[test]
fn truncate_preserves_short_strings() {
assert_eq!(truncate("hi", 10), "hi");
}
#[test]
fn truncate_appends_ellipsis_when_over_max() {
let t = truncate(&"a".repeat(300), 50);
assert_eq!(t.chars().count(), 51);
assert!(t.ends_with('…'));
}
}