fraiseql-server 2.16.0

//! Health check endpoint.

use axum::{Json, extract::State, http::StatusCode, response::IntoResponse};
use serde::Serialize;
use tracing::{debug, error};

use crate::routes::graphql::AppState;

/// Health check response.
#[derive(Debug, Serialize)]
pub struct HealthResponse {
    /// Overall server status: `"healthy"`, `"degraded"`, or `"unhealthy"`.
    ///
    /// - `"healthy"` — database and all enabled subsystems are reachable.
    /// - `"degraded"` — database is healthy but an optional subsystem is failing. Returns HTTP 200
    ///   so load balancers keep the pod in rotation; alert on the field value.
    /// - `"unhealthy"` — database is unreachable. Returns HTTP 503.
    pub status: String,

    /// Database status.
    pub database: DatabaseStatus,

    /// Observer runtime health (present when the `observers` feature is compiled in).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub observers: Option<ObserverHealth>,

    /// Cache / Redis health (present when a Redis cache backend is configured).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub cache: Option<CacheHealth>,

    /// Secrets backend health (present when Vault or another secrets backend is configured).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub secrets: Option<SecretsHealth>,

    /// Federation circuit breaker state (present when federation is configured).
    #[cfg(feature = "federation")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub federation: Option<FederationHealth>,

    /// Server version.
    pub version: String,

    /// 32-character hex SHA-256 content hash of the compiled schema.
    ///
    /// Operators can compare this value across server instances to verify
    /// all instances are running the same schema. Different values indicate
    /// a schema mismatch (e.g. partial rollout or stale deployment).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub schema_hash: Option<String>,
}

/// Federation circuit breaker health snapshot.
#[cfg(feature = "federation")]
#[derive(Debug, Serialize)]
pub struct FederationHealth {
    /// Whether federation is configured at all.
    pub configured: bool,
    /// Per-entity circuit breaker state.
    pub subgraphs:  Vec<crate::federation::circuit_breaker::SubgraphCircuitHealth>,
}

/// Observer runtime health snapshot.
#[derive(Debug, Serialize)]
pub struct ObserverHealth {
    /// Whether the observer runtime is currently running.
    pub running:          bool,
    /// **Lifetime** count of events the runtime has finished processing.
    ///
    /// Monotonic: it counts throughput, not backlog, and never decreases while the
    /// process lives. This field used to be called `pending_events` and documented
    /// as "approximate number of events pending in the internal queue" while
    /// carrying this same counter (#875, item 2), so an operator alerting on
    /// `pending_events > N` got an alert that fired permanently after the Nth
    /// successful event and never cleared — while a real backlog stayed invisible.
    ///
    /// The observer runtime is checkpoint-driven and holds no queue whose depth
    /// could be reported, so the field is named for what it actually is rather
    /// than fabricating a depth. A genuine backlog metric would need a new source
    /// on `RuntimeHealth`.
    pub events_processed: usize,
    /// Last error message from the observer runtime, if any.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub last_error:       Option<String>,
}

/// Cache subsystem health.
#[derive(Debug, Serialize)]
pub struct CacheHealth {
    /// Whether the cache backend is reachable (Redis ping succeeded, or always
    /// `true` for the in-memory backend).
    pub connected: bool,
    /// Cache backend type: `"redis"` or `"in-memory"`.
    pub backend:   String,
}

/// Secrets backend health.
#[derive(Debug, Serialize)]
pub struct SecretsHealth {
    /// Whether the secrets backend is reachable and the token is valid.
    pub connected: bool,
    /// Backend type: `"vault"`, `"env"`, or `"file"`.
    pub backend:   String,
}

/// Readiness response (subset of `HealthResponse`).
#[derive(Debug, Serialize)]
pub struct ReadinessResponse {
    /// `"ready"` or `"not_ready"`.
    pub status: String,
    /// Human-readable reason when `status = "not_ready"`.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub reason: Option<String>,
}

/// Database status.
#[derive(Debug, Serialize)]
pub struct DatabaseStatus {
    /// Connection status.
    pub connected: bool,

    /// Database type.
    pub database_type: String,

    /// Active connections (if available).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub active_connections: Option<usize>,

    /// Idle connections (if available).
    #[serde(skip_serializing_if = "Option::is_none")]
    pub idle_connections: Option<usize>,
}

/// Federation health response.
#[cfg(feature = "federation")]
#[derive(Debug, Serialize)]
pub struct FederationHealthResponse {
    /// Overall federation status: healthy, degraded, unhealthy, unknown
    pub status: String,

    /// Per-subgraph status
    pub subgraphs: Vec<crate::federation::SubgraphHealthStatus>,

    /// Response timestamp
    pub timestamp: String,
}

/// Health check handler.
///
/// Returns server and database health status.
///
/// # Response Codes
///
/// - 200: Everything healthy
/// - 503: Database connection failed
pub async fn health_handler(State(state): State<AppState>) -> impl IntoResponse {
    debug!("Health check requested");

    // Bind executor guard once for consistency within this handler
    let executor = state.executor();

    // Perform real database health check
    let health_result = executor.health_check().await;
    let db_healthy = health_result.is_ok();

    let db_type = executor.database_type();
    let metrics = executor.pool_metrics();

    let database = if db_healthy {
        DatabaseStatus {
            connected:          true,
            database_type:      format!("{db_type:?}"),
            active_connections: Some(metrics.active_connections as usize),
            idle_connections:   Some(metrics.idle_connections as usize),
        }
    } else {
        error!("Database health check failed: {:?}", health_result.err());
        DatabaseStatus {
            connected:          false,
            database_type:      format!("{db_type:?}"),
            active_connections: Some(metrics.active_connections as usize),
            idle_connections:   Some(metrics.idle_connections as usize),
        }
    };

    let schema_hash = Some(executor.schema().content_hash());

    #[cfg(feature = "federation")]
    let federation = state.circuit_breaker.as_ref().map(|cb| FederationHealth {
        configured: true,
        subgraphs:  cb.health_snapshot(),
    });

    // Probe observer health when the runtime is attached to AppState.
    #[cfg(feature = "observers")]
    let observers = if let Some(ref runtime) = state.observer_runtime {
        let rt = runtime.read().await;
        let health = rt.health();
        #[allow(clippy::cast_possible_truncation)]
        // Reason: events_processed is a counter that won't realistically exceed usize on any target
        let events_processed = health.events_processed as usize;
        Some(ObserverHealth {
            running: health.running,
            events_processed,
            last_error: if health.errors > 0 {
                Some(format!("{} errors encountered", health.errors))
            } else {
                None
            },
        })
    } else {
        None
    };
    #[cfg(not(feature = "observers"))]
    let observers: Option<ObserverHealth> = None;

    // Probe cache health when the Arrow cache is attached to AppState.
    #[cfg(feature = "arrow")]
    let cache = state.cache.as_ref().map(|_| CacheHealth {
        connected: true, // In-memory cache is always "connected"
        backend:   "in-memory".to_string(),
    });
    #[cfg(not(feature = "arrow"))]
    let cache: Option<CacheHealth> = None;

    // Probe secrets backend health.
    #[cfg(feature = "secrets")]
    let secrets = if let Some(ref sm) = state.secrets_manager {
        let connected = sm.health_check().await.is_ok();
        Some(SecretsHealth {
            connected,
            backend: sm.backend_name().to_string(),
        })
    } else {
        None
    };
    #[cfg(not(feature = "secrets"))]
    let secrets: Option<SecretsHealth> = None;

    #[cfg(feature = "federation")]
    let status =
        determine_status(db_healthy, observers.as_ref(), secrets.as_ref(), federation.as_ref());
    #[cfg(not(feature = "federation"))]
    let status = determine_status(db_healthy, observers.as_ref(), secrets.as_ref());

    let response = HealthResponse {
        status: status.to_string(),
        database,
        observers,
        cache,
        secrets,
        #[cfg(feature = "federation")]
        federation,
        version: env!("CARGO_PKG_VERSION").to_string(),
        schema_hash,
    };

    let status_code = if status == "unhealthy" {
        StatusCode::SERVICE_UNAVAILABLE
    } else {
        StatusCode::OK
    };

    (status_code, Json(response))
}

/// Liveness probe response.
#[derive(Debug, Serialize)]
pub struct LivenessResponse {
    /// Always `"alive"`. The handler cannot return anything else: if it answered at all,
    /// the process is running and its listener is serving.
    pub status: String,
}

/// Liveness probe handler — is this *process* alive?
///
/// Always `200 OK`. It touches no database, no cache, no secrets backend and no
/// subgraph, because a liveness probe answers exactly one question, and restarting the
/// pod is the only thing the kubelet can do with the answer. Restarting cannot fix an
/// unreachable dependency.
///
/// That is why this exists (#1217). `/health` returns **503 when the database is
/// unreachable**, while its own doc comment used to prescribe it as the `livenessProbe`
/// and describe it as "always 200 while process is alive". Every deployment artifact in
/// this repository followed that comment, so a PostgreSQL failover restarted every pod
/// for as long as the outage lasted — a restart storm on top of the incident. The Helm
/// chart and the plain manifests moved to a `tcpSocket` probe in 2.15.0 as a stopgap,
/// which cannot see a process that is deadlocked but still holding its listener. This
/// endpoint can: answering it requires the runtime to schedule a task and write a
/// response.
///
/// Kubernetes usage:
/// - `livenessProbe` → `GET /live` — 200 while the process can serve a request at all
/// - `readinessProbe` → `GET /readiness` — 503 while the database is unreachable
/// - `/health` is the operator-facing status endpoint: the full subsystem report, 200 when healthy
///   or degraded, 503 when the database is down. Do not probe with it.
pub async fn liveness_handler() -> impl IntoResponse {
    debug!("Liveness check requested");
    (
        StatusCode::OK,
        Json(LivenessResponse {
            status: "alive".to_string(),
        }),
    )
}

/// Readiness probe handler.
///
/// Returns `200 OK` when the server can serve traffic (database reachable),
/// or `503 Service Unavailable` when it cannot.
///
/// Kubernetes usage:
/// - `livenessProbe` → `GET /live` (200 while the process can serve a request)
/// - `readinessProbe` → `GET /readiness` (503 while not ready to serve traffic)
pub async fn readiness_handler(State(state): State<AppState>) -> impl IntoResponse {
    debug!("Readiness check requested");

    let db_healthy = state.executor().health_check().await.is_ok();

    if db_healthy {
        (
            StatusCode::OK,
            Json(ReadinessResponse {
                status: "ready".to_string(),
                reason: None,
            }),
        )
    } else {
        (
            StatusCode::SERVICE_UNAVAILABLE,
            Json(ReadinessResponse {
                status: "not_ready".to_string(),
                reason: Some("Database connection unavailable".to_string()),
            }),
        )
    }
}

/// Federation health check handler.
///
/// Returns federation-specific health status.
///
/// When federation is configured in the compiled schema, reports `healthy`.
/// When federation is not configured, reports `not_configured`.
///
/// # Response Codes
///
/// - 200: Federation status retrieved
#[cfg(feature = "federation")]
pub async fn federation_health_handler(State(state): State<AppState>) -> impl IntoResponse {
    debug!("Federation health check requested");

    let executor = state.executor();
    let schema = executor.schema();
    let (status, status_code) = match schema.federation.as_ref() {
        Some(fed) if fed.enabled => ("healthy", StatusCode::OK),
        _ => ("not_configured", StatusCode::OK),
    };

    let subgraphs = state.circuit_breaker.as_ref().map_or_else(Vec::new, |cb| {
        cb.health_snapshot()
            .into_iter()
            .map(|entry| {
                let available = matches!(
                    entry.state,
                    crate::federation::circuit_breaker::CircuitHealthState::Closed
                        | crate::federation::circuit_breaker::CircuitHealthState::HalfOpen
                );
                crate::federation::SubgraphHealthStatus {
                    name: entry.subgraph,
                    available,
                    latency_ms: 0.0,
                    last_check: chrono::Utc::now().to_rfc3339(),
                    error_count_last_60s: 0,
                    error_rate_percent: 0.0,
                }
            })
            .collect()
    });

    let response = FederationHealthResponse {
        status: status.to_string(),
        subgraphs,
        timestamp: chrono::Utc::now().to_rfc3339(),
    };

    (status_code, Json(response))
}

/// Determines overall health status.
///
/// - `"unhealthy"` (503): database is down
/// - `"degraded"` (200): database is up but one or more optional subsystems are failing
/// - `"healthy"` (200): all enabled subsystems are operational
#[cfg(feature = "federation")]
pub(crate) fn determine_status(
    db_healthy: bool,
    observers: Option<&ObserverHealth>,
    secrets: Option<&SecretsHealth>,
    federation: Option<&FederationHealth>,
) -> &'static str {
    if !db_healthy {
        return "unhealthy";
    }

    let observers_degraded = observers.is_some_and(|o| !o.running);
    let secrets_degraded = secrets.is_some_and(|s| !s.connected);
    let federation_degraded = federation.is_some_and(|f| {
        f.configured
            && f.subgraphs.iter().any(|sg| {
                matches!(sg.state, crate::federation::circuit_breaker::CircuitHealthState::Open)
            })
    });

    if observers_degraded || secrets_degraded || federation_degraded {
        "degraded"
    } else {
        "healthy"
    }
}

#[cfg(not(feature = "federation"))]
pub(crate) fn determine_status(
    db_healthy: bool,
    observers: Option<&ObserverHealth>,
    secrets: Option<&SecretsHealth>,
) -> &'static str {
    if !db_healthy {
        return "unhealthy";
    }

    let observers_degraded = observers.is_some_and(|o| !o.running);
    let secrets_degraded = secrets.is_some_and(|s| !s.connected);

    if observers_degraded || secrets_degraded {
        "degraded"
    } else {
        "healthy"
    }
}