use super::{AgentHttpService, HttpError, failure};
use crate::http_transport::{connection, count_failure};
use axum::{
extract::Request,
middleware::{self, Next},
};
use stateknot_core::{BoxFuture, EventId};
use std::{net::SocketAddr, sync::Arc};
use thiserror::Error;
use tokio::{
net::TcpListener,
task::{JoinHandle, JoinSet},
time::{Instant, sleep, timeout, timeout_at},
};
use tokio_util::sync::CancellationToken;
mod health;
mod options;
pub use health::{AgentHttpServerHealth, AgentHttpServerStatus};
pub use options::AgentHttpServerOptions;
pub trait AgentHttpReadiness: Send + Sync {
fn check(&self) -> BoxFuture<'_, Result<(), AgentHttpReadinessError>>;
}
#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
#[error("HTTP ingress host dependencies are unavailable")]
pub struct AgentHttpReadinessError;
impl AgentHttpReadiness for stateknot_runtime::agent_policy::AgentResourcePolicy {
fn check(&self) -> BoxFuture<'_, Result<(), AgentHttpReadinessError>> {
Box::pin(async move { self.check_readiness().map_err(|_| AgentHttpReadinessError) })
}
}
#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum AgentHttpServerError {
#[error("HTTP ingress requires a loopback listener")]
InvalidListener,
#[error("HTTP ingress service has already been claimed or stopped")]
AlreadyClaimed,
#[error("HTTP ingress dependencies are unavailable")]
Unavailable,
#[error("HTTP ingress runtime is stopped")]
Stopped,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct AgentHttpDrainReport {
pub forced_connections: usize,
pub connection_failures: u64,
pub rejected_connections: u64,
pub listener_failed: bool,
}
pub struct AgentHttpServer {
address: SocketAddr,
health: AgentHttpServerHealth,
stop: CancellationToken,
task: Option<JoinHandle<AgentHttpDrainReport>>,
}
impl AgentHttpServer {
pub async fn start(
listener: TcpListener,
http: AgentHttpService,
host: Arc<dyn AgentHttpReadiness>,
options: AgentHttpServerOptions,
) -> Result<Self, AgentHttpServerError> {
Self::start_supervised(listener, http, host, options, None).await
}
pub(crate) async fn start_supervised(
listener: TcpListener,
http: AgentHttpService,
host: Arc<dyn AgentHttpReadiness>,
options: AgentHttpServerOptions,
supervisor: Option<crate::agent_host::AgentHostHealth>,
) -> Result<Self, AgentHttpServerError> {
let address = listener
.local_addr()
.map_err(|_| AgentHttpServerError::InvalidListener)?;
if !address.ip().is_loopback() {
return Err(AgentHttpServerError::InvalidListener);
}
if supervisor.is_none() {
http.claim_server()?;
}
let health = AgentHttpServerHealth::new(options.freshness, http.inner.stream_tasks.clone());
let guard = RuntimeGuard {
http,
health: health.clone(),
};
if !check(&guard.http, host.as_ref(), &options).await
|| guard.http.inner.shutdown.is_cancelled()
{
return Err(AgentHttpServerError::Unavailable);
}
health.update(AgentHttpServerStatus::Ready);
let stop = CancellationToken::new();
let task = tokio::spawn(run(
listener,
guard,
host,
options,
stop.clone(),
supervisor,
));
Ok(Self {
address,
health,
stop,
task: Some(task),
})
}
#[must_use]
pub const fn local_addr(&self) -> SocketAddr {
self.address
}
#[must_use]
pub fn health(&self) -> AgentHttpServerHealth {
self.health.clone()
}
pub fn begin_shutdown(&self) {
self.health.update(AgentHttpServerStatus::Draining);
self.stop.cancel();
}
pub async fn shutdown(&mut self) -> Result<AgentHttpDrainReport, AgentHttpServerError> {
self.begin_shutdown();
self.wait().await
}
pub async fn wait(&mut self) -> Result<AgentHttpDrainReport, AgentHttpServerError> {
let result = self
.task
.as_mut()
.ok_or(AgentHttpServerError::Stopped)?
.await
.map_err(|_| AgentHttpServerError::Stopped);
self.task.take();
result
}
}
impl Drop for AgentHttpServer {
fn drop(&mut self) {
self.begin_shutdown();
self.health.update(AgentHttpServerStatus::Stopped);
if let Some(task) = self.task.take() {
task.abort();
}
}
}
struct RuntimeGuard {
http: AgentHttpService,
health: AgentHttpServerHealth,
}
impl Drop for RuntimeGuard {
fn drop(&mut self) {
self.http.shutdown();
self.health.update(AgentHttpServerStatus::Stopped);
}
}
async fn check(
http: &AgentHttpService,
host: &dyn AgentHttpReadiness,
options: &AgentHttpServerOptions,
) -> bool {
timeout(options.probe_timeout, async {
http.inner
.service
.check_readiness()
.await
.map_err(|_| AgentHttpReadinessError)?;
host.check().await
})
.await
.is_ok_and(|result| result.is_ok())
}
async fn monitor(
guard: &RuntimeGuard,
host: &dyn AgentHttpReadiness,
options: &AgentHttpServerOptions,
) {
loop {
sleep(options.probe_interval).await;
let status = if check(&guard.http, host, options).await {
AgentHttpServerStatus::Ready
} else {
AgentHttpServerStatus::Unavailable
};
guard.health.update(status);
}
}
async fn run(
listener: TcpListener,
guard: RuntimeGuard,
host: Arc<dyn AgentHttpReadiness>,
options: AgentHttpServerOptions,
stop: CancellationToken,
supervisor: Option<crate::agent_host::AgentHostHealth>,
) -> AgentHttpDrainReport {
let gate = guard.health.clone();
let router =
guard
.http
.router()
.layer(middleware::from_fn(move |request: Request, next: Next| {
let gate = gate.clone();
let supervisor = supervisor.clone();
async move {
if gate.is_ready()
&& supervisor
.as_ref()
.is_none_or(crate::agent_host::AgentHostHealth::allows_ingress)
{
next.run(request).await
} else {
failure(HttpError::Unavailable, EventId::generate())
}
}
}));
let mut tasks = JoinSet::new();
let drain = CancellationToken::new();
let mut report = AgentHttpDrainReport::default();
{
let probe = monitor(&guard, host.as_ref(), &options);
tokio::pin!(probe);
loop {
tokio::select! {
biased;
() = stop.cancelled() => break,
() = guard.http.inner.shutdown.cancelled() => break,
() = &mut probe => break,
result = tasks.join_next(), if !tasks.is_empty() => {
count_failure(result.as_ref(), &mut report);
}
accepted = listener.accept() => {
let Ok((socket, _)) = accepted else { report.listener_failed = true; break; };
if tasks.len() == options.max_connections {
report.rejected_connections = report.rejected_connections.saturating_add(1);
drop(socket);
} else {
tasks.spawn(connection(socket, router.clone(), drain.clone(), options.clone(), guard.health.connection()));
}
}
}
}
} guard.health.update(AgentHttpServerStatus::Draining);
drop(listener);
drain.cancel();
let deadline = Instant::now() + options.drain_timeout;
while !tasks.is_empty() {
if let Ok(result) = timeout_at(deadline, tasks.join_next()).await {
count_failure(result.as_ref(), &mut report);
} else {
report.forced_connections = tasks.len();
guard.http.shutdown();
tasks.abort_all();
while tasks.join_next().await.is_some() {}
}
}
guard.http.shutdown();
guard.http.inner.stream_tasks.close();
guard.http.inner.stream_tasks.wait().await;
report
}