use crate::{
agent_http::{
AgentHttpDrainReport, AgentHttpReadiness, AgentHttpServer, AgentHttpServerError,
AgentHttpServerHealth, AgentHttpServerOptions, AgentHttpService,
},
agent_maintenance::{
AgentMaintenance, AgentMaintenanceBinding, AgentMaintenanceError, AgentMaintenanceHealth,
AgentMaintenanceOptions, AgentMaintenanceReadiness, AgentMaintenanceReport,
},
agent_worker::{
AgentWorker, AgentWorkerBinding, AgentWorkerError, AgentWorkerHealth, AgentWorkerOptions,
AgentWorkerReadiness, AgentWorkerReport,
},
};
use futures_util::FutureExt;
use std::{net::SocketAddr, panic::AssertUnwindSafe, sync::Arc, time::Duration};
use thiserror::Error;
use tokio::{net::TcpListener, task::JoinHandle, time::sleep};
use tokio_util::sync::CancellationToken;
mod health;
pub mod operations;
use health::Phase;
pub use health::{AgentHostHealth, AgentHostStatus};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum AgentHostRole {
Http,
Worker,
Maintenance,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum AgentHostFailure {
Startup(AgentHostRole),
Role(AgentHostRole),
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
pub enum AgentHostError {
#[error("Agent host requires a loopback listener")]
InvalidListener,
#[error("Agent host ingress has already been claimed or stopped")]
AlreadyClaimed,
#[error("Agent host is stopped")]
Stopped,
}
pub struct AgentHostBindings {
http: AgentHttpService,
worker: AgentWorkerBinding,
maintenance: AgentMaintenanceBinding,
}
impl AgentHostBindings {
#[must_use]
pub fn new(
http: AgentHttpService,
worker: AgentWorkerBinding,
maintenance: AgentMaintenanceBinding,
) -> Self {
Self {
http,
worker,
maintenance,
}
}
}
pub struct AgentHostDependencies {
pub http: Arc<dyn AgentHttpReadiness>,
pub worker: Arc<dyn AgentWorkerReadiness>,
pub maintenance: Arc<dyn AgentMaintenanceReadiness>,
}
#[derive(Clone, Debug, Default)]
pub struct AgentHostOptions {
pub http: AgentHttpServerOptions,
pub worker: AgentWorkerOptions,
pub maintenance: AgentMaintenanceOptions,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct AgentHostReport {
pub failure: Option<AgentHostFailure>,
pub http: Option<Result<AgentHttpDrainReport, AgentHttpServerError>>,
pub worker: Option<Result<AgentWorkerReport, AgentWorkerError>>,
pub maintenance: Option<Result<AgentMaintenanceReport, AgentMaintenanceError>>,
}
pub struct AgentHost {
address: SocketAddr,
health: AgentHostHealth,
stop: CancellationToken,
task: Option<JoinHandle<AgentHostReport>>,
}
impl AgentHost {
pub fn launch(
listener: TcpListener,
bindings: AgentHostBindings,
dependencies: AgentHostDependencies,
options: AgentHostOptions,
) -> Result<Self, AgentHostError> {
let address = listener
.local_addr()
.map_err(|_| AgentHostError::InvalidListener)?;
if !address.ip().is_loopback() {
return Err(AgentHostError::InvalidListener);
}
bindings
.http
.claim_server()
.map_err(|_| AgentHostError::AlreadyClaimed)?;
let health = AgentHostHealth::new();
let stop = CancellationToken::new();
let guard = Roles {
service: bindings.http.clone(),
health: health.clone(),
http: None,
worker: None,
maintenance: None,
};
let task = tokio::spawn(run(
listener,
bindings,
dependencies,
options,
guard,
stop.clone(),
));
Ok(Self {
address,
health,
stop,
task: Some(task),
})
}
#[must_use]
pub const fn local_addr(&self) -> SocketAddr {
self.address
}
#[must_use]
pub fn health(&self) -> AgentHostHealth {
self.health.clone()
}
pub async fn wait_ready(&self) -> Result<(), AgentHostError> {
loop {
match self.health.status() {
AgentHostStatus::Ready => return Ok(()),
AgentHostStatus::Stopped | AgentHostStatus::Draining => {
return Err(AgentHostError::Stopped);
}
_ => sleep(Duration::from_millis(20)).await,
}
}
}
pub fn begin_shutdown(&self) {
self.health.draining();
self.stop.cancel();
}
pub async fn wait(&mut self) -> Result<AgentHostReport, AgentHostError> {
let result = self.task.as_mut().ok_or(AgentHostError::Stopped)?.await;
self.task = None;
result.map_err(|_| AgentHostError::Stopped)
}
pub async fn shutdown(&mut self) -> Result<AgentHostReport, AgentHostError> {
self.begin_shutdown();
self.wait().await
}
}
impl Drop for AgentHost {
fn drop(&mut self) {
self.begin_shutdown();
self.health.update(|state| state.phase = Phase::Stopped);
if let Some(task) = &self.task {
task.abort();
}
}
}
struct Roles {
service: AgentHttpService,
health: AgentHostHealth,
http: Option<AgentHttpServer>,
worker: Option<AgentWorker>,
maintenance: Option<AgentMaintenance>,
}
impl Drop for Roles {
fn drop(&mut self) {
self.health.update(|state| state.phase = Phase::Stopped);
self.service.shutdown();
}
}
async fn start(
listener: TcpListener,
bindings: AgentHostBindings,
dependencies: AgentHostDependencies,
options: AgentHostOptions,
roles: &mut Roles,
stage: &mut AgentHostRole,
) -> Result<(), ()> {
*stage = AgentHostRole::Maintenance;
let maintenance = AgentMaintenance::start(
bindings.maintenance,
dependencies.maintenance,
options.maintenance,
)
.await
.map_err(|_| ())?;
let maintenance_health = maintenance.health();
roles
.health
.update(|state| state.maintenance = Some(maintenance_health.clone()));
roles.maintenance = Some(maintenance);
*stage = AgentHostRole::Worker;
let worker = AgentWorker::start_supervised(
bindings.worker,
dependencies.worker,
options.worker,
Some(maintenance_health),
)
.await
.map_err(|_| ())?;
roles
.health
.update(|state| state.worker = Some(worker.health()));
roles.worker = Some(worker);
*stage = AgentHostRole::Http;
let http = AgentHttpServer::start_supervised(
listener,
bindings.http,
dependencies.http,
options.http,
Some(roles.health.clone()),
)
.await
.map_err(|_| ())?;
roles
.health
.update(|state| state.http = Some(http.health()));
roles.http = Some(http);
roles.health.update(|state| {
if state.phase == Phase::Starting {
state.phase = Phase::Running;
}
});
Ok(())
}
async fn run(
listener: TcpListener,
bindings: AgentHostBindings,
dependencies: AgentHostDependencies,
options: AgentHostOptions,
mut roles: Roles,
stop: CancellationToken,
) -> AgentHostReport {
let mut report = AgentHostReport::default();
let mut stage = AgentHostRole::Maintenance;
let startup = tokio::select! {
biased;
() = stop.cancelled() => None,
result = AssertUnwindSafe(start(listener, bindings, dependencies, options, &mut roles, &mut stage)).catch_unwind() => Some(result),
};
if let Some(result) = startup {
if matches!(result, Ok(Ok(()))) {
tokio::select! {
biased;
() = stop.cancelled() => {},
result = roles.http.as_mut().expect("started ingress").wait() => {
report.http = Some(result); roles.health.fail(AgentHostFailure::Role(AgentHostRole::Http));
},
result = roles.worker.as_mut().expect("started Worker").wait() => {
report.worker = Some(result); roles.health.fail(AgentHostFailure::Role(AgentHostRole::Worker));
},
result = roles.maintenance.as_mut().expect("started maintenance").wait() => {
report.maintenance = Some(result); roles.health.fail(AgentHostFailure::Role(AgentHostRole::Maintenance));
},
}
} else {
roles.health.fail(AgentHostFailure::Startup(stage));
}
}
roles.health.draining();
if let Some(http) = &mut roles.http {
if report.http.is_none() {
report.http = Some(http.shutdown().await);
}
}
if let Some(worker) = &mut roles.worker {
if report.worker.is_none() {
report.worker = Some(worker.shutdown().await);
}
}
if let Some(maintenance) = &mut roles.maintenance {
if report.maintenance.is_none() {
report.maintenance = Some(maintenance.shutdown().await);
}
}
if report
.http
.is_some_and(|r| r.is_err() || r.is_ok_and(|r| r.listener_failed))
{
roles
.health
.fail(AgentHostFailure::Role(AgentHostRole::Http));
}
if report
.worker
.is_some_and(|r| r.is_err() || r.is_ok_and(|r| r.failure.is_some()))
{
roles
.health
.fail(AgentHostFailure::Role(AgentHostRole::Worker));
}
if report
.maintenance
.is_some_and(|r| r.is_err() || r.is_ok_and(|r| r.failure.is_some()))
{
roles
.health
.fail(AgentHostFailure::Role(AgentHostRole::Maintenance));
}
report.failure = roles.health.failure();
while roles
.health
.http()
.is_some_and(|h| h.active_connections() != 0 || h.active_streams() != 0)
|| roles
.health
.worker()
.is_some_and(|h| h.active_ticks() != 0 || h.active_nodes() != 0)
|| roles
.health
.maintenance()
.is_some_and(|h| h.active_ticks() != 0)
{
sleep(Duration::from_millis(5)).await;
}
report
}