use crate::{
agent_maintenance::{
AgentMaintenance, AgentMaintenanceBinding, AgentMaintenanceError, AgentMaintenanceHealth,
AgentMaintenanceMutationOptions, AgentMaintenanceOptions, AgentMaintenanceReadiness,
AgentMaintenanceReport, AgentMaintenanceStatus,
},
agent_worker::{
AgentWorker, AgentWorkerBinding, AgentWorkerError, AgentWorkerExecutionOptions,
AgentWorkerHealth, AgentWorkerOptions, AgentWorkerReadiness, AgentWorkerReport,
AgentWorkerStatus,
},
};
use serde::{Serialize, de::DeserializeOwned};
use stateknot_core::{AgentSubmissionKey, BudgetLimits, TenantId};
use stateknot_runtime::{
AgentRunSnapshot, AgentRunTerminalOutcome, AgentServiceAuthorizer, AgentServiceBuildError,
AgentServiceCaller, AgentServiceError, AgentServiceRegistry, AgentServiceV1,
DurableTenantSchedulerBuildError, ExecutableGraphRegistry, GraphLifecycleEvidenceProvider,
JsonSchemaRegistryBuilder, TypedAgent, TypedAgentInputError, TypedAgentOutputError,
register_standard_agent_deadline_event_schema, register_standard_child_join_event_schema,
register_standard_child_reconciliation_event_schema,
register_standard_run_failure_close_event_schema,
};
use stateknot_store_postgres::{PostgresStore, StoreError};
use std::{
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use thiserror::Error;
use tokio::{task::JoinHandle, time::Instant};
use tokio_util::sync::CancellationToken;
const MIN_POLL_INTERVAL: Duration = Duration::from_millis(10);
const MAX_POLL_INTERVAL: Duration = Duration::from_secs(10);
const MIN_WAIT_TIMEOUT: Duration = Duration::from_millis(10);
const MAX_WAIT_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60);
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum InProcessAgentBindingError {
#[error(transparent)]
Service(#[from] AgentServiceBuildError),
#[error(transparent)]
Worker(#[from] DurableTenantSchedulerBuildError),
#[error("standard Agent maintenance schemas are unavailable")]
MaintenanceSchemas,
#[error("Agent maintenance binding is invalid: {0}")]
Maintenance(AgentMaintenanceError),
}
pub struct InProcessAgentBinding {
store: PostgresStore,
tenant: TenantId,
service: AgentServiceV1,
worker: AgentWorkerBinding,
maintenance: AgentMaintenanceBinding,
}
impl InProcessAgentBinding {
#[allow(clippy::too_many_arguments)]
pub fn tenant(
store: PostgresStore,
executable: ExecutableGraphRegistry,
deployments: AgentServiceRegistry,
authorizer: Arc<dyn AgentServiceAuthorizer>,
evidence: Arc<dyn GraphLifecycleEvidenceProvider>,
tenant: TenantId,
worker_options: AgentWorkerExecutionOptions,
maintenance_options: AgentMaintenanceMutationOptions,
) -> Result<Self, InProcessAgentBindingError> {
let service =
AgentServiceV1::new(store.clone(), executable.clone(), deployments, authorizer)?;
let worker = AgentWorkerBinding::tenant(
store.clone(),
executable,
evidence,
tenant.clone(),
worker_options,
)?;
let mut schemas = JsonSchemaRegistryBuilder::with_default_limits();
register_standard_agent_deadline_event_schema(&mut schemas)
.map_err(|_| InProcessAgentBindingError::MaintenanceSchemas)?;
register_standard_child_reconciliation_event_schema(&mut schemas)
.map_err(|_| InProcessAgentBindingError::MaintenanceSchemas)?;
register_standard_child_join_event_schema(&mut schemas)
.map_err(|_| InProcessAgentBindingError::MaintenanceSchemas)?;
register_standard_run_failure_close_event_schema(&mut schemas)
.map_err(|_| InProcessAgentBindingError::MaintenanceSchemas)?;
let schemas = schemas
.build()
.map_err(|_| InProcessAgentBindingError::MaintenanceSchemas)?;
let maintenance = AgentMaintenanceBinding::new(
store.clone(),
schemas,
vec![tenant.clone()],
maintenance_options,
)
.map_err(InProcessAgentBindingError::Maintenance)?;
Ok(Self {
store,
tenant,
service,
worker,
maintenance,
})
}
}
pub struct InProcessAgentDependencies {
pub worker: Arc<dyn AgentWorkerReadiness>,
pub maintenance: Arc<dyn AgentMaintenanceReadiness>,
}
#[derive(Clone, Debug, Default)]
pub struct InProcessAgentRuntimeOptions {
pub worker: AgentWorkerOptions,
pub maintenance: AgentMaintenanceOptions,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum InProcessAgentRole {
Worker,
Maintenance,
}
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct InProcessAgentRuntimeReport {
pub failure: Option<InProcessAgentRole>,
pub worker: Option<Result<AgentWorkerReport, AgentWorkerError>>,
pub maintenance: Option<Result<AgentMaintenanceReport, AgentMaintenanceError>>,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
#[non_exhaustive]
pub enum InProcessAgentStartError {
#[error("Agent maintenance could not start: {0}")]
Maintenance(AgentMaintenanceError),
#[error("Agent Worker could not start: {0}")]
Worker(AgentWorkerError),
#[error("Agent Worker startup failed and maintenance cleanup did not join")]
WorkerCleanup,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum InProcessAgentRuntimeStatus {
Ready,
Unavailable,
Draining,
Stopped,
}
#[derive(Clone)]
pub struct InProcessAgentRuntimeHealth {
worker: AgentWorkerHealth,
maintenance: AgentMaintenanceHealth,
stopping: Arc<AtomicBool>,
}
impl InProcessAgentRuntimeHealth {
#[must_use]
pub fn status(&self) -> InProcessAgentRuntimeStatus {
if self.stopping.load(Ordering::Acquire)
&& !matches!(
(self.worker.status(), self.maintenance.status()),
(AgentWorkerStatus::Stopped, _) | (_, AgentMaintenanceStatus::Stopped)
)
{
return InProcessAgentRuntimeStatus::Draining;
}
match (self.worker.status(), self.maintenance.status()) {
(AgentWorkerStatus::Stopped, _) | (_, AgentMaintenanceStatus::Stopped) => {
InProcessAgentRuntimeStatus::Stopped
}
(AgentWorkerStatus::Draining, _) | (_, AgentMaintenanceStatus::Draining) => {
InProcessAgentRuntimeStatus::Draining
}
(AgentWorkerStatus::Ready, AgentMaintenanceStatus::Ready) => {
InProcessAgentRuntimeStatus::Ready
}
_ => InProcessAgentRuntimeStatus::Unavailable,
}
}
#[must_use]
pub const fn worker(&self) -> &AgentWorkerHealth {
&self.worker
}
#[must_use]
pub const fn maintenance(&self) -> &AgentMaintenanceHealth {
&self.maintenance
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
pub enum InProcessAgentRuntimeError {
#[error("in-process Agent runtime is stopped")]
Stopped,
}
pub struct InProcessAgentRuntime {
store: PostgresStore,
tenant: TenantId,
service: AgentServiceV1,
health: InProcessAgentRuntimeHealth,
stop: CancellationToken,
task: Option<JoinHandle<InProcessAgentRuntimeReport>>,
}
impl InProcessAgentRuntime {
pub async fn start(
binding: InProcessAgentBinding,
dependencies: InProcessAgentDependencies,
options: InProcessAgentRuntimeOptions,
) -> Result<Self, InProcessAgentStartError> {
let mut maintenance = AgentMaintenance::start(
binding.maintenance,
dependencies.maintenance,
options.maintenance,
)
.await
.map_err(InProcessAgentStartError::Maintenance)?;
let maintenance_health = maintenance.health();
let worker = match AgentWorker::start_supervised(
binding.worker,
dependencies.worker,
options.worker,
Some(maintenance_health.clone()),
)
.await
{
Ok(worker) => worker,
Err(error) => {
return if maintenance.shutdown().await.is_ok() {
Err(InProcessAgentStartError::Worker(error))
} else {
Err(InProcessAgentStartError::WorkerCleanup)
};
}
};
let stopping = Arc::new(AtomicBool::new(false));
let health = InProcessAgentRuntimeHealth {
worker: worker.health(),
maintenance: maintenance_health,
stopping,
};
let stop = CancellationToken::new();
let task = tokio::spawn(supervise(worker, maintenance, stop.clone()));
Ok(Self {
store: binding.store,
tenant: binding.tenant,
service: binding.service,
health,
stop,
task: Some(task),
})
}
#[must_use]
pub fn health(&self) -> InProcessAgentRuntimeHealth {
self.health.clone()
}
pub fn agent<I, O>(
&self,
codec: TypedAgent<I, O>,
caller: AgentServiceCaller,
) -> Result<InProcessAgent<'_, I, O>, InProcessAgentBindError> {
if caller.tenant_id() != &self.tenant {
return Err(InProcessAgentBindError::TenantMismatch);
}
Ok(InProcessAgent {
runtime: self,
codec,
caller,
options: InProcessAgentRunOptions::default(),
})
}
pub fn begin_shutdown(&self) {
self.health.stopping.store(true, Ordering::Release);
self.stop.cancel();
}
pub async fn wait(
&mut self,
) -> Result<InProcessAgentRuntimeReport, InProcessAgentRuntimeError> {
let result = self
.task
.as_mut()
.ok_or(InProcessAgentRuntimeError::Stopped)?
.await;
self.task = None;
result.map_err(|_| InProcessAgentRuntimeError::Stopped)
}
pub async fn shutdown(
&mut self,
) -> Result<InProcessAgentRuntimeReport, InProcessAgentRuntimeError> {
self.begin_shutdown();
self.wait().await
}
}
impl Drop for InProcessAgentRuntime {
fn drop(&mut self) {
self.begin_shutdown();
if let Some(task) = &self.task {
task.abort();
}
}
}
async fn supervise(
mut worker: AgentWorker,
mut maintenance: AgentMaintenance,
stop: CancellationToken,
) -> InProcessAgentRuntimeReport {
let mut report = InProcessAgentRuntimeReport::default();
tokio::select! {
biased;
() = stop.cancelled() => {},
result = worker.wait() => {
report.worker = Some(result);
report.failure = Some(InProcessAgentRole::Worker);
},
result = maintenance.wait() => {
report.maintenance = Some(result);
report.failure = Some(InProcessAgentRole::Maintenance);
},
}
if report.worker.is_none() {
report.worker = Some(worker.shutdown().await);
}
if report.maintenance.is_none() {
report.maintenance = Some(maintenance.shutdown().await);
}
report
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
pub enum InProcessAgentBindError {
#[error("Agent caller tenant does not match the in-process runtime tenant")]
TenantMismatch,
}
#[derive(Clone, Copy, Debug)]
pub struct InProcessAgentRunOptions {
poll_interval: Duration,
wait_timeout: Duration,
}
impl InProcessAgentRunOptions {
pub fn new(
poll_interval: Duration,
wait_timeout: Duration,
) -> Result<Self, InProcessAgentRunOptionsError> {
if !(MIN_POLL_INTERVAL..=MAX_POLL_INTERVAL).contains(&poll_interval)
|| !(MIN_WAIT_TIMEOUT..=MAX_WAIT_TIMEOUT).contains(&wait_timeout)
{
return Err(InProcessAgentRunOptionsError);
}
Ok(Self {
poll_interval,
wait_timeout,
})
}
#[must_use]
pub const fn poll_interval(self) -> Duration {
self.poll_interval
}
#[must_use]
pub const fn wait_timeout(self) -> Duration {
self.wait_timeout
}
}
impl Default for InProcessAgentRunOptions {
fn default() -> Self {
Self {
poll_interval: Duration::from_millis(50),
wait_timeout: Duration::from_secs(30),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)]
#[error("in-process Agent wait options are outside supported bounds")]
pub struct InProcessAgentRunOptionsError;
pub struct InProcessAgentRequest<I> {
key: AgentSubmissionKey,
input: I,
budget_limits: BudgetLimits,
}
impl<I> InProcessAgentRequest<I> {
#[must_use]
pub const fn new(key: AgentSubmissionKey, input: I, budget_limits: BudgetLimits) -> Self {
Self {
key,
input,
budget_limits,
}
}
#[must_use]
pub const fn submission_key(&self) -> &AgentSubmissionKey {
&self.key
}
#[must_use]
pub const fn input(&self) -> &I {
&self.input
}
#[must_use]
pub const fn budget_limits(&self) -> &BudgetLimits {
&self.budget_limits
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum InProcessAgentPendingReason {
WaitTimeout,
RuntimeUnavailable,
}
#[derive(Debug)]
#[non_exhaustive]
pub enum InProcessAgentRun<O> {
Succeeded {
output: O,
snapshot: AgentRunSnapshot,
},
Failed {
snapshot: AgentRunSnapshot,
},
Cancelled {
snapshot: AgentRunSnapshot,
},
Pending {
snapshot: AgentRunSnapshot,
reason: InProcessAgentPendingReason,
},
Quarantined {
snapshot: AgentRunSnapshot,
},
}
impl<O> InProcessAgentRun<O> {
#[must_use]
pub const fn snapshot(&self) -> &AgentRunSnapshot {
match self {
Self::Succeeded { snapshot, .. }
| Self::Failed { snapshot }
| Self::Cancelled { snapshot }
| Self::Pending { snapshot, .. }
| Self::Quarantined { snapshot } => snapshot,
}
}
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum InProcessAgentRunError {
#[error(transparent)]
Input(#[from] TypedAgentInputError),
#[error(transparent)]
Service(#[from] AgentServiceError),
#[error(transparent)]
Store(#[from] StoreError),
#[error(transparent)]
Output(#[from] TypedAgentOutputError),
#[error("durable Agent snapshot does not match its immutable admission")]
AdmissionMismatch,
#[error("terminal Agent snapshot omitted its durable outcome")]
MissingTerminalOutcome,
#[error("in-process Agent runtime is unavailable for new submissions")]
RuntimeUnavailable,
#[error("terminal Agent outcome is unsupported by this runtime")]
UnsupportedTerminalOutcome,
}
pub struct InProcessAgent<'runtime, I, O> {
runtime: &'runtime InProcessAgentRuntime,
codec: TypedAgent<I, O>,
caller: AgentServiceCaller,
options: InProcessAgentRunOptions,
}
impl<I, O> InProcessAgent<'_, I, O> {
#[must_use]
pub const fn with_options(mut self, options: InProcessAgentRunOptions) -> Self {
self.options = options;
self
}
#[must_use]
pub const fn codec(&self) -> &TypedAgent<I, O> {
&self.codec
}
}
impl<I, O> InProcessAgent<'_, I, O>
where
I: Serialize,
O: DeserializeOwned,
{
pub async fn run(
&self,
request: InProcessAgentRequest<I>,
) -> Result<InProcessAgentRun<O>, InProcessAgentRunError> {
if self.runtime.health.status() != InProcessAgentRuntimeStatus::Ready {
return Err(InProcessAgentRunError::RuntimeUnavailable);
}
let durable_request = self
.codec
.prepare_request(&request.input, request.budget_limits)?;
let admission = self
.runtime
.service
.submit(
self.caller.clone(),
&request.key,
self.codec.descriptor().metadata().identity(),
durable_request,
)
.await?;
let run_id = admission.snapshot().provenance().run_id();
let mut snapshot = self
.runtime
.service
.load(self.caller.clone(), run_id)
.await?;
let deadline = Instant::now() + self.options.wait_timeout;
loop {
if snapshot.is_quarantined() {
return Ok(InProcessAgentRun::Quarantined { snapshot });
}
if snapshot.status().is_terminal() {
return self.decode_terminal(snapshot).await;
}
if matches!(
self.runtime.health.status(),
InProcessAgentRuntimeStatus::Draining | InProcessAgentRuntimeStatus::Stopped
) {
return Ok(InProcessAgentRun::Pending {
snapshot,
reason: InProcessAgentPendingReason::RuntimeUnavailable,
});
}
let now = Instant::now();
if now >= deadline {
return Ok(InProcessAgentRun::Pending {
snapshot,
reason: InProcessAgentPendingReason::WaitTimeout,
});
}
tokio::time::sleep(
self.options
.poll_interval
.min(deadline.saturating_duration_since(now)),
)
.await;
snapshot = self
.runtime
.service
.load(self.caller.clone(), run_id)
.await?;
}
}
async fn decode_terminal(
&self,
snapshot: AgentRunSnapshot,
) -> Result<InProcessAgentRun<O>, InProcessAgentRunError> {
let stored = self
.runtime
.store
.load_agent_admission(self.caller.tenant_id(), snapshot.provenance().run_id())
.await?;
let admission = stored.admission();
let intent = admission.intent();
if admission.digest() != snapshot.admission_digest()
|| intent.provenance() != snapshot.provenance()
|| intent.descriptor().metadata().identity()
!= self.codec.descriptor().metadata().identity()
{
return Err(InProcessAgentRunError::AdmissionMismatch);
}
match snapshot
.outcome()
.ok_or(InProcessAgentRunError::MissingTerminalOutcome)?
{
AgentRunTerminalOutcome::Succeeded { result } => {
let output = self.codec.decode_result(
result,
snapshot.provenance(),
intent.request(),
intent.budget(),
)?;
Ok(InProcessAgentRun::Succeeded { output, snapshot })
}
AgentRunTerminalOutcome::Failed { .. } => Ok(InProcessAgentRun::Failed { snapshot }),
AgentRunTerminalOutcome::Cancelled { .. } => {
Ok(InProcessAgentRun::Cancelled { snapshot })
}
_ => Err(InProcessAgentRunError::UnsupportedTerminalOutcome),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn wait_options_are_finite() {
assert!(
InProcessAgentRunOptions::new(Duration::from_millis(10), Duration::from_millis(10))
.is_ok()
);
assert!(
InProcessAgentRunOptions::new(Duration::from_millis(9), Duration::from_secs(1))
.is_err()
);
assert!(
InProcessAgentRunOptions::new(Duration::from_millis(10), Duration::from_secs(86_401))
.is_err()
);
}
}