use std::{collections::BTreeMap, fmt, sync::Arc};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::json;
use stateknot_core::{
AgentAdmissionAuthority, AgentAdmissionBudgetLayer, AgentDescriptor, AgentRequest,
AgentSubmissionKey, BoundedJson, BoxFuture, CapabilityIdentity, CheckpointState, CompiledGraph,
Digest, EventId, Failure, FailureCategory, FailureCode, FailureId, FailureMessage,
FailureOrigin, GraphReference, GraphSchemaValidationError, JournalAppend, JournalEventIntent,
JournalEventKind, JournalExpectation, JournalPayload, PrincipalIdentity, RetryAdvice,
RunCancellationRequest, RunId, RunStatus, RunTransition, TenantId,
};
use stateknot_store_postgres::{
AppendOutcome, PostgresStore, RunProjection, StoreError, WaitAbandonmentCommitOutcome,
};
use thiserror::Error;
mod activity;
pub mod policy;
mod readiness;
pub use activity::AgentRunActivityPage;
pub use readiness::AgentServiceReadinessError;
use crate::{
AgentRunAdmissionOutcome, AgentRunIds, AgentRunSnapshot, DurableAgentAdmissionError,
DurableAgentAdmissionRequest, DurableAgentAdmissionRequestError, DurableAgentRuns,
DurableAgentRunsBuildError, DurableAgentRunsError, ExecutableGraphRegistry,
ProviderNativeAgentGraph, StandardAgentServiceControlSchemaError,
standard_agent_service_control_event_schema,
};
pub const AGENT_SERVICE_API_VERSION: u16 = 1;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AgentServiceCaller {
tenant_id: TenantId,
principal: PrincipalIdentity,
}
impl AgentServiceCaller {
#[must_use]
pub const fn new(tenant_id: TenantId, principal: PrincipalIdentity) -> Self {
Self {
tenant_id,
principal,
}
}
#[must_use]
pub const fn tenant_id(&self) -> &TenantId {
&self.tenant_id
}
#[must_use]
pub const fn principal(&self) -> &PrincipalIdentity {
&self.principal
}
}
#[derive(Clone, Debug)]
pub struct AgentServiceSubmissionAuthorization {
caller: AgentServiceCaller,
agent: CapabilityIdentity,
request: AgentRequest,
}
impl AgentServiceSubmissionAuthorization {
#[must_use]
pub const fn caller(&self) -> &AgentServiceCaller {
&self.caller
}
#[must_use]
pub const fn agent(&self) -> &CapabilityIdentity {
&self.agent
}
#[must_use]
pub const fn request(&self) -> &AgentRequest {
&self.request
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum AgentServiceRunTarget {
Run(RunId),
Submission(Digest),
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum AgentServiceRunOperation {
Read,
Cancel,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AgentServiceRunAuthorization {
caller: AgentServiceCaller,
target: AgentServiceRunTarget,
operation: AgentServiceRunOperation,
}
impl AgentServiceRunAuthorization {
#[must_use]
pub const fn caller(&self) -> &AgentServiceCaller {
&self.caller
}
#[must_use]
pub const fn target(&self) -> &AgentServiceRunTarget {
&self.target
}
#[must_use]
pub const fn operation(&self) -> AgentServiceRunOperation {
self.operation
}
}
#[derive(Clone, Debug)]
pub struct AgentServiceSubmissionGrant {
authority: AgentAdmissionAuthority,
budget_layers: Vec<AgentAdmissionBudgetLayer>,
}
impl AgentServiceSubmissionGrant {
#[must_use]
pub fn new(
authority: AgentAdmissionAuthority,
budget_layers: Vec<AgentAdmissionBudgetLayer>,
) -> Self {
Self {
authority,
budget_layers,
}
}
#[must_use]
pub const fn authority(&self) -> &AgentAdmissionAuthority {
&self.authority
}
#[must_use]
pub fn budget_layers(&self) -> &[AgentAdmissionBudgetLayer] {
&self.budget_layers
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct AgentServiceRunGrant {
principal: PrincipalIdentity,
policy: CapabilityIdentity,
policy_digest: Digest,
decision_digest: Digest,
}
impl AgentServiceRunGrant {
#[must_use]
pub const fn new(
principal: PrincipalIdentity,
policy: CapabilityIdentity,
policy_digest: Digest,
decision_digest: Digest,
) -> Self {
Self {
principal,
policy,
policy_digest,
decision_digest,
}
}
#[must_use]
pub const fn principal(&self) -> &PrincipalIdentity {
&self.principal
}
#[must_use]
pub const fn policy(&self) -> &CapabilityIdentity {
&self.policy
}
#[must_use]
pub const fn policy_digest(&self) -> Digest {
self.policy_digest
}
#[must_use]
pub const fn decision_digest(&self) -> Digest {
self.decision_digest
}
}
#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum AgentServiceAuthorizationError {
#[error("Agent service authentication was not accepted")]
Unauthenticated,
#[error("Agent service authorization was denied")]
Denied,
#[error("Agent service authorization is temporarily unavailable")]
Unavailable,
#[error("Agent service authorization evidence is invalid")]
InvalidEvidence,
}
pub trait AgentServiceAuthorizer: Send + Sync + 'static {
fn authorize_submission(
&self,
context: AgentServiceSubmissionAuthorization,
) -> BoxFuture<'_, Result<AgentServiceSubmissionGrant, AgentServiceAuthorizationError>>;
fn authorize_run(
&self,
context: AgentServiceRunAuthorization,
) -> BoxFuture<'_, Result<AgentServiceRunGrant, AgentServiceAuthorizationError>>;
}
pub trait AgentServiceDeployment: Send + Sync + 'static {
fn descriptor(&self) -> &AgentDescriptor;
fn graph(&self) -> &CompiledGraph;
fn initial_state(&self) -> Result<CheckpointState, AgentServiceDeploymentError>;
}
impl AgentServiceDeployment for ProviderNativeAgentGraph {
fn descriptor(&self) -> &AgentDescriptor {
ProviderNativeAgentGraph::descriptor(self)
}
fn graph(&self) -> &CompiledGraph {
ProviderNativeAgentGraph::graph(self)
}
fn initial_state(&self) -> Result<CheckpointState, AgentServiceDeploymentError> {
ProviderNativeAgentGraph::initial_state(self)
.map_err(|_| AgentServiceDeploymentError::InvalidInitialState)
}
}
#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum AgentServiceDeploymentError {
#[error("Agent service deployment produced invalid initial state")]
InvalidInitialState,
}
struct AgentServiceBinding {
descriptor: AgentDescriptor,
graph: GraphReference,
deployment: Arc<dyn AgentServiceDeployment>,
}
#[derive(Default)]
pub struct AgentServiceRegistryBuilder {
bindings: BTreeMap<CapabilityIdentity, AgentServiceBinding>,
}
impl AgentServiceRegistryBuilder {
pub const MAX_BINDINGS: usize = 4096;
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn register(
&mut self,
deployment: Arc<dyn AgentServiceDeployment>,
) -> Result<(), AgentServiceRegistryError> {
if self.bindings.len() == Self::MAX_BINDINGS {
return Err(AgentServiceRegistryError::TooManyBindings);
}
let descriptor = deployment.descriptor().clone();
let graph = deployment.graph();
if descriptor.input_schema() != graph.input_schema()
|| descriptor.output_schema() != graph.output_schema()
{
return Err(AgentServiceRegistryError::SchemaMismatch);
}
let graph = graph.reference();
let identity = descriptor.metadata().identity().clone();
if self.bindings.contains_key(&identity) {
return Err(AgentServiceRegistryError::DuplicateIdentity {
identity: Box::new(identity),
});
}
self.bindings.insert(
identity,
AgentServiceBinding {
descriptor,
graph,
deployment,
},
);
Ok(())
}
#[must_use]
pub fn build(self) -> AgentServiceRegistry {
AgentServiceRegistry {
bindings: Arc::new(self.bindings),
}
}
}
impl fmt::Debug for AgentServiceRegistryBuilder {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("AgentServiceRegistryBuilder")
.field("bindings", &self.bindings.len())
.finish_non_exhaustive()
}
}
#[derive(Clone)]
pub struct AgentServiceRegistry {
bindings: Arc<BTreeMap<CapabilityIdentity, AgentServiceBinding>>,
}
impl AgentServiceRegistry {
fn resolve(
&self,
identity: &CapabilityIdentity,
) -> Result<&AgentServiceBinding, AgentServiceRegistryError> {
let binding = self.bindings.get(identity).ok_or_else(|| {
AgentServiceRegistryError::MissingBinding {
identity: Box::new(identity.clone()),
}
})?;
if binding.deployment.descriptor() != &binding.descriptor
|| binding.deployment.graph().reference() != binding.graph
{
return Err(AgentServiceRegistryError::DeploymentDrift {
identity: Box::new(identity.clone()),
});
}
Ok(binding)
}
#[must_use]
pub fn len(&self) -> usize {
self.bindings.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.bindings.is_empty()
}
}
impl fmt::Debug for AgentServiceRegistry {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("AgentServiceRegistry")
.field("bindings", &self.bindings.len())
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum AgentServiceRegistryError {
#[error("Agent service registry contains too many bindings")]
TooManyBindings,
#[error("Agent service identity was registered more than once")]
DuplicateIdentity {
identity: Box<CapabilityIdentity>,
},
#[error("Agent service identity has no installed deployment")]
MissingBinding {
identity: Box<CapabilityIdentity>,
},
#[error("Agent service descriptor and graph schemas do not match")]
SchemaMismatch,
#[error("Agent service deployment differs from its startup snapshot")]
DeploymentDrift {
identity: Box<CapabilityIdentity>,
},
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AgentCancellationIds {
event_id: EventId,
failure_id: FailureId,
}
impl AgentCancellationIds {
#[must_use]
pub const fn new(event_id: EventId, failure_id: FailureId) -> Self {
Self {
event_id,
failure_id,
}
}
#[must_use]
pub fn generate() -> Self {
Self::new(EventId::generate(), FailureId::generate())
}
#[must_use]
pub const fn event_id(self) -> EventId {
self.event_id
}
#[must_use]
pub const fn failure_id(self) -> FailureId {
self.failure_id
}
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum AgentCancellationOutcome {
Committed(AgentRunSnapshot),
Idempotent(AgentRunSnapshot),
}
impl AgentCancellationOutcome {
#[must_use]
pub const fn snapshot(&self) -> &AgentRunSnapshot {
match self {
Self::Committed(snapshot) | Self::Idempotent(snapshot) => snapshot,
}
}
}
#[derive(Clone)]
pub struct AgentServiceV1 {
store: PostgresStore,
runs: DurableAgentRuns,
deployments: AgentServiceRegistry,
authorizer: Arc<dyn AgentServiceAuthorizer>,
schemas: crate::JsonSchemaRegistry,
control_event_schema: stateknot_core::SchemaReference,
}
impl AgentServiceV1 {
pub fn new(
store: PostgresStore,
executable: ExecutableGraphRegistry,
deployments: AgentServiceRegistry,
authorizer: Arc<dyn AgentServiceAuthorizer>,
) -> Result<Self, AgentServiceBuildError> {
let (control_event_schema, _) = standard_agent_service_control_event_schema()?;
if !executable.schemas().contains(&control_event_schema) {
return Err(AgentServiceBuildError::ControlEventSchemaUnavailable);
}
let schemas = executable.schemas().clone();
let runs = DurableAgentRuns::new(store.clone(), executable)?;
Ok(Self {
store,
runs,
deployments,
authorizer,
schemas,
control_event_schema,
})
}
pub fn submit<'service>(
&'service self,
caller: AgentServiceCaller,
key: &'service AgentSubmissionKey,
agent: &'service CapabilityIdentity,
request: AgentRequest,
) -> BoxFuture<'service, Result<AgentRunAdmissionOutcome, AgentServiceError>> {
Box::pin(self.submit_inner(caller, key, agent, request))
}
async fn submit_inner(
&self,
caller: AgentServiceCaller,
key: &AgentSubmissionKey,
agent: &CapabilityIdentity,
request: AgentRequest,
) -> Result<AgentRunAdmissionOutcome, AgentServiceError> {
let grant = self
.authorizer
.authorize_submission(AgentServiceSubmissionAuthorization {
caller: caller.clone(),
agent: agent.clone(),
request: request.clone(),
})
.await?;
if grant.authority().principal() != caller.principal() {
return Err(AgentServiceError::AuthorizationPrincipalMismatch);
}
let binding = self.deployments.resolve(agent)?;
if let Some(outcome) = self
.recover_submission(
&caller,
key,
&binding.descriptor,
&request,
&binding.graph,
&grant,
)
.await?
{
return Ok(outcome);
}
let initial_state = binding.deployment.initial_state()?;
if initial_state.schema() != binding.graph.state_schema() {
return Err(AgentServiceError::Deployment(
AgentServiceDeploymentError::InvalidInitialState,
));
}
let durable = DurableAgentAdmissionRequest::new(
caller.tenant_id().clone(),
AgentRunIds::generate(),
binding.descriptor.clone(),
request.clone(),
grant.budget_layers.clone(),
binding.graph.clone(),
grant.authority.clone(),
initial_state,
)?;
match self.runs.submit(key, durable).await {
Ok(outcome) => Ok(outcome),
Err(DurableAgentRunsError::Admission(DurableAgentAdmissionError::Store(
StoreError::AgentSubmissionConflict,
))) => self
.recover_submission(
&caller,
key,
&binding.descriptor,
&request,
&binding.graph,
&grant,
)
.await?
.ok_or(AgentServiceError::SubmissionConflict),
Err(error) => Err(error.into()),
}
}
pub async fn load(
&self,
caller: AgentServiceCaller,
run_id: RunId,
) -> Result<AgentRunSnapshot, AgentServiceError> {
self.authorize_run(
&caller,
AgentServiceRunTarget::Run(run_id),
AgentServiceRunOperation::Read,
)
.await?;
self.runs
.load(caller.tenant_id(), run_id)
.await
.map_err(Into::into)
}
pub async fn load_by_key(
&self,
caller: AgentServiceCaller,
key: &AgentSubmissionKey,
) -> Result<AgentRunSnapshot, AgentServiceError> {
self.authorize_run(
&caller,
AgentServiceRunTarget::Submission(key.digest_for(caller.tenant_id())),
AgentServiceRunOperation::Read,
)
.await?;
self.runs
.load_by_key(caller.tenant_id(), key)
.await
.map_err(Into::into)
}
pub async fn request_cancellation(
&self,
caller: AgentServiceCaller,
run_id: RunId,
ids: AgentCancellationIds,
) -> Result<AgentCancellationOutcome, AgentServiceError> {
let grant = self
.authorize_run(
&caller,
AgentServiceRunTarget::Run(run_id),
AgentServiceRunOperation::Cancel,
)
.await?;
let stored = self
.store
.load_agent_admission(caller.tenant_id(), run_id)
.await?;
let lifecycle = stored.run().lifecycle();
if let Some(existing) = lifecycle.cancellation_request() {
if existing.failure().id() != ids.failure_id
|| existing.failure().caused_by_event_id() != Some(ids.event_id)
{
return Err(AgentServiceError::ConflictingCancellation);
}
let snapshot = self.runs.load(caller.tenant_id(), run_id).await?;
return Ok(AgentCancellationOutcome::Idempotent(snapshot));
}
if lifecycle.status().is_terminal() {
return Err(AgentServiceError::TerminalRun);
}
if lifecycle.status() == RunStatus::Pending {
return Err(AgentServiceError::InvalidRunState);
}
let requested_at = self.store.observe_database_clock().await?;
let failure = cancellation_failure(ids.failure_id).with_caused_by_event(ids.event_id);
let request = RunCancellationRequest::new(failure, requested_at)
.map_err(|_| AgentServiceError::ControlEventInvariant)?;
let data = BoundedJson::try_from_value(json!({
"operation": "agent_cancellation_requested",
"admission_digest": digest_hex(stored.admission().digest()),
"policy_digest": digest_hex(grant.policy_digest()),
"decision_digest": digest_hex(grant.decision_digest()),
"failure_id": ids.failure_id.to_string()
}))
.map_err(|_| AgentServiceError::ControlEventInvariant)?;
self.runs_registry()
.validate_bounded(&self.control_event_schema, &data)
.map_err(AgentServiceError::control_schema)?;
let payload = JournalPayload::new(
self.control_event_schema.clone(),
JournalEventKind::new("agent-cancellation-requested")
.map_err(|_| AgentServiceError::ControlEventInvariant)?,
data,
)
.map_err(|_| AgentServiceError::ControlEventInvariant)?;
let intent = JournalEventIntent::control_plane(
caller.tenant_id().clone(),
run_id,
ids.event_id,
payload,
)
.map_err(|_| AgentServiceError::ControlEventInvariant)?;
let head = stored
.run()
.journal_head()
.cloned()
.ok_or(AgentServiceError::InvalidRunState)?;
let append = JournalAppend::new(JournalExpectation::exact(head), intent)
.map_err(|_| AgentServiceError::ControlEventInvariant)?;
let transition = RunTransition::RequestCancellation { request };
let committed = if lifecycle.status() == RunStatus::Waiting {
matches!(
self.store
.append_control_plane_abandon_waits(append, lifecycle.revision(), transition,)
.await?,
WaitAbandonmentCommitOutcome::Committed { .. }
)
} else {
matches!(
self.store
.append_control_plane(
append,
RunProjection::transition(lifecycle.revision(), transition),
)
.await?,
AppendOutcome::Committed(_)
)
};
let snapshot = self.runs.load(caller.tenant_id(), run_id).await?;
Ok(if committed {
AgentCancellationOutcome::Committed(snapshot)
} else {
AgentCancellationOutcome::Idempotent(snapshot)
})
}
async fn authorize_run(
&self,
caller: &AgentServiceCaller,
target: AgentServiceRunTarget,
operation: AgentServiceRunOperation,
) -> Result<AgentServiceRunGrant, AgentServiceError> {
let grant = self
.authorizer
.authorize_run(AgentServiceRunAuthorization {
caller: caller.clone(),
target,
operation,
})
.await?;
if grant.principal() != caller.principal() {
return Err(AgentServiceError::AuthorizationPrincipalMismatch);
}
Ok(grant)
}
async fn recover_submission(
&self,
caller: &AgentServiceCaller,
key: &AgentSubmissionKey,
descriptor: &AgentDescriptor,
request: &AgentRequest,
graph: &GraphReference,
grant: &AgentServiceSubmissionGrant,
) -> Result<Option<AgentRunAdmissionOutcome>, AgentServiceError> {
let stored = match self
.store
.load_agent_submission(caller.tenant_id(), key)
.await
{
Ok(stored) => stored,
Err(StoreError::AgentSubmissionNotFound) => return Ok(None),
Err(error) => return Err(error.into()),
};
let intent = stored.admission().admission().intent();
if intent.provenance().tenant_id() != caller.tenant_id()
|| intent.descriptor() != descriptor
|| intent.request() != request
|| intent.budget_layers() != grant.budget_layers()
|| intent.graph() != graph
|| intent.authority() != grant.authority()
{
return Err(AgentServiceError::SubmissionConflict);
}
let snapshot = self.runs.load_by_key(caller.tenant_id(), key).await?;
Ok(Some(AgentRunAdmissionOutcome::Idempotent(snapshot)))
}
fn runs_registry(&self) -> &crate::JsonSchemaRegistry {
&self.schemas
}
}
impl fmt::Debug for AgentServiceV1 {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("AgentServiceV1")
.field("api_version", &AGENT_SERVICE_API_VERSION)
.field("deployments", &self.deployments)
.field("control_event_schema", &self.control_event_schema)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
#[non_exhaustive]
pub enum AgentServiceBuildError {
#[error(transparent)]
Runs(#[from] DurableAgentRunsBuildError),
#[error(transparent)]
ControlEventSchema(#[from] StandardAgentServiceControlSchemaError),
#[error("standard Agent service control event schema is unavailable")]
ControlEventSchemaUnavailable,
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum AgentServiceError {
#[error(transparent)]
Registry(#[from] AgentServiceRegistryError),
#[error(transparent)]
Authorization(#[from] AgentServiceAuthorizationError),
#[error("Agent service authorization principal does not match the caller")]
AuthorizationPrincipalMismatch,
#[error(transparent)]
Deployment(#[from] AgentServiceDeploymentError),
#[error(transparent)]
AdmissionRequest(#[from] DurableAgentAdmissionRequestError),
#[error(transparent)]
Runs(#[from] DurableAgentRunsError),
#[error(transparent)]
Store(#[from] StoreError),
#[error("Agent run has a different committed cancellation request")]
ConflictingCancellation,
#[error("Agent submission key was reused with different content")]
SubmissionConflict,
#[error("terminal Agent run cannot accept cancellation")]
TerminalRun,
#[error("Agent run is not in a cancellable durable state")]
InvalidRunState,
#[error("Agent service control event could not be constructed")]
ControlEventInvariant,
#[error("Agent service control event schema validation failed: {source}")]
ControlEventSchema {
#[source]
source: GraphSchemaValidationError,
},
}
impl AgentServiceError {
const fn control_schema(source: GraphSchemaValidationError) -> Self {
Self::ControlEventSchema { source }
}
}
fn cancellation_failure(id: FailureId) -> Failure {
Failure::new(
id,
FailureCategory::Cancelled,
FailureCode::new("agent.service.cancelled_by_caller")
.expect("static cancellation failure code is valid"),
FailureOrigin::new("stateknot.runtime.agent_service")
.expect("static cancellation failure origin is valid"),
FailureMessage::new("The Agent run was cancelled by an authorized caller.")
.expect("static cancellation message is valid"),
RetryAdvice::Never,
)
.expect("static cancellation failure semantics are coherent")
}
fn digest_hex(digest: Digest) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut value = String::with_capacity(Digest::SHA256_LEN * 2);
for byte in digest.as_bytes() {
value.push(char::from(HEX[usize::from(byte >> 4)]));
value.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
value
}