use std::sync::Arc;
use chrono::{DateTime, Duration, Utc};
use rusty_agent_runtime::agents::{
AgentId, EscalationNotice, SupervisionAttempt, SupervisionPolicy, SupervisionTrigger,
ESCALATION_MESSAGE_KIND,
};
use rusty_agent_runtime::durable::ErrorClass;
use rusty_agent_runtime::journal::{Clock, EventDraft, Journal};
use rusty_agent_runtime::record::{Effect, RunEvent, RunEventKind};
use serde_json::{json, Value};
use crate::agents::AgentRecord;
use crate::auth::TenantContext;
use crate::server_store::{ServerStore, StoreResult};
use crate::tasks::{self, TaskRecord, TaskStatus};
#[derive(Debug, Clone)]
pub(crate) enum Trigger {
TurnFailed {
error_class: ErrorClass,
message: String,
task_id: String,
},
DeadlineBreached,
ManualRestart { reason: String },
}
impl Trigger {
fn kind(&self) -> SupervisionTrigger {
match self {
Self::TurnFailed { .. } => SupervisionTrigger::TurnFailed,
Self::DeadlineBreached => SupervisionTrigger::DeadlineBreached,
Self::ManualRestart { .. } => SupervisionTrigger::ManualRestart,
}
}
fn error_class(&self) -> Option<ErrorClass> {
match self {
Self::TurnFailed { error_class, .. } => Some(*error_class),
Self::DeadlineBreached | Self::ManualRestart { .. } => None,
}
}
fn message(&self) -> String {
match self {
Self::TurnFailed { message, .. } => message.clone(),
Self::DeadlineBreached => "agent deadline breached (cancellation by clock)".to_string(),
Self::ManualRestart { reason } => reason.clone(),
}
}
fn task_id(&self) -> Option<String> {
match self {
Self::TurnFailed { task_id, .. } => Some(task_id.clone()),
Self::DeadlineBreached | Self::ManualRestart { .. } => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Decision {
Unsupervised,
Suppressed,
Restart {
ordinal: u32,
},
Escalate,
}
pub(crate) fn decide(
policy: Option<&SupervisionPolicy>,
attempts: &[SupervisionAttempt],
escalated: bool,
trigger: &Trigger,
now: DateTime<Utc>,
) -> Decision {
if matches!(trigger, Trigger::ManualRestart { .. }) {
return Decision::Restart {
ordinal: attempts.len() as u32 + 1,
};
}
let Some(policy) = policy else {
return Decision::Unsupervised;
};
if escalated {
return Decision::Suppressed;
}
if !policy.allows_restart_after(trigger.error_class()) {
return Decision::Escalate;
}
let window_start = now - Duration::milliseconds(policy.period_ms.min(i64::MAX as u64) as i64);
let recent_failures = attempts
.iter()
.filter(|a| a.at >= window_start && !matches!(a.trigger, SupervisionTrigger::ManualRestart))
.count() as u32;
if recent_failures < policy.intensity {
Decision::Restart {
ordinal: attempts.len() as u32 + 1,
}
} else {
Decision::Escalate
}
}
pub(crate) fn supervision_journal_run_id(tenant: &str, agent_external: &str) -> String {
format!("agent-supervision:{tenant}:{agent_external}")
}
async fn journal_event(
store: &Arc<dyn ServerStore>,
tenant: &TenantContext,
agent_external: &str,
kind: RunEventKind,
output: Value,
) -> StoreResult<String> {
let run_id = supervision_journal_run_id(tenant.tenant(), agent_external);
let journal = match store.get_journal(&run_id).await? {
Some(snapshot) => Journal::from_snapshot(snapshot, Clock::System).map_err(|e| {
format!(
"supervision journal for agent `{agent_external}` failed its integrity check: {e}"
)
})?,
None => Journal::new(
run_id,
AgentId::new(agent_external).thread_id(),
Clock::System,
),
};
let event_id = journal.record(
EventDraft::new(kind, Effect::Pure)
.output(output)
.status(rusty_agent_runtime::record::EventStatus::Ok),
);
store.put_journal(&journal.snapshot()).await?;
Ok(event_id)
}
fn supervision_payload(
policy: Option<&SupervisionPolicy>,
trigger: &Trigger,
decision: &str,
ordinal: u32,
attempts: &[SupervisionAttempt],
) -> Value {
json!({
"decision": decision,
"policy": policy,
"trigger": trigger.kind(),
"error_class": trigger.error_class(),
"message": trigger.message(),
"task_id": trigger.task_id(),
"restart_ordinal": ordinal,
"attempts": attempts,
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum EscalationDelivery {
Mailbox {
task_id: String,
deduplicated: bool,
},
DeadLetter {
task_id: String,
},
}
async fn deliver_escalation(
store: &Arc<dyn ServerStore>,
tenant: &TenantContext,
agent_external: &str,
policy: &SupervisionPolicy,
attempts: Vec<SupervisionAttempt>,
now: DateTime<Utc>,
) -> StoreResult<EscalationDelivery> {
let notice = EscalationNotice {
agent_id: agent_external.to_string(),
policy: policy.clone(),
attempts,
escalated_at: now,
};
let payload = serde_json::to_value(¬ice)
.map_err(|e| format!("escalation notice serialization failed: {e}"))?;
let ordinal = notice.attempts.last().map(|a| a.ordinal).unwrap_or(0);
let idempotency_key = format!("escalation:{agent_external}:{ordinal}");
if let Some(supervisor) = &policy.supervisor {
if let Some(record) = store.get_agent(&tenant.scope(supervisor)).await? {
if record
.manifest
.accepts_kind(ESCALATION_MESSAGE_KIND)
.is_some()
{
let task = TaskRecord::new(
tasks::NewTask {
task_id: uuid::Uuid::new_v4().to_string(),
tenant: tenant.tenant().to_string(),
kind: ESCALATION_MESSAGE_KIND.to_string(),
payload,
pool: tasks::DEFAULT_POOL.to_string(),
recipient: Some(AgentId::new(supervisor).mailbox_recipient()),
max_attempts: tasks::DEFAULT_MAX_ATTEMPTS,
idempotency_key: Some(idempotency_key),
effect: None,
run_id: None,
thread_id: None,
deadline: None,
worker_version: None,
parent: None,
},
now,
);
let (task, deduplicated) = store.enqueue_task(&task).await?;
return Ok(EscalationDelivery::Mailbox {
task_id: task.task_id,
deduplicated,
});
}
}
}
let mut task = TaskRecord::new(
tasks::NewTask {
task_id: uuid::Uuid::new_v4().to_string(),
tenant: tenant.tenant().to_string(),
kind: ESCALATION_MESSAGE_KIND.to_string(),
payload,
pool: tasks::DEFAULT_POOL.to_string(),
recipient: None,
max_attempts: tasks::DEFAULT_MAX_ATTEMPTS,
idempotency_key: Some(idempotency_key),
effect: None,
run_id: None,
thread_id: None,
deadline: None,
worker_version: None,
parent: None,
},
now,
);
task.status = TaskStatus::Dead;
task.last_error = Some(
"escalation dead-lettered: no supervisor declared, or none accepting `escalated`"
.to_string(),
);
let (task, _) = store.dead_letter_task(&task).await?;
Ok(EscalationDelivery::DeadLetter {
task_id: task.task_id,
})
}
#[derive(Debug, Clone)]
pub(crate) struct SupervisionOutcome {
pub decision: Decision,
pub event_id: Option<String>,
pub delivery: Option<EscalationDelivery>,
}
pub(crate) async fn supervise(
store: &Arc<dyn ServerStore>,
tenant: &TenantContext,
agent_external: &str,
mut agent: AgentRecord,
trigger: Trigger,
now: DateTime<Utc>,
) -> StoreResult<SupervisionOutcome> {
let policy = agent.manifest.supervision.clone();
let decision = decide(
policy.as_ref(),
&agent.supervision.attempts,
agent.supervision.escalated,
&trigger,
now,
);
match decision {
Decision::Unsupervised => Ok(SupervisionOutcome {
decision,
event_id: None,
delivery: None,
}),
Decision::Suppressed => {
agent.supervision.suppressed_failures += 1;
store.update_agent(&agent).await?;
Ok(SupervisionOutcome {
decision,
event_id: None,
delivery: None,
})
}
Decision::Restart { ordinal } => {
let manual = matches!(trigger, Trigger::ManualRestart { .. });
agent.supervision.attempts.push(SupervisionAttempt {
ordinal,
trigger: trigger.kind(),
error_class: trigger.error_class(),
message: trigger.message(),
task_id: trigger.task_id(),
at: now,
});
if manual {
agent.supervision.escalated = false;
agent.supervision.deadline_breached = false;
agent.supervision.suppressed_failures = 0;
}
store.update_agent(&agent).await?;
let event_id = journal_event(
store,
tenant,
agent_external,
RunEventKind::SupervisionEvent,
supervision_payload(
policy.as_ref(),
&trigger,
"restart",
ordinal,
&agent.supervision.attempts,
),
)
.await?;
Ok(SupervisionOutcome {
decision,
event_id: Some(event_id),
delivery: None,
})
}
Decision::Escalate => {
let policy = policy.expect("an escalation decision requires a declared policy");
let ordinal = agent.supervision.next_ordinal();
agent.supervision.attempts.push(SupervisionAttempt {
ordinal,
trigger: trigger.kind(),
error_class: trigger.error_class(),
message: trigger.message(),
task_id: trigger.task_id(),
at: now,
});
agent.supervision.escalated = true;
store.update_agent(&agent).await?;
let event_id = journal_event(
store,
tenant,
agent_external,
RunEventKind::SupervisionEvent,
supervision_payload(
Some(&policy),
&trigger,
"escalate",
ordinal,
&agent.supervision.attempts,
),
)
.await?;
let delivery = deliver_escalation(
store,
tenant,
agent_external,
&policy,
agent.supervision.attempts.clone(),
now,
)
.await?;
match &delivery {
EscalationDelivery::Mailbox {
task_id,
deduplicated,
} => tracing::info!(
agent = %agent_external,
%task_id,
%deduplicated,
"supervision escalated to the supervisor's mailbox"
),
EscalationDelivery::DeadLetter { task_id } => tracing::warn!(
agent = %agent_external,
%task_id,
"supervision escalation dead-lettered (root or undeliverable)"
),
}
Ok(SupervisionOutcome {
decision,
event_id: Some(event_id),
delivery: Some(delivery),
})
}
}
}
pub(crate) async fn on_deadline_breach(
store: &Arc<dyn ServerStore>,
tenant: &TenantContext,
agent_external: &str,
mut agent: AgentRecord,
now: DateTime<Utc>,
) -> StoreResult<(tasks::RunCancellation, SupervisionOutcome)> {
agent.supervision.deadline_breached = true;
store.update_agent(&agent).await?;
let recipient = AgentId::new(agent_external).mailbox_recipient();
let cancellation = store
.cancel_agent_tasks(tenant.tenant(), &recipient, now)
.await?;
let outcome = supervise(
store,
tenant,
agent_external,
agent,
Trigger::DeadlineBreached,
now,
)
.await?;
Ok((cancellation, outcome))
}
pub(crate) async fn journal_agent_exit(
store: &Arc<dyn ServerStore>,
tenant: &TenantContext,
agent_external: &str,
disposition: &str,
detail: Value,
) -> StoreResult<String> {
journal_event(
store,
tenant,
agent_external,
RunEventKind::AgentExit,
json!({
"disposition": disposition,
"detail": detail,
}),
)
.await
}
pub(crate) async fn supervision_events(
store: &Arc<dyn ServerStore>,
tenant: &TenantContext,
agent_external: &str,
) -> StoreResult<Vec<RunEvent>> {
let run_id = supervision_journal_run_id(tenant.tenant(), agent_external);
let Some(snapshot) = store.get_journal(&run_id).await? else {
return Ok(Vec::new());
};
let journal = Journal::from_snapshot(snapshot, Clock::System).map_err(|e| {
format!("supervision journal for agent `{agent_external}` failed its integrity check: {e}")
})?;
Ok(journal.events())
}
#[cfg(test)]
mod tests {
use super::*;
use rusty_agent_runtime::agents::RestartPolicy;
fn policy(restart: RestartPolicy, intensity: u32) -> SupervisionPolicy {
SupervisionPolicy::new(restart, intensity, 60_000)
}
fn failed(message: &str) -> Trigger {
Trigger::TurnFailed {
error_class: ErrorClass::Transient,
message: message.to_string(),
task_id: "task-1".to_string(),
}
}
fn attempt(ordinal: u32, at: DateTime<Utc>) -> SupervisionAttempt {
SupervisionAttempt {
ordinal,
trigger: SupervisionTrigger::TurnFailed,
error_class: Some(ErrorClass::Transient),
message: format!("failure {ordinal}"),
task_id: Some("task-1".to_string()),
at,
}
}
#[test]
fn unsupervised_agents_are_unmanaged() {
let now = Utc::now();
assert_eq!(
decide(None, &[], false, &failed("boom"), now),
Decision::Unsupervised
);
let cancelled = Trigger::TurnFailed {
error_class: ErrorClass::Cancelled,
message: "drained".to_string(),
task_id: "t".to_string(),
};
assert_eq!(
decide(None, &[], false, &cancelled, now),
Decision::Unsupervised
);
}
#[test]
fn permanent_restarts_until_the_intensity_is_exceeded_then_escalates() {
let now = Utc::now();
let p = policy(RestartPolicy::Permanent, 2);
assert_eq!(
decide(Some(&p), &[], false, &failed("1"), now),
Decision::Restart { ordinal: 1 }
);
let history: Vec<_> = (1..=2).map(|o| attempt(o, now)).collect();
assert_eq!(
decide(Some(&p), &history, false, &failed("3"), now),
Decision::Escalate
);
}
#[test]
fn attempts_outside_the_period_do_not_consume_the_budget() {
let now = Utc::now();
let p = SupervisionPolicy::new(RestartPolicy::Permanent, 1, 60_000);
let old = vec![attempt(1, now - Duration::minutes(10))];
assert_eq!(
decide(Some(&p), &old, false, &failed("recent"), now),
Decision::Restart { ordinal: 2 }
);
let edge = vec![attempt(1, now - Duration::seconds(59))];
assert_eq!(
decide(Some(&p), &edge, false, &failed("recent"), now),
Decision::Escalate
);
}
#[test]
fn transient_restarts_failures_but_escalates_cancellations() {
let now = Utc::now();
let p = policy(RestartPolicy::Transient, 3);
assert!(matches!(
decide(Some(&p), &[], false, &failed("boom"), now),
Decision::Restart { .. }
));
let cancelled = Trigger::TurnFailed {
error_class: ErrorClass::Cancelled,
message: "cancelled".to_string(),
task_id: "t".to_string(),
};
assert_eq!(
decide(Some(&p), &[], false, &cancelled, now),
Decision::Escalate
);
assert_eq!(
decide(Some(&p), &[], false, &Trigger::DeadlineBreached, now),
Decision::Escalate
);
}
#[test]
fn temporary_escalates_the_first_failure() {
let now = Utc::now();
let p = policy(RestartPolicy::Temporary, 5);
assert_eq!(
decide(Some(&p), &[], false, &failed("boom"), now),
Decision::Escalate
);
}
#[test]
fn the_escalation_latch_suppresses_further_action() {
let now = Utc::now();
let p = policy(RestartPolicy::Permanent, 5);
assert_eq!(
decide(Some(&p), &[], true, &failed("still broken"), now),
Decision::Suppressed
);
assert_eq!(
decide(Some(&p), &[], true, &Trigger::DeadlineBreached, now),
Decision::Suppressed
);
}
#[test]
fn manual_restart_always_restarts_and_needs_no_policy() {
let now = Utc::now();
let manual = Trigger::ManualRestart {
reason: "operator reset".to_string(),
};
assert_eq!(
decide(None, &[], true, &manual, now),
Decision::Restart { ordinal: 1 }
);
let p = policy(RestartPolicy::Temporary, 0);
assert_eq!(
decide(Some(&p), &[], true, &manual, now),
Decision::Restart { ordinal: 1 }
);
}
#[test]
fn manual_restarts_in_history_do_not_consume_the_budget() {
let now = Utc::now();
let p = SupervisionPolicy::new(RestartPolicy::Permanent, 1, 60_000);
let mut history = vec![attempt(1, now)];
history.push(SupervisionAttempt {
ordinal: 2,
trigger: SupervisionTrigger::ManualRestart,
error_class: None,
message: "operator reset".to_string(),
task_id: None,
at: now,
});
assert_eq!(
decide(Some(&p), &history, false, &failed("second crash"), now),
Decision::Escalate
);
let only_manual = vec![history[1].clone()];
assert_eq!(
decide(Some(&p), &only_manual, false, &failed("first crash"), now),
Decision::Restart { ordinal: 2 }
);
}
}