use super::{Agent, AgentRunTime};
use crate::{
error::AgentError,
shared::{Message, Role, ToolCall},
state::AgentState,
trajectory::TrajectoryEventKind,
};
use serde::{Deserialize, Serialize};
use std::fmt;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
const MAX_BACKOFF: Duration = Duration::from_secs(30);
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FailureKind {
MaxIter,
Provider,
ProviderPermanent,
ProviderRateLimited,
ProviderStream,
Truncated,
ContentFilter,
Middleware,
Tool,
SandboxOpen,
SandboxSave,
Storage,
}
impl FailureKind {
pub fn is_retryable(&self) -> bool {
matches!(
self,
FailureKind::Provider
| FailureKind::ProviderRateLimited
| FailureKind::ProviderStream
| FailureKind::Tool
| FailureKind::SandboxOpen
| FailureKind::SandboxSave
| FailureKind::Storage
)
}
pub fn as_str(&self) -> &'static str {
match self {
FailureKind::MaxIter => "max_iter",
FailureKind::Provider => "provider",
FailureKind::ProviderPermanent => "provider_permanent",
FailureKind::ProviderRateLimited => "provider_rate_limited",
FailureKind::ProviderStream => "provider_stream",
FailureKind::Truncated => "truncated",
FailureKind::ContentFilter => "content_filter",
FailureKind::Middleware => "middleware",
FailureKind::Tool => "tool",
FailureKind::SandboxOpen => "sandbox_open",
FailureKind::SandboxSave => "sandbox_save",
FailureKind::Storage => "storage",
}
}
}
impl fmt::Display for FailureKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(thiserror::Error, Debug, Serialize, Deserialize, Clone)]
#[error("{kind}: {message}")]
pub struct AgentFailure {
pub kind: FailureKind,
pub message: String,
pub tool_call: Option<ToolCall>,
}
impl AgentFailure {
pub(crate) fn new(kind: FailureKind, message: impl Into<String>) -> Self {
Self {
kind,
message: message.into(),
tool_call: None,
}
}
pub(crate) fn tool(tool_call: ToolCall, message: impl Into<String>) -> Self {
Self {
kind: FailureKind::Tool,
message: message.into(),
tool_call: Some(tool_call),
}
}
}
impl AgentRunTime {
pub(crate) fn fail_with(
&self,
ctx: AgentState,
kind: FailureKind,
message: impl Into<String>,
) -> Agent {
Agent::Fail(ctx, AgentFailure::new(kind, message))
}
pub(crate) fn fail_tool(
&self,
ctx: AgentState,
tool_call: ToolCall,
message: impl Into<String>,
) -> Agent {
Agent::Fail(ctx, AgentFailure::tool(tool_call, message))
}
pub(crate) async fn handle_fail(
&self,
mut ctx: AgentState,
mut failure: AgentFailure,
) -> Result<Agent, AgentError> {
loop {
self.emit(
&ctx.session_id,
TrajectoryEventKind::Failed {
kind: failure.kind.as_str().to_string(),
message: failure.message.clone(),
},
)
.await;
if !failure.kind.is_retryable() {
return Ok(Agent::Fail(ctx, failure));
}
ctx.consecutive_fail_count = ctx.consecutive_fail_count.saturating_add(1);
if ctx.consecutive_fail_count > self.max_consecutive_fail_count {
return Ok(Agent::Fail(ctx, failure));
}
if failure.tool_call.is_none() && failure.kind != FailureKind::Storage {
if let Err(e) = self
.transcript_storage
.append_message(
&ctx.session_id,
Message::text(
Role::System,
format!("{} failed: {}", failure.kind, failure.message),
),
)
.await
{
failure = AgentFailure::new(FailureKind::Storage, e.to_string());
continue;
}
}
self.retry_backoff(ctx.consecutive_fail_count).await;
return Ok(Agent::Ready(ctx));
}
}
async fn retry_backoff(&self, attempt: u32) {
let base = self.retry_backoff;
if base.is_zero() {
return;
}
let exponent = attempt.saturating_sub(1).min(6); let delay = base.saturating_mul(1u32 << exponent).min(MAX_BACKOFF);
let half = delay / 2;
let half_nanos = (half.as_nanos() as u64).max(1);
let jitter_source = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.subsec_nanos() as u64)
.unwrap_or(0);
let jitter = Duration::from_nanos(jitter_source % half_nanos);
tokio::time::sleep(half + jitter).await;
}
}