tea-kernel 0.1.0

Headless model-tool agent state machine for tea-rs
Documentation
//! Provider retry with exponential backoff.

use crate::common;

use std::str::FromStr;
use std::sync::Mutex;
use std::time::Duration;

use tea_kernel::{
    AgentKernel, KernelClock, KernelDeadlineFuture, KernelError, KernelErrorCode,
    KernelEventFuture, KernelEventSink, KernelRunConfig, ModelRetryPolicy, RunState,
};
use tea_model::{ModelEvent, ModelFailure, ModelFailureCode, ModelResponseInfo, Utf8Delta};
use tea_policy::{
    ActorId, CodingWorkspacePolicy, ExecutionSurface, PolicyEngine, PolicyEnvironment,
    PolicyExecutionTarget,
};
use tea_protocol::{
    AgentEvent, EventEnvelope, ProtocolMetadata, ProtocolTimestamp, RetryClass, RunStatus,
    SessionId, SessionSequence,
};
use tea_testkit::ScriptedModelResponse;
use tea_tools::ToolRegistry;

use common::{EventCollector, TestIds, provider, session_id, store, timestamp};
use tea_session::SessionStore;

#[derive(Debug, Clone, Copy, Default)]
struct RealSleepClock;
impl KernelClock for RealSleepClock {
    fn now(&self) -> Result<ProtocolTimestamp, KernelError> {
        Ok(timestamp())
    }
    fn sleep_until(&self, deadline: ProtocolTimestamp) -> KernelDeadlineFuture<'_> {
        let now = self.now().unwrap();
        let duration = (deadline.as_utc() - now.as_utc())
            .to_std()
            .unwrap_or(Duration::ZERO);
        Box::pin(async move {
            tokio::time::sleep(duration).await;
        })
    }
}

#[derive(Debug)]
struct CancelOnRetry {
    events: Mutex<Vec<EventEnvelope>>,
    cancellation: tea_control::CancellationScope,
}

impl CancelOnRetry {
    fn new(cancellation: tea_control::CancellationScope) -> Self {
        Self {
            events: Mutex::new(Vec::new()),
            cancellation,
        }
    }

    fn events(&self) -> Vec<EventEnvelope> {
        self.events.lock().unwrap().clone()
    }
}

impl KernelEventSink for CancelOnRetry {
    fn last_sequence(&self, session_id: SessionId) -> Option<SessionSequence> {
        self.events
            .lock()
            .ok()?
            .iter()
            .rev()
            .find(|event| event.session_id() == session_id)
            .map(EventEnvelope::sequence)
    }

    fn emit(&self, event: EventEnvelope) -> KernelEventFuture<'_> {
        Box::pin(async move {
            let cancel = matches!(event.event(), AgentEvent::ModelRetryScheduled { .. });
            self.events.lock().unwrap().push(event);
            if cancel {
                self.cancellation.cancel();
            }
            Ok(())
        })
    }
}

fn config_with_retries(attempts: u32) -> KernelRunConfig {
    KernelRunConfig::new(
        ActorId::from_str("user:alice").unwrap(),
        PolicyEnvironment::new(
            ExecutionSurface::Test,
            PolicyExecutionTarget::Native,
            ProtocolMetadata::default(),
        ),
    )
    .with_retry_policy(
        ModelRetryPolicy::new(attempts, Duration::from_millis(1), Duration::from_millis(2))
            .unwrap(),
    )
}

#[tokio::test]
async fn retryable_failure_then_success_completes() {
    let provider = provider([
        ScriptedModelResponse::failure(ModelFailureCode::Unavailable, "transient"),
        ScriptedModelResponse::text(["recovered"]),
    ]);
    let store = store().await;
    let tools = ToolRegistry::new();
    let mut policy = PolicyEngine::new();
    policy.add_rule(CodingWorkspacePolicy).unwrap();
    let events = EventCollector::default();

    let outcome = AgentKernel::new(
        &provider,
        &tools,
        &policy,
        &store,
        &RealSleepClock,
        &TestIds::default(),
        &events,
    )
    .run(
        session_id(),
        &config_with_retries(3),
        tea_control::CancellationScope::new(),
    )
    .await
    .unwrap();
    assert_eq!(outcome.state(), RunState::Completed);
    let requests = provider.captured_requests().unwrap();
    assert_eq!(
        requests.len(),
        2,
        "provider should be called twice after one retry"
    );
    let retry_events = events
        .events()
        .into_iter()
        .filter_map(|event| match event.event() {
            AgentEvent::ModelRetryScheduled {
                message_id,
                attempt,
                max_retries,
                delay_ms,
            } => Some((
                "scheduled",
                *message_id,
                *attempt,
                *max_retries,
                Some(*delay_ms),
            )),
            AgentEvent::ModelRetryStarted {
                message_id,
                attempt,
                max_retries,
            } => Some(("started", *message_id, *attempt, *max_retries, None)),
            _ => None,
        })
        .collect::<Vec<_>>();
    assert_eq!(retry_events.len(), 2);
    assert_eq!(retry_events[0].0, "scheduled");
    assert_eq!(retry_events[0].2, 1);
    assert_eq!(retry_events[0].3, 2);
    assert_eq!(retry_events[0].4, Some(1));
    assert_eq!(retry_events[1].0, "started");
    assert_eq!(retry_events[1].1, retry_events[0].1);
}

#[tokio::test]
async fn retry_discards_partial_attempt_and_uses_server_delay() {
    let failure = ModelFailure::safe(
        ModelFailureCode::Unavailable,
        "HTTP 503: Service temporarily unavailable",
        RetryClass::AfterBackoff,
    )
    .unwrap()
    .with_retry_after(Duration::from_millis(2));
    let provider = provider([
        ScriptedModelResponse::events([
            ModelEvent::Started(ModelResponseInfo::new()),
            ModelEvent::TextDelta(Utf8Delta::new("stale").unwrap()),
            ModelEvent::Failed(failure),
        ]),
        ScriptedModelResponse::text(["recovered"]),
    ]);
    let store = store().await;
    let tools = ToolRegistry::new();
    let mut policy = PolicyEngine::new();
    policy.add_rule(CodingWorkspacePolicy).unwrap();
    let events = EventCollector::default();

    AgentKernel::new(
        &provider,
        &tools,
        &policy,
        &store,
        &RealSleepClock,
        &TestIds::default(),
        &events,
    )
    .run(
        session_id(),
        &config_with_retries(3),
        tea_control::CancellationScope::new(),
    )
    .await
    .unwrap();

    let events = events.events();
    let scheduled = events
        .iter()
        .find_map(|event| match event.event() {
            AgentEvent::ModelRetryScheduled {
                message_id,
                delay_ms,
                ..
            } => Some((*message_id, *delay_ms)),
            _ => None,
        })
        .unwrap();
    assert_eq!(scheduled.1, 2);
    let streamed_message_ids = events
        .iter()
        .filter_map(|event| match event.event() {
            AgentEvent::MessageDelta { message_id, .. } => Some(*message_id),
            _ => None,
        })
        .collect::<Vec<_>>();
    assert!(streamed_message_ids.iter().all(|id| *id == scheduled.0));

    let snapshot = store.load(session_id()).await.unwrap();
    let assistant = snapshot
        .state()
        .messages()
        .iter()
        .rev()
        .find(|message| matches!(message, tea_protocol::CanonicalMessage::Assistant { .. }))
        .unwrap();
    let serialized = serde_json::to_string(assistant).unwrap();
    assert!(serialized.contains("recovered"));
    assert!(!serialized.contains("stale"));
}

#[tokio::test]
async fn cancellation_during_retry_backoff_prevents_next_request() {
    let provider = provider([
        ScriptedModelResponse::failure(ModelFailureCode::Unavailable, "transient"),
        ScriptedModelResponse::text(["must not run"]),
    ]);
    let store = store().await;
    let tools = ToolRegistry::new();
    let mut policy = PolicyEngine::new();
    policy.add_rule(CodingWorkspacePolicy).unwrap();
    let cancellation = tea_control::CancellationScope::new();
    let events = CancelOnRetry::new(cancellation.clone());

    let error = AgentKernel::new(
        &provider,
        &tools,
        &policy,
        &store,
        &RealSleepClock,
        &TestIds::default(),
        &events,
    )
    .run(session_id(), &config_with_retries(3), cancellation)
    .await
    .unwrap_err();

    assert_eq!(error.code(), KernelErrorCode::Cancelled);
    assert_eq!(provider.captured_requests().unwrap().len(), 1);
    assert!(
        events
            .events()
            .iter()
            .any(|event| matches!(event.event(), AgentEvent::ModelRetryScheduled { .. }))
    );
    assert!(
        !events
            .events()
            .iter()
            .any(|event| matches!(event.event(), AgentEvent::ModelRetryStarted { .. }))
    );
}

#[tokio::test]
async fn never_failure_terminates_without_retry() {
    let provider = provider([
        ScriptedModelResponse::failure(ModelFailureCode::InvalidRequest, "bad request"),
        ScriptedModelResponse::text(["should not run"]),
    ]);
    let store = store().await;
    let tools = ToolRegistry::new();
    let mut policy = PolicyEngine::new();
    policy.add_rule(CodingWorkspacePolicy).unwrap();
    let events = EventCollector::default();

    let error = AgentKernel::new(
        &provider,
        &tools,
        &policy,
        &store,
        &RealSleepClock,
        &TestIds::default(),
        &events,
    )
    .run(
        session_id(),
        &config_with_retries(3),
        tea_control::CancellationScope::new(),
    )
    .await
    .unwrap_err();
    assert_eq!(error.code(), KernelErrorCode::ModelFailure);
    assert_eq!(error.message(), "model provider request failed");
    assert!(!error.is_safe_diagnostic());
    let requests = provider.captured_requests().unwrap();
    assert_eq!(requests.len(), 1, "Never failure must not retry");
}

#[tokio::test]
async fn retry_exhausted_after_policy_limit() {
    let provider = provider([
        ScriptedModelResponse::failure(ModelFailureCode::Unavailable, "transient"),
        ScriptedModelResponse::failure(ModelFailureCode::Unavailable, "transient"),
    ]);
    let store = store().await;
    let tools = ToolRegistry::new();
    let mut policy = PolicyEngine::new();
    policy.add_rule(CodingWorkspacePolicy).unwrap();
    let events = EventCollector::default();

    let error = AgentKernel::new(
        &provider,
        &tools,
        &policy,
        &store,
        &RealSleepClock,
        &TestIds::default(),
        &events,
    )
    .run(
        session_id(),
        &config_with_retries(2),
        tea_control::CancellationScope::new(),
    )
    .await
    .unwrap_err();
    assert_eq!(error.code(), KernelErrorCode::RetryExhausted);
    assert_eq!(error.message(), "model retry policy was exhausted");
    assert!(!error.is_safe_diagnostic());
    let requests = provider.captured_requests().unwrap();
    assert_eq!(
        requests.len(),
        2,
        "policy allows two attempts before exhaustion"
    );
    let snapshot = store.load(session_id()).await.unwrap();
    assert!(matches!(
        snapshot.records().last().unwrap().record(),
        tea_protocol::SessionRecord::RunInterrupted { .. }
    ));
    // Sanity: a finished run was never emitted for this session.
    let _ = RunStatus::Completed;
}