aether-evals 0.2.8

Dockerized eval harness for Aether AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

use crate::{Task, containers::ContainerError};
use aether_core::events::AgentMessage;
use futures::Stream;
use thiserror::Error;

pub type AgentRunResult = Result<AgentMessage, RunError>;

/// Trait for an agent under evaluation.
///
/// Implementors run a [`Task`] in their configured execution environment and stream the observed
/// [`AgentMessage`]s.
///
/// # Example
///
/// ```ignore
/// let mut messages = agent.run(Task::new("write hello.txt"));
/// ```
pub trait Agent: Send + Sync {
    fn run(&self, task: Task) -> impl Stream<Item = AgentRunResult> + Send + '_;
}

/// Errors that can occur when running an agent
#[derive(Debug, Error)]
pub enum RunError {
    #[error("Agent execution failed: {0}")]
    ExecutionFailed(String),

    #[error("Agent configuration error: {0}")]
    ConfigurationError(String),

    #[error("Agent command exited without emitting a terminal AgentMessage: {stderr}")]
    CommandExitWithoutTerminal { stderr: String },

    #[error("Agent command emitted invalid AgentMessage JSON line: {line}")]
    AgentMessageJsonLine {
        line: String,
        #[source]
        source: serde_json::Error,
    },

    #[error(transparent)]
    Container(#[from] ContainerError),
}