aether-evals 0.3.19

Dockerized eval harness for Aether AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

use aether_core::events::{AgentEvent, TurnEvent};
use futures::{StreamExt, pin_mut};

use aether_evals::{
    Agent, Container, ContainerError, DockerAgent, Image, Task, Transcript, TranscriptError, Workspace, WorkspaceError,
};

#[derive(Debug, thiserror::Error)]
enum DockerAgentTestError {
    #[error(transparent)]
    Workspace(#[from] WorkspaceError),
    #[error(transparent)]
    Container(#[from] ContainerError),
    #[error(transparent)]
    Transcript(#[from] TranscriptError),
}

#[tokio::test]
async fn docker_agent_direct_agent_event_eval() -> Result<(), DockerAgentTestError> {
    let workspace = Workspace::empty()?;
    let container = Container::builder(Image::parse("alpine:3").unwrap()).start(&workspace).await?;
    let agent = DockerAgent::new(
        container,
        vec![
            "/bin/sh".to_string(),
            "-c".to_string(),
            r#"printf '%s\n' '{"category":"message","event":{"type":"text","message_id":"m1","chunk":"ok","is_complete":true}}' '{"category":"turn","event":{"type":"ended","outcome":{"status":"completed"}}}'"#
                .to_string(),
        ],
    );

    let trace = Transcript::from_stream(agent.run(Task::new("Reply with one short sentence."))).await?;
    assert!(trace.events().iter().any(|message| matches!(message, AgentEvent::Turn(TurnEvent::Ended { .. }))));
    Ok(())
}

#[tokio::test]
async fn container_exec_shell_returns_non_zero_exit_codes() -> Result<(), DockerAgentTestError> {
    let workspace = Workspace::empty()?;
    let container = Container::builder(Image::parse("alpine:3").unwrap()).start(&workspace).await?;
    let output = container.exec_shell("echo nope >&2; exit 7").await?;

    assert_eq!(output.exit_code, 7);
    assert!(output.stderr.contains("nope"));
    Ok(())
}

#[tokio::test]
async fn docker_agent_waits_for_cleanup() -> Result<(), DockerAgentTestError> {
    let workspace = Workspace::empty()?;
    let container = Container::builder(Image::parse("alpine:3").unwrap()).start(&workspace).await?;
    let agent = DockerAgent::new(container.clone(), vec![
        "/bin/sh".to_string(), "-c".to_string(),
        r#"mkfifo /tmp/release; printf '%s\n' '{"category":"turn","event":{"type":"ended","outcome":{"status":"completed"}}}'; read release < /tmp/release; echo cleaned-up > /workspace/cleanup; printf 'trailing output\n'"#.to_string(),
    ]);
    let stream = agent.run(Task::new("cleanup"));
    pin_mut!(stream);
    assert!(matches!(stream.next().await, Some(Ok(AgentEvent::Turn(TurnEvent::Ended { .. })))));
    container.exec_shell("echo release > /tmp/release").await?;
    assert!(stream.next().await.is_none());
    assert_eq!(container.exec_shell("cat /workspace/cleanup").await?.stdout, "cleaned-up\n");
    Ok(())
}