aether_evals/agents/
agent.rs

1use super::docker::DockerError;
2use aether_core::events::AgentMessage;
3use std::future::Future;
4use std::path::{Path, PathBuf};
5use thiserror::Error;
6use tokio::sync::mpsc::Sender;
7
8/// Configuration for running an agent on a specific task
9pub struct AgentConfig<'a> {
10    pub workspace_root: &'a Path,
11    pub relative_cwd: Option<&'a Path>,
12    pub task_prompt: &'a str,
13}
14
15impl AgentConfig<'_> {
16    /// The working directory the agent should operate in
17    pub fn workspace(&self) -> PathBuf {
18        match self.relative_cwd {
19            Some(relative) => self.workspace_root.join(relative),
20            None => self.workspace_root.to_path_buf(),
21        }
22    }
23}
24
25/// Wrap a task prompt with the standard eval prompt
26pub(crate) fn build_task_prompt(task_prompt: &str, workspace: &Path) -> String {
27    [
28        "Complete the following task:".to_string(),
29        format!("<task>{task_prompt}</task>"),
30        format!(
31            "CRITICAL INSTRUCTIONS: when working on this task, you MUST only operate within this directory: {}",
32            workspace.display()
33        ),
34    ]
35    .join("\n")
36}
37
38/// Trait for an agent under evaluation.
39///
40/// Implementors are responsible for:
41/// - Creating their own MCP connections (if needed)
42/// - Running the agent with the provided configuration
43/// - Sending `AgentMessage`s to the provided channel
44/// - Sending `AgentMessage::Done` when the agent finishes
45///
46/// # Example
47///
48/// ```ignore
49/// struct MyAgent;
50///
51/// impl Agent for MyAgent {
52///     async fn run(&self, config: AgentConfig<'_>, tx: Sender<AgentMessage>) -> Result<(), RunError> {
53///         tx.send(AgentMessage::text("msg_1", "Hello", true, "test")).await
54///             .map_err(|e| RunError::ChannelSendFailed(e.to_string()))?;
55///         tx.send(AgentMessage::Done).await
56///             .map_err(|e| RunError::ChannelSendFailed(e.to_string()))?;
57///         Ok(())
58///     }
59/// }
60/// ```
61pub trait Agent: Send + Sync {
62    fn run(
63        &self,
64        config: AgentConfig<'_>,
65        tx: Sender<AgentMessage>,
66    ) -> impl Future<Output = Result<(), RunError>> + Send;
67}
68
69/// Errors that can occur when running an agent
70#[derive(Debug, Error)]
71pub enum RunError {
72    #[error("Agent execution failed: {0}")]
73    ExecutionFailed(String),
74
75    #[error("Failed to send event: {0}")]
76    ChannelSendFailed(String),
77
78    #[error("Agent configuration error: {0}")]
79    ConfigurationError(String),
80
81    #[error(transparent)]
82    Docker(#[from] DockerError),
83}
84
85#[cfg(test)]
86mod tests {
87    use super::*;
88
89    #[test]
90    fn build_task_prompt_wraps_prompt_and_pins_workspace() {
91        let prompt = build_task_prompt("write hello.txt", Path::new("/tmp/work"));
92
93        assert!(prompt.contains("<task>write hello.txt</task>"));
94        assert!(prompt.contains("/tmp/work"));
95    }
96}
aether_evals/agents/agent.rs

aether_evals/agents/
agent.rs
