aether_evals/evals/
run_eval.rs

1use super::report::EvalReport;
2use crate::agents::{Agent, AgentConfig, is_terminal};
3use crate::{EvalRunError, Workspace};
4use aether_core::events::AgentMessage;
5use tokio::sync::mpsc;
6
7#[tracing::instrument(skip(agent, prompt, workspace))]
8pub async fn run_eval<T: Agent>(
9    agent: &T,
10    prompt: impl Into<String>,
11    workspace: Workspace,
12) -> Result<EvalReport, EvalRunError> {
13    let prompt = prompt.into();
14    let messages = run_agent(agent, &workspace, &prompt).await?;
15    let (agent_diff, reference_diff) = workspace.capture_git_diffs();
16    Ok(EvalReport::new(prompt, workspace, messages, agent_diff, reference_diff))
17}
18
19async fn run_agent<T: Agent>(
20    agent: &T,
21    workspace: &Workspace,
22    task_prompt: &str,
23) -> Result<Vec<AgentMessage>, EvalRunError> {
24    let (tx, mut rx) = mpsc::channel(100);
25    let agent_task = agent.run(
26        AgentConfig { workspace_root: workspace.root_path(), relative_cwd: workspace.relative_cwd(), task_prompt },
27        tx,
28    );
29
30    let message_task = async {
31        let mut messages = Vec::new();
32        while let Some(message) = rx.recv().await {
33            let is_done = is_terminal(&message);
34            messages.push(message);
35            if is_done {
36                break;
37            }
38        }
39        messages
40    };
41
42    let (run_result, messages) = tokio::join!(agent_task, message_task);
43    run_result?;
44    Ok(messages)
45}
46
47#[cfg(test)]
48mod tests {
49    use super::*;
50    use crate::{FakeAgent, RunError, Workspace};
51    use std::sync::{Arc, Mutex};
52    use tokio::sync::mpsc::Sender;
53
54    #[tokio::test]
55    async fn run_eval_returns_report_with_prompt_workspace_and_messages() {
56        let report =
57            run_eval(&FakeAgent::with_tool_call("bash", "success"), "do the thing", Workspace::empty().unwrap())
58                .await
59                .unwrap();
60
61        assert_eq!(report.prompt(), "do the thing");
62        assert!(report.workspace().path().exists());
63        assert!(report.tool_called("bash"));
64        assert!(matches!(report.messages().last(), Some(AgentMessage::Done)));
65    }
66
67    #[tokio::test]
68    async fn run_eval_passes_raw_prompt_without_scaffolding() {
69        let agent = CapturingAgent::default();
70        run_eval(&agent, "do the thing", Workspace::empty().unwrap()).await.unwrap();
71
72        assert_eq!(agent.captured_task_prompt(), Some("do the thing".to_string()));
73    }
74
75    #[derive(Default)]
76    struct CapturingAgent {
77        task_prompt: Arc<Mutex<Option<String>>>,
78    }
79
80    impl CapturingAgent {
81        fn captured_task_prompt(&self) -> Option<String> {
82            self.task_prompt.lock().unwrap().clone()
83        }
84    }
85
86    impl Agent for CapturingAgent {
87        async fn run(&self, config: AgentConfig<'_>, tx: Sender<AgentMessage>) -> Result<(), RunError> {
88            *self.task_prompt.lock().unwrap() = Some(config.task_prompt.to_string());
89            tx.send(AgentMessage::Done).await.map_err(|e| RunError::ChannelSendFailed(e.to_string()))
90        }
91    }
92}
aether_evals/evals/run_eval.rs

aether_evals/evals/
run_eval.rs
