scouter_evaluate/tasks/
agent.rs

1use crate::evaluate::agent::AgentContextBuilder;
2use crate::tasks::evaluator::AssertionEvaluator;
3use crate::{error::EvaluationError, tasks::traits::EvaluationTask};
4use pyo3::prelude::*;
5use pyo3::pyfunction;
6use pythonize::depythonize;
7use scouter_types::genai::{AgentAssertionTask, AssertionResult, AssertionResults};
8use serde_json::Value;
9use std::collections::HashMap;
10use tracing::error;
11
12impl EvaluationTask for AgentAssertionTask {
13    fn execute(&self, context: &Value) -> Result<AssertionResult, EvaluationError> {
14        AssertionEvaluator::evaluate_assertion(context, self)
15    }
16}
17
18#[pyfunction]
19#[pyo3(signature = (tasks, context))]
20pub fn execute_agent_assertion_tasks(
21    tasks: Vec<AgentAssertionTask>,
22    context: &Bound<'_, PyAny>,
23) -> Result<AssertionResults, EvaluationError> {
24    let context: serde_json::Value = depythonize(context).map_err(|e| {
25        EvaluationError::GenAIEvaluatorError(format!("Failed to deserialize context: {}", e))
26    })?;
27    let results: HashMap<String, AssertionResult> = tasks
28        .iter()
29        .map(|task| {
30            let context_builder =
31                AgentContextBuilder::from_context(&context, task.provider.as_ref())?;
32            let resolved = context_builder.build_context(&task.assertion)?;
33            task.execute(&resolved)
34                .map(|result| (task.id.clone(), result))
35        })
36        .collect::<Result<HashMap<String, AssertionResult>, EvaluationError>>()?;
37
38    Ok(AssertionResults { results })
39}
40
41pub(crate) fn execute_agent_assertions(
42    context: &AgentContextBuilder,
43    tasks: &[AgentAssertionTask],
44) -> Result<AssertionResults, EvaluationError> {
45    if tasks.is_empty() {
46        return Ok(AssertionResults {
47            results: HashMap::new(),
48        });
49    }
50
51    let results: HashMap<String, AssertionResult> = tasks
52        .iter()
53        .map(|task| {
54            let resolved = context.build_context(&task.assertion)?;
55            task.execute(&resolved)
56                .map(|result| (task.id.clone(), result))
57        })
58        .collect::<Result<HashMap<String, AssertionResult>, EvaluationError>>()
59        .inspect_err(|e| {
60            error!("Error executing agent assertions: {:?}", e);
61        })?;
62
63    Ok(AssertionResults { results })
64}
scouter_evaluate/tasks/agent.rs

scouter_evaluate/tasks/
agent.rs
