Struct EvalCase 

pub struct EvalCase {
    pub name: String,
    pub input: String,
    pub expected_tools: Option<Vec<ExpectedToolCall>>,
    pub output_contains: Vec<String>,
    pub output_not_contains: Vec<String>,
    pub reference_output: Option<String>,
    pub max_cost_usd: Option<f64>,
    pub max_latency_ms: Option<u64>,
    pub max_tool_calls: Option<usize>,
}

A single evaluation test case.

Fields

name: String

Human-readable name for the test case.

input: String

The task input to send to the agent.

expected_tools: Option<Vec<ExpectedToolCall>>

Expected tool calls in order (if Some). Empty vec means “expect no tools.”

output_contains: Vec<String>

Strings that should appear in the agent’s output.

output_not_contains: Vec<String>

Strings that must NOT appear in the agent’s output.

reference_output: Option<String>

Optional reference output for similarity scoring.

max_cost_usd: Option<f64>

Maximum acceptable cost in USD for this case.

max_latency_ms: Option<u64>

Maximum acceptable total LLM latency in milliseconds.

max_tool_calls: Option<usize>

Maximum acceptable number of tool calls.

Implementations

impl EvalCase

pub fn new(name: impl Into<String>, input: impl Into<String>) -> Self

Create a new eval case with a name and input task.

pub fn expect_tool(self, name: impl Into<String>) -> Self

Expect a specific tool to be called (order-independent).

pub fn expect_tool_at(self, name: impl Into<String>, position: usize) -> Self

Expect a tool at a specific position in the trajectory (0-indexed).

pub fn expect_no_tools(self) -> Self

Expect no tool calls at all.

pub fn expect_output_contains(self, text: impl Into<String>) -> Self

Expect the output to contain a string.

pub fn expect_output_not_contains(self, text: impl Into<String>) -> Self

Expect the output to NOT contain a string.

pub fn reference_output(self, text: impl Into<String>) -> Self

Set a reference output for similarity scoring.

pub fn expect_max_cost_usd(self, max: f64) -> Self

Set maximum acceptable cost in USD.

pub fn expect_max_latency_ms(self, max: u64) -> Self

Set maximum acceptable total LLM latency in milliseconds.

pub fn expect_max_tool_calls(self, max: usize) -> Self

Set maximum acceptable number of tool calls.

Trait Implementations

impl Clone for EvalCase

fn clone(&self) -> EvalCase

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for EvalCase

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for EvalCase

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for EvalCase

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.