Struct EvalScenario 

pub struct EvalScenario {
    pub id: String,
    pub initial_query: String,
    pub predefined_turns: Vec<String>,
    pub simulated_user_persona: Option<String>,
    pub termination_signal: Option<String>,
    pub max_turns: usize,
    pub expected_outcome: Option<String>,
    pub tasks: AssertionTasks,
    pub metadata: Option<HashMap<String, Value>>,
}

A single test case in an offline agent evaluation run.

Scenarios drive EvalScenarios (the orchestrator). At minimum, supply an initial_query. Everything else is optional.

Task attachment

Use tasks to attach scenario-level evaluation tasks (any of the four task types: AssertionTask, LLMJudgeTask, AgentAssertionTask, TraceAssertionTask). These are evaluated against the agent’s final response for this specific scenario. Sub-agent evaluation happens holistically across all scenarios via the profiles already registered on ScouterQueue.

Multi-turn scenarios

Populate predefined_turns with follow-up queries (executed in order after initial_query). Leave empty for single-turn evaluation.

ReAct / simulated-user scenarios

simulated_user_persona and termination_signal are placeholder fields for future ReAct support. Setting them has no effect in the current implementation.

Fields

id: String

initial_query: String

predefined_turns: Vec<String>

simulated_user_persona: Option<String>

termination_signal: Option<String>

max_turns: usize

expected_outcome: Option<String>

tasks: AssertionTasks

metadata: Option<HashMap<String, Value>>

Implementations

impl EvalScenario

pub fn new( initial_query: String, tasks: Option<&Bound<'_, PyList>>, id: Option<String>, expected_outcome: Option<String>, predefined_turns: Option<Vec<String>>, simulated_user_persona: Option<String>, termination_signal: Option<String>, max_turns: usize, metadata: Option<&Bound<'_, PyDict>>, ) -> Result<Self, TypeError>

pub fn __str__(&self) -> String

pub fn model_dump_json(&self) -> String

pub fn model_dump(&self, py: Python<'_>) -> Result<Py<PyDict>, TypeError>

pub fn assertion_tasks(&self) -> Vec<AssertionTask>

pub fn llm_judge_tasks(&self) -> Vec<LLMJudgeTask>

pub fn trace_assertion_tasks(&self) -> Vec<TraceAssertionTask>

pub fn agent_assertion_tasks(&self) -> Vec<AgentAssertionTask>

pub fn has_tasks(&self) -> bool

pub fn is_multi_turn(&self) -> bool

Returns true when predefined_turns is non-empty (scripted multi-turn scenario).

pub fn is_reactive(&self) -> bool

Returns true when a simulated_user_persona is set (ReAct/reactive scenario).

Note: ReAct execution is deferred — this field is a placeholder until potato_head supports simulated user LLMs.

Trait Implementations

impl Clone for EvalScenario

fn clone(&self) -> EvalScenario

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for EvalScenario

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for EvalScenario

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl<'py> IntoPyObject<'py> for EvalScenario

type Target = EvalScenario

The Python output type

type Output = Bound<'py, <EvalScenario as IntoPyObject<'py>>::Target>

The smart pointer type to use.

type Error = PyErr

The type returned in the event of a conversion error.

fn into_pyobject( self, py: Python<'py>, ) -> Result<<Self as IntoPyObject<'_>>::Output, <Self as IntoPyObject<'_>>::Error>

Performs the conversion.

impl PartialEq for EvalScenario

fn eq(&self, other: &EvalScenario) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl PyClass for EvalScenario

type Frozen = False

Whether the pyclass is frozen.

impl PyClassImpl for EvalScenario

const IS_BASETYPE: bool = false

#[pyclass(subclass)]

const IS_SUBCLASS: bool = false

#[pyclass(extends=…)]

const IS_MAPPING: bool = false

#[pyclass(mapping)]

const IS_SEQUENCE: bool = false

#[pyclass(sequence)]

const IS_IMMUTABLE_TYPE: bool = false

#[pyclass(immutable_type)]

const RAW_DOC: &'static CStr = /// A single test case in an offline agent evaluation run. /// /// Scenarios drive `EvalScenarios` (the orchestrator). At minimum, supply an /// `initial_query`. Everything else is optional. /// /// ## Task attachment /// Use `tasks` to attach scenario-level evaluation tasks (any of the four task /// types: `AssertionTask`, `LLMJudgeTask`, `AgentAssertionTask`, /// `TraceAssertionTask`). These are evaluated against the agent's **final /// response** for this specific scenario. Sub-agent evaluation happens /// holistically across all scenarios via the profiles already registered on /// `ScouterQueue`. /// /// ## Multi-turn scenarios /// Populate `predefined_turns` with follow-up queries (executed in order after /// `initial_query`). Leave empty for single-turn evaluation. /// /// ## ReAct / simulated-user scenarios /// `simulated_user_persona` and `termination_signal` are placeholder fields /// for future ReAct support. Setting them has no effect in the current /// implementation.

Docstring for the class provided on the struct or enum.

const DOC: &'static CStr

Fully rendered class doc, including the text_signature if a constructor is defined.

type BaseType = PyAny

Base class

type ThreadChecker = SendablePyClass<EvalScenario>

This handles following two situations:

type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::MutableChild

Immutable or mutable

type Dict = PyClassDummySlot

Specify this class has #[pyclass(dict)] or not.

type WeakRef = PyClassDummySlot

Specify this class has #[pyclass(weakref)] or not.

type BaseNativeType = PyAny

The closest native ancestor. This is PyAny by default, and when you declare #[pyclass(extends=PyDict)], it’s PyDict.

fn items_iter() -> PyClassItemsIter

fn lazy_type_object() -> &'static LazyTypeObject<Self>

fn dict_offset() -> Option<isize>

fn weaklist_offset() -> Option<isize>

impl PyClassNewTextSignature for EvalScenario

const TEXT_SIGNATURE: &'static str = "(initial_query, tasks=None, id=None, expected_outcome=None, predefined_turns=None, simulated_user_persona=None, termination_signal=None, max_turns=10, metadata=None)"

impl PyMethods<EvalScenario> for PyClassImplCollector<EvalScenario>

fn py_methods(self) -> &'static PyClassItems

impl PyTypeInfo for EvalScenario

const NAME: &'static str = "EvalScenario"

Class name.

const MODULE: Option<&'static str> = ::core::option::Option::None

Module name, if any.

fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject

Returns the PyTypeObject instance for this type.

fn type_object(py: Python<'_>) -> Bound<'_, PyType>

Returns the safe abstraction over the type object.

fn is_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type or a subclass of this type.

fn is_exact_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type.

impl Serialize for EvalScenario

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl DerefToPyAny for EvalScenario

impl ExtractPyClassWithClone for EvalScenario

impl StructuralPartialEq for EvalScenario