Struct EvalRunner 

pub struct EvalRunner { /* private fields */ }

Evaluation runner using Orchestrator directly

Example

let runner = EvalRunner::new(scenario, runtime.handle().clone())
    .with_runs(5)
    .with_seed(42)
    .with_task(SwarmTask::new("Find the auth handler"))
    .with_manager_factory(|| Box::new(MyManager::new()))
    .with_batch_invoker_factory(|| Box::new(MyInvoker::new()));

let report = runner.run()?;

Implementations

impl EvalRunner

pub fn new(scenario: EvalScenario, runtime: Handle) -> Self

pub fn with_verbose(self, verbose: bool) -> Self

Enable verbose output (print tick snapshots)

pub fn with_exploration(self, enable: bool) -> Self

Enable ExplorationSpace tracking

pub fn with_dependency_graph(self, graph: DependencyGraph) -> Self

Set dependency graph for action sequencing

The graph defines valid action transitions and terminal conditions. When set, actions will be filtered based on the graph structure.

pub fn with_learning_store(self, path: impl AsRef<Path>) -> Self

Enable LearningStore for cross-session learning

When enabled, statistics will be saved after each run and loaded as prior for subsequent runs. This enables incremental learning across sessions.

Example

runner.with_learning_store("~/.swarm-engine/learning")

pub fn with_train_trigger(self, trigger: Arc<dyn TrainTrigger>) -> Self

Set TrainTrigger for Learning

Controls when offline learning is executed after eval runs. Default: Learn after every run (if learning_store is configured).

Example

use swarm_engine_core::learn::CountTrigger;

// Run learning after every 5 eval iterations
runner.with_train_trigger(Arc::new(CountTrigger::new(5)))

pub fn skip_learned_action_order(self, skip: bool) -> Self

Skip learned action order from offline model

When enabled, the learned dependency graph (action_order) from the offline model will not be applied. This is useful for testing without learned priors.

pub fn with_trace_subscriber(self, subscriber: Arc<dyn TraceSubscriber>) -> Self

Set trace subscriber for ActionEvent output

The subscriber will receive all ActionEvents during evaluation. Use InMemoryTraceSubscriber to collect events and dump them after evaluation, or JsonlTraceSubscriber for real-time output.

Example

use std::sync::Arc;
use swarm_engine_core::events::InMemoryTraceSubscriber;

let trace = Arc::new(InMemoryTraceSubscriber::new());
runner.with_trace_subscriber(trace.clone());
// ... run evaluation ...
trace.dump_to_file("trace.jsonl")?;

pub fn with_scenario_profile(self, profile: ScenarioProfile) -> Self

Apply a ScenarioProfile to use learned exploration parameters and strategies

This converts the profile to an OfflineModel and applies it during execution. When set, the profile’s learned components (exploration params, strategy config, action order) will be used instead of defaults.

Example

let store = ProfileStore::new("~/.swarm-engine/profiles")?;
let profile = store.load("troubleshooting")?;
runner.with_scenario_profile(profile);

pub fn with_runs(self, runs: usize) -> Self

pub fn with_seed(self, seed: u64) -> Self

pub fn with_task(self, task: SwarmTask) -> Self

Set the task to execute

pub fn with_manager_factory<F>(self, factory: F) -> Self

where F: Fn() -> Box<dyn ManagerAgent> + Send + Sync + 'static,

Set manager factory (creates new Manager for each run)

pub fn with_batch_invoker_factory<F>(self, factory: F) -> Self

where F: Fn() -> Box<dyn BatchInvoker> + Send + Sync + 'static,

Set batch invoker factory (creates new BatchInvoker for each run)

pub fn with_extensions_factory<F>(self, factory: F) -> Self

where F: Fn() -> Extensions + Send + Sync + 'static,

Set extensions factory (creates new Extensions for each run)

pub fn with_operator_provider_factory<F>(self, factory: F) -> Self

where F: Fn() -> Box<dyn OperatorProvider<NodeRules>> + Send + Sync + 'static,

Set OperatorProvider factory (creates new provider for each run)

Use this to configure the Selection strategy for exploration. Default is AdaptiveProvider if not specified.

Example

use swarm_engine_core::exploration::{HybridLlmProvider, ReviewPolicy};
use swarm_engine_llm::LlmStrategyAdvisor;

runner.with_operator_provider_factory(|| {
    let advisor = LlmStrategyAdvisor::new(decider.clone(), handle.clone());
    let policy = ReviewPolicy::default();
    Box::new(HybridLlmProvider::new(advisor, policy))
})

pub fn run(&self) -> Result<EvalReport>