Struct EvalRunner 

pub struct EvalRunner { /* private fields */ }

Evaluation runner using Orchestrator directly

Example

let runner = EvalRunner::new(scenario, runtime.handle().clone())
    .with_runs(5)
    .with_seed(42)
    .with_task(SwarmTask::new("Find the auth handler"))
    .with_manager_factory(|| Box::new(MyManager::new()))
    .with_batch_invoker_factory(|| Box::new(MyInvoker::new()));

let report = runner.run()?;

Implementations

impl EvalRunner

pub fn new(scenario: EvalScenario, runtime: Handle) -> Self

pub fn with_verbose(self, verbose: bool) -> Self

Enable verbose output (print tick snapshots)

pub fn with_exploration(self, enable: bool) -> Self

Enable ExplorationSpace tracking

pub fn with_dependency_graph(self, graph: DependencyGraph) -> Self

Set dependency graph for action sequencing

The graph defines valid action transitions and terminal conditions. When set, actions will be filtered based on the graph structure.

pub fn with_action_events_path(self, path: impl Into<PathBuf>) -> Self

Set path to save action events (JSONL format)

When set, all action events will be saved to the specified path. Each run will append to a file with the run index suffix.

Example

runner.with_action_events_path("./logs/events.jsonl")
// Creates: ./logs/events_run0.jsonl, ./logs/events_run1.jsonl, ...

pub fn with_learning_store(self, path: impl AsRef<Path>) -> Self

Enable LearningStore for cross-session learning

When enabled, statistics will be saved after each run and loaded as prior for subsequent runs. This enables incremental learning across sessions.

Example

runner.with_learning_store("~/.swarm-engine/learning")

pub fn with_train_trigger(self, trigger: Arc<dyn TrainTrigger>) -> Self

Set TrainTrigger for Learning

Controls when offline learning is executed after eval runs. Default: Learn after every run (if learning_store is configured).

Example

use swarm_engine_core::learn::CountTrigger;

// Run learning after every 5 eval iterations
runner.with_train_trigger(Arc::new(CountTrigger::new(5)))

pub fn with_runs(self, runs: usize) -> Self

pub fn with_seed(self, seed: u64) -> Self

pub fn with_task(self, task: SwarmTask) -> Self

Set the task to execute

pub fn with_manager_factory<F>(self, factory: F) -> Self

where F: Fn() -> Box<dyn ManagerAgent> + Send + Sync + 'static,

Set manager factory (creates new Manager for each run)

pub fn with_batch_invoker_factory<F>(self, factory: F) -> Self

where F: Fn() -> Box<dyn BatchInvoker> + Send + Sync + 'static,

Set batch invoker factory (creates new BatchInvoker for each run)

pub fn with_extensions_factory<F>(self, factory: F) -> Self

where F: Fn() -> Extensions + Send + Sync + 'static,

Set extensions factory (creates new Extensions for each run)

pub fn with_operator_provider_factory<F>(self, factory: F) -> Self

where F: Fn() -> Box<dyn OperatorProvider<NodeRules>> + Send + Sync + 'static,

Set OperatorProvider factory (creates new provider for each run)

Use this to configure the Selection strategy for exploration. Default is AdaptiveProvider if not specified.

Example

use swarm_engine_core::exploration::{HybridLlmProvider, ReviewPolicy};
use swarm_engine_llm::LlmStrategyAdvisor;

runner.with_operator_provider_factory(|| {
    let advisor = LlmStrategyAdvisor::new(decider.clone(), handle.clone());
    let policy = ReviewPolicy::default();
    Box::new(HybridLlmProvider::new(advisor, policy))
})

pub fn run(&self) -> Result<EvalReport>