Struct EvalPack 

pub struct EvalPack {
    pub name: String,
    pub version: String,
    pub domain: String,
    pub path: PathBuf,
    pub hash: String,
    pub tasks: Vec<Task>,
}

Fields

name: String

version: String

domain: String

path: PathBuf

hash: String

tasks: Vec<Task>

Implementations

impl EvalPack

pub fn load(source: &str, base: Option<&Path>) -> Result<EvalPack, Error>

pub fn check(&self) -> Vec<PackIssue>

Check that the pack is ready to measure stacks with: required fields set, prompts and fixtures present, no unfilled PLACEHOLDERs, and tasks that can fail.

pub fn split(&self, search_frac: f64, seed: u64) -> (Vec<Task>, Vec<Task>)

Deterministic search/holdout split over task ids.

pub fn grade(&self, task: &Task, workspace: &Path) -> Vec<Grade>

Grade a task against its workspace. v0.1 uses artifact checks by default.

Trait Implementations

impl Clone for EvalPack

fn clone(&self) -> EvalPack

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for EvalPack

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.