eval-magic 0.5.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

//! Stage 4 — `grade`.
//!
//! Decomposed into focused units:
//!
//! - [`transcript_check`] — grade a `transcript_check` assertion (regex over the
//!   run's tool invocations).
//! - [`judge_tasks`] — emit LLM judge tasks + the skill-invocation meta-check
//!   (`emit_judge_tasks`), the default mode.
//! - [`finalize`] — fold judge responses + transcript checks into `grading.json`
//!   (`--finalize` mode).
//!
//! Both modes operate over a shared [`GradeContext`] assembled by the CLI.

pub mod finalize;
pub mod judge_tasks;
pub mod transcript_check;

use std::path::Path;

use crate::core::{ConditionsRecord, EvalsConfig};

pub use finalize::{FinalizeSummary, finalize};
pub use judge_tasks::{EmitSummary, check_skill_invoked_from_transcript, emit_judge_tasks};
pub use transcript_check::grade_transcript_check;

/// The resolved inputs both grade modes read: the iteration directory, the
/// conditions manifest, and the validated evals config (its `skill_name` is the
/// one used in meta-check rubrics).
pub struct GradeContext<'a> {
    pub iteration_dir: &'a Path,
    pub conditions: &'a ConditionsRecord,
    pub evals: &'a EvalsConfig,
}