eval-magic 0.5.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

//! The error type shared by every validator in this module.

/// A validation failure. Each variant carries the `path` of the artifact under
/// inspection so messages are actionable.
///
/// (The field is named `path` rather than `source` because `thiserror` reserves
/// a field named `source` for the underlying `std::error::Error` cause.)
#[derive(Debug, thiserror::Error)]
pub enum ValidationError {
    /// `data` failed structural validation against the named schema. `details`
    /// is one `  <instance-path> <message>` line per failure.
    #[error("{path}: does not match the {schema} schema:\n{details}")]
    SchemaMismatch {
        path: String,
        schema: String,
        details: String,
    },

    /// Two evals share an `id` — a uniqueness constraint JSON Schema (draft-07)
    /// can't express, so it is checked by hand after structural validation.
    #[error("{path}: evals[{index}].id duplicate: {id}")]
    DuplicateId {
        path: String,
        index: usize,
        id: String,
    },

    /// The data matched the schema but could not be deserialized into the
    /// requested type — a contract drift between schema and Rust type.
    #[error("{path}: invalid value after schema validation: {message}")]
    Deserialize { path: String, message: String },
}