skilltest-core 0.3.0

Core library for skilltest: run AI skills on harness/model platforms and score transcripts with natural-language evals.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

//! Error type for the core library. The mapping from these errors to process
//! exit codes lives in the CLI (see `exit.rs` for the documented codes).

use std::path::PathBuf;

/// Result alias used throughout the core library.
pub type Result<T> = std::result::Result<T, Error>;

/// Everything that can go wrong while loading configuration, parsing skill or
/// test-case definitions, talking to a provider, or running evals.
///
/// Variants are grouped so the CLI can map them onto stable exit codes: input
/// problems (`Config`, `Yaml`, `Skill`, `Validation`) are the user's to fix;
/// `Provider` problems are environmental.
#[derive(Debug, thiserror::Error)]
pub enum Error {
    /// A file the user pointed us at could not be read.
    #[error("could not read {path}: {source}")]
    Io {
        path: PathBuf,
        #[source]
        source: std::io::Error,
    },

    /// A YAML document (config or test case) failed to parse.
    #[error("invalid YAML in {path}: {source}")]
    Yaml {
        path: PathBuf,
        #[source]
        source: serde_yaml::Error,
    },

    /// A test case or config was syntactically valid YAML but semantically
    /// wrong (e.g. a numeric eval with `min > max`).
    #[error("invalid test definition: {0}")]
    Invalid(String),

    /// The provider command could not be spawned or did not behave. `kind`, when
    /// set, classifies the failure (e.g. `"auth"`, `"rate_limit"`,
    /// `"model_not_found"`, `"quota"`) so the CLI can distinguish a broken
    /// environment from a broken skill.
    #[error("provider error ({context}): {message}")]
    Provider {
        context: String,
        message: String,
        kind: Option<String>,
    },

    /// A skill definition failed validation. Carries the human-readable
    /// findings so the CLI can print them.
    #[error("skill validation failed with {} finding(s)", .0.len())]
    Validation(Vec<String>),
}

impl Error {
    /// Construct a [`Error::Provider`] with no classification.
    pub fn provider(context: impl Into<String>, message: impl std::fmt::Display) -> Self {
        Error::Provider {
            context: context.into(),
            message: message.to_string(),
            kind: None,
        }
    }

    /// Construct a classified [`Error::Provider`] (e.g. `kind = "auth"`).
    pub fn provider_classified(
        context: impl Into<String>,
        message: impl std::fmt::Display,
        kind: impl Into<String>,
    ) -> Self {
        Error::Provider {
            context: context.into(),
            message: message.to_string(),
            kind: Some(kind.into()),
        }
    }
}