Skip to main content

evalhub_schema/eval/
v1.rs

1//! The 1.0 shape of the Eval record (`evalhub.eval/1.0`), with `runs[]` in
2//! the body.
3//!
4//! **Deprecated: accepted by 0.2.0 only, removed in 0.3.0.** This module exists for the
5//! compatibility window the crate's versioning rule promises (see
6//! "Versioning of the schema itself" in the crate doc): removing `runs`
7//! from the body is a major bump of the record schema, and the hub keeps
8//! accepting the previous major for one release. 0.2.0 accepts a
9//! `evalhub.eval/1.0` body and converts it at ingest into a
10//! `evalhub.eval/2.0` header ([`super::Eval`]) plus one [`crate::run::Run`]
11//! per element of `runs[]`. 0.3.0 removes this module, the
12//! `schemas/eval.json` document generated from it, and the conversion.
13//!
14//! New producers write [`super::Eval`] and [`crate::run::Run`]. Nothing
15//! here is extended: a 1.0 body is frozen as it was in 0.1.x, and
16//! `schemas/eval.json` is byte-for-byte the document 0.1.x served, so a
17//! 0.1.x client that validates against it keeps working for the window.
18//!
19//! The note is prose rather than `#[deprecated]` on purpose. The attribute
20//! would fire on every internal use (the schema tests, the 1.0 → 2.0
21//! conversion, the data migration), and the workspace builds with
22//! `clippy -D warnings`, so each of those uses would need an `allow` that
23//! says nothing the date in this paragraph does not.
24//!
25//! # Shape
26//!
27//! ```text
28//! Eval (1.0)
29//! ├── schema        "evalhub.eval/1.0"
30//! ├── title / producer / eval_kind / origin
31//! ├── model / task / harness / generation / trial / env
32//! ├── runs[]        { run_id, started_at, ended_at, outcome, calls, artifacts[] }
33//! ├── relations[] / attachments[] / redaction
34//! └── ext
35//! ```
36//!
37//! `outcome` mixed the state of the execution (`error`, `skipped`) with a
38//! verdict on it (`pass`, `fail`). In 2.0 the run keeps only the state
39//! ([`crate::run::RunStatus`]) and the verdict moves to the Card
40//! ([`crate::card::RunResult`]); the conversion keeps the dropped `outcome`
41//! verbatim in the run's `meta`.
42//!
43//! The doc comments on the types below are part of `schemas/eval.json`
44//! (they become its `description`s) and are therefore frozen with it.
45
46use chrono::{DateTime, Utc};
47use schemars::JsonSchema;
48use serde::{Deserialize, Serialize};
49
50use super::{EvalKind, Origin};
51use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
52use crate::facet::{Env, Generation, Harness, Model, Task, Trial};
53
54/// An Eval: the material a Card was measured from.
55#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
56#[serde(deny_unknown_fields)]
57pub struct Eval {
58    /// Schema identifier; must be `evalhub.eval/1.0`.
59    #[schemars(extend("const" = "evalhub.eval/1.0"))]
60    pub schema: String,
61    /// Human-readable title.
62    pub title: String,
63    /// The software that wrote this record.
64    pub producer: Producer,
65    /// What the material is.
66    pub eval_kind: EvalKind,
67    /// How the material came to exist.
68    pub origin: Origin,
69    /// The model facet.
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub model: Option<Model>,
72    /// The task facet.
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    pub task: Option<Task>,
75    /// The harness facet.
76    #[serde(default, skip_serializing_if = "Option::is_none")]
77    pub harness: Option<Harness>,
78    /// The generation facet.
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub generation: Option<Generation>,
81    /// The trial facet.
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub trial: Option<Trial>,
84    /// The environment facet.
85    #[serde(default, skip_serializing_if = "Option::is_none")]
86    pub env: Option<Env>,
87    /// The runs. Required when `eval_kind` is `run_set`.
88    #[serde(default, skip_serializing_if = "Vec::is_empty")]
89    pub runs: Vec<Run>,
90    /// Edges to other Evals (`core/derived_from`, `core/subset_of`).
91    #[serde(default, skip_serializing_if = "Vec::is_empty")]
92    pub relations: Vec<Relation>,
93    /// Files this record refers to.
94    #[serde(default, skip_serializing_if = "Vec::is_empty")]
95    pub attachments: Vec<Attachment>,
96    /// What the producer removed before publishing.
97    #[serde(default, skip_serializing_if = "Option::is_none")]
98    pub redaction: Option<Redaction>,
99    /// Producer-private extensions keyed by namespace.
100    #[serde(default)]
101    pub ext: Ext,
102}
103
104/// One run in a `run_set`.
105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
106#[serde(deny_unknown_fields)]
107pub struct Run {
108    /// Run identifier, unique within the Eval.
109    pub run_id: String,
110    /// When the run started.
111    #[serde(default, skip_serializing_if = "Option::is_none")]
112    pub started_at: Option<DateTime<Utc>>,
113    /// When the run ended.
114    #[serde(default, skip_serializing_if = "Option::is_none")]
115    pub ended_at: Option<DateTime<Utc>>,
116    /// How the run ended.
117    pub outcome: Outcome,
118    /// `attachments[].path` of the model calls made during the run.
119    #[serde(default, skip_serializing_if = "Option::is_none")]
120    pub calls: Option<String>,
121    /// `attachments[].path` entries of artifacts the run produced.
122    #[serde(default, skip_serializing_if = "Vec::is_empty")]
123    pub artifacts: Vec<String>,
124}
125
126/// How a run ended.
127#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
128#[serde(rename_all = "snake_case")]
129pub enum Outcome {
130    /// The run completed and passed.
131    Pass,
132    /// The run completed and failed.
133    Fail,
134    /// The run errored before completion.
135    Error,
136    /// The run was skipped.
137    Skipped,
138}