evalhub_schema/eval/v1.rs
1//! The 1.0 shape of the Eval record (`evalhub.eval/1.0`), with `runs[]` in
2//! the body.
3//!
4//! **Deprecated: accepted by 0.2.0 only, removed in 0.3.0.** This module exists for the
5//! compatibility window the crate's versioning rule promises (see
6//! "Versioning of the schema itself" in the crate doc): removing `runs`
7//! from the body is a major bump of the record schema, and the hub keeps
8//! accepting the previous major for one release. 0.2.0 accepts a
9//! `evalhub.eval/1.0` body and converts it at ingest into a
10//! `evalhub.eval/2.0` header ([`super::Eval`]) plus one [`crate::run::Run`]
11//! per element of `runs[]`. 0.3.0 removes this module, the
12//! `schemas/eval.json` document generated from it, and the conversion.
13//!
14//! New producers write [`super::Eval`] and [`crate::run::Run`]. Nothing
15//! here is extended: a 1.0 body is frozen as it was in 0.1.x, and
16//! `schemas/eval.json` is byte-for-byte the document 0.1.x served, so a
17//! 0.1.x client that validates against it keeps working for the window.
18//!
19//! The note is prose rather than `#[deprecated]` on purpose. The attribute
20//! would fire on every internal use (the schema tests, the 1.0 → 2.0
21//! conversion, the data migration), and the workspace builds with
22//! `clippy -D warnings`, so each of those uses would need an `allow` that
23//! says nothing the date in this paragraph does not.
24//!
25//! # Shape
26//!
27//! ```text
28//! Eval (1.0)
29//! ├── schema "evalhub.eval/1.0"
30//! ├── title / producer / eval_kind / origin
31//! ├── model / task / harness / generation / trial / env
32//! ├── runs[] { run_id, started_at, ended_at, outcome, calls, artifacts[] }
33//! ├── relations[] / attachments[] / redaction
34//! └── ext
35//! ```
36//!
37//! `outcome` mixed the state of the execution (`error`, `skipped`) with a
38//! verdict on it (`pass`, `fail`). In 2.0 the run keeps only the state
39//! ([`crate::run::RunStatus`]) and the verdict moves to the Card
40//! ([`crate::card::RunResult`]); the conversion keeps the dropped `outcome`
41//! verbatim in the run's `meta`.
42//!
43//! The doc comments on the types below are part of `schemas/eval.json`
44//! (they become its `description`s) and are therefore frozen with it.
45
46use chrono::{DateTime, Utc};
47use schemars::JsonSchema;
48use serde::{Deserialize, Serialize};
49
50use super::{EvalKind, Origin};
51use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
52use crate::facet::{Env, Generation, Harness, Model, Task, Trial};
53
54/// An Eval: the material a Card was measured from.
55#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
56#[serde(deny_unknown_fields)]
57pub struct Eval {
58 /// Schema identifier; must be `evalhub.eval/1.0`.
59 #[schemars(extend("const" = "evalhub.eval/1.0"))]
60 pub schema: String,
61 /// Human-readable title.
62 pub title: String,
63 /// The software that wrote this record.
64 pub producer: Producer,
65 /// What the material is.
66 pub eval_kind: EvalKind,
67 /// How the material came to exist.
68 pub origin: Origin,
69 /// The model facet.
70 #[serde(default, skip_serializing_if = "Option::is_none")]
71 pub model: Option<Model>,
72 /// The task facet.
73 #[serde(default, skip_serializing_if = "Option::is_none")]
74 pub task: Option<Task>,
75 /// The harness facet.
76 #[serde(default, skip_serializing_if = "Option::is_none")]
77 pub harness: Option<Harness>,
78 /// The generation facet.
79 #[serde(default, skip_serializing_if = "Option::is_none")]
80 pub generation: Option<Generation>,
81 /// The trial facet.
82 #[serde(default, skip_serializing_if = "Option::is_none")]
83 pub trial: Option<Trial>,
84 /// The environment facet.
85 #[serde(default, skip_serializing_if = "Option::is_none")]
86 pub env: Option<Env>,
87 /// The runs. Required when `eval_kind` is `run_set`.
88 #[serde(default, skip_serializing_if = "Vec::is_empty")]
89 pub runs: Vec<Run>,
90 /// Edges to other Evals (`core/derived_from`, `core/subset_of`).
91 #[serde(default, skip_serializing_if = "Vec::is_empty")]
92 pub relations: Vec<Relation>,
93 /// Files this record refers to.
94 #[serde(default, skip_serializing_if = "Vec::is_empty")]
95 pub attachments: Vec<Attachment>,
96 /// What the producer removed before publishing.
97 #[serde(default, skip_serializing_if = "Option::is_none")]
98 pub redaction: Option<Redaction>,
99 /// Producer-private extensions keyed by namespace.
100 #[serde(default)]
101 pub ext: Ext,
102}
103
104/// One run in a `run_set`.
105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
106#[serde(deny_unknown_fields)]
107pub struct Run {
108 /// Run identifier, unique within the Eval.
109 pub run_id: String,
110 /// When the run started.
111 #[serde(default, skip_serializing_if = "Option::is_none")]
112 pub started_at: Option<DateTime<Utc>>,
113 /// When the run ended.
114 #[serde(default, skip_serializing_if = "Option::is_none")]
115 pub ended_at: Option<DateTime<Utc>>,
116 /// How the run ended.
117 pub outcome: Outcome,
118 /// `attachments[].path` of the model calls made during the run.
119 #[serde(default, skip_serializing_if = "Option::is_none")]
120 pub calls: Option<String>,
121 /// `attachments[].path` entries of artifacts the run produced.
122 #[serde(default, skip_serializing_if = "Vec::is_empty")]
123 pub artifacts: Vec<String>,
124}
125
126/// How a run ended.
127#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
128#[serde(rename_all = "snake_case")]
129pub enum Outcome {
130 /// The run completed and passed.
131 Pass,
132 /// The run completed and failed.
133 Fail,
134 /// The run errored before completion.
135 Error,
136 /// The run was skipped.
137 Skipped,
138}