Skip to main content

evalhub_schema/
eval.rs

1//! The Eval record: the material a Card was measured from.
2//!
3//! An Eval is a named, versioned set of runs, prompts, tasks or traces. It is
4//! what makes two Cards comparable: if both point at the same Eval version
5//! through `core/uses_eval`, they were measured on the same material, and the
6//! comparison view in `evalhub_server::api::relations` can line them up.
7//!
8//! The name is deliberately not "Source". Source suggests source code; an
9//! Eval is the evaluation itself, as a body of material, distinct from the
10//! Card that summarises it.
11//!
12//! # Shape
13//!
14//! ```text
15//! Eval
16//! ├── schema        "evalhub.eval/1.0"
17//! ├── title
18//! ├── producer      { name, version }
19//! ├── eval_kind     run_set | prompt_set | task_set | trace_set
20//! ├── origin        live | reconstructed | imported
21//! ├── model / task / harness / generation / trial / env   (no grading facet)
22//! ├── runs[]        { run_id, started_at, ended_at, outcome, calls, artifacts[] }
23//! │                 required when eval_kind = run_set
24//! ├── relations[]   derived_from / subset_of → other Evals
25//! ├── attachments[]
26//! ├── redaction
27//! └── ext
28//! ```
29//!
30//! A `run_set` example:
31//!
32//! ```json
33//! {
34//!   "schema": "evalhub.eval/1.0",
35//!   "title": "single2 K4, 33 runs",
36//!   "producer": {"name": "my-harness", "version": "0.4.1"},
37//!   "eval_kind": "run_set",
38//!   "origin": "live",
39//!   "harness": {"name": "my-harness", "version": "0.4.1"},
40//!   "model": {"id": "qwen3.6-32b"},
41//!   "task": {"id": "single2", "version": "3", "split": "test", "n": 33},
42//!   "generation": {"temperature": 0.0},
43//!   "trial": {"k": 4},
44//!   "env": {"git": {"origin": "https://github.com/x/y", "commit": "abc123", "dirty": false}},
45//!   "runs": [{
46//!     "run_id": "r1",
47//!     "started_at": "2026-09-20T10:00:00Z", "ended_at": "2026-09-20T10:02:11Z",
48//!     "outcome": "pass",
49//!     "calls": "calls/r1.jsonl",
50//!     "artifacts": ["artifacts/r1/diff.patch"]
51//!   }],
52//!   "relations": [{"type": "core/subset_of", "to": "alice/single2-full@1"}],
53//!   "attachments": [
54//!     {"path": "calls/r1.jsonl", "sha256": "…", "size": 90210, "media_type": "application/x-ndjson"},
55//!     {"path": "artifacts/r1/diff.patch", "sha256": "…", "size": 3311, "media_type": "text/x-diff"}
56//!   ],
57//!   "redaction": {"applied": false},
58//!   "ext": {}
59//! }
60//! ```
61//!
62//! # `eval_kind` and `origin`
63//!
64//! `eval_kind` says what the material is. A `run_set` has `runs[]`, each with
65//! an outcome and a pointer (`calls`) to an attachment holding the model
66//! calls. The other kinds carry their material entirely in attachments.
67//!
68//! `origin` says how the material came to exist: `live` (captured while the
69//! evaluation ran), `reconstructed` (rebuilt afterwards from logs),
70//! `imported` (converted from another format, such as an Inspect `.eval`
71//! file or an EEE datastore entry). A reader comparing Cards can weigh a
72//! reconstructed Eval differently; the hub does not.
73//!
74//! # No grading facet
75//!
76//! An Eval has six of the seven facets. Grading belongs to the Card, because
77//! the same material can be graded several ways, and each grading is a
78//! different Card pointing at the same Eval.
79//!
80//! # Privacy across the relation
81//!
82//! A public Card may point at a private Eval. A reader without access sees
83//! that an Eval exists, its `version_id`, and the `content_hash` commitment —
84//! enough to know the Card is anchored to something specific — and nothing
85//! else. No title, no runs, no attachment URLs.
86
87use chrono::{DateTime, Utc};
88use schemars::JsonSchema;
89use serde::{Deserialize, Serialize};
90
91use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
92use crate::facet::{Env, Generation, Harness, Model, Task, Trial};
93
94/// An Eval: the material a Card was measured from.
95#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
96#[serde(deny_unknown_fields)]
97pub struct Eval {
98    /// Schema identifier; must be `evalhub.eval/1.0`.
99    #[schemars(extend("const" = "evalhub.eval/1.0"))]
100    pub schema: String,
101    /// Human-readable title.
102    pub title: String,
103    /// The software that wrote this record.
104    pub producer: Producer,
105    /// What the material is.
106    pub eval_kind: EvalKind,
107    /// How the material came to exist.
108    pub origin: Origin,
109    /// The model facet.
110    #[serde(default, skip_serializing_if = "Option::is_none")]
111    pub model: Option<Model>,
112    /// The task facet.
113    #[serde(default, skip_serializing_if = "Option::is_none")]
114    pub task: Option<Task>,
115    /// The harness facet.
116    #[serde(default, skip_serializing_if = "Option::is_none")]
117    pub harness: Option<Harness>,
118    /// The generation facet.
119    #[serde(default, skip_serializing_if = "Option::is_none")]
120    pub generation: Option<Generation>,
121    /// The trial facet.
122    #[serde(default, skip_serializing_if = "Option::is_none")]
123    pub trial: Option<Trial>,
124    /// The environment facet.
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub env: Option<Env>,
127    /// The runs. Required when `eval_kind` is `run_set`.
128    #[serde(default, skip_serializing_if = "Vec::is_empty")]
129    pub runs: Vec<Run>,
130    /// Edges to other Evals (`core/derived_from`, `core/subset_of`).
131    #[serde(default, skip_serializing_if = "Vec::is_empty")]
132    pub relations: Vec<Relation>,
133    /// Files this record refers to.
134    #[serde(default, skip_serializing_if = "Vec::is_empty")]
135    pub attachments: Vec<Attachment>,
136    /// What the producer removed before publishing.
137    #[serde(default, skip_serializing_if = "Option::is_none")]
138    pub redaction: Option<Redaction>,
139    /// Producer-private extensions keyed by namespace.
140    #[serde(default)]
141    pub ext: Ext,
142}
143
144/// What an Eval's material is.
145#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
146#[serde(rename_all = "snake_case")]
147pub enum EvalKind {
148    /// A set of runs, each with its model calls.
149    RunSet,
150    /// A set of prompts.
151    PromptSet,
152    /// A set of tasks.
153    TaskSet,
154    /// A set of traces.
155    TraceSet,
156}
157
158/// How an Eval's material came to exist.
159#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
160#[serde(rename_all = "snake_case")]
161pub enum Origin {
162    /// Captured while the evaluation ran.
163    Live,
164    /// Rebuilt afterwards from logs.
165    Reconstructed,
166    /// Converted from another format.
167    Imported,
168}
169
170/// One run in a `run_set`.
171#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
172#[serde(deny_unknown_fields)]
173pub struct Run {
174    /// Run identifier, unique within the Eval.
175    pub run_id: String,
176    /// When the run started.
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub started_at: Option<DateTime<Utc>>,
179    /// When the run ended.
180    #[serde(default, skip_serializing_if = "Option::is_none")]
181    pub ended_at: Option<DateTime<Utc>>,
182    /// How the run ended.
183    pub outcome: Outcome,
184    /// `attachments[].path` of the model calls made during the run.
185    #[serde(default, skip_serializing_if = "Option::is_none")]
186    pub calls: Option<String>,
187    /// `attachments[].path` entries of artifacts the run produced.
188    #[serde(default, skip_serializing_if = "Vec::is_empty")]
189    pub artifacts: Vec<String>,
190}
191
192/// How a run ended.
193#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
194#[serde(rename_all = "snake_case")]
195pub enum Outcome {
196    /// The run completed and passed.
197    Pass,
198    /// The run completed and failed.
199    Fail,
200    /// The run errored before completion.
201    Error,
202    /// The run was skipped.
203    Skipped,
204}