evalhub_schema/eval.rs
1//! The Eval record: the material a Card was measured from.
2//!
3//! An Eval is a named, versioned set of runs, prompts, tasks or traces. It is
4//! what makes two Cards comparable: if both point at the same Eval version
5//! through `core/uses_eval`, they were measured on the same material, and the
6//! comparison view in `evalhub_server::api::relations` can line them up.
7//!
8//! The name is deliberately not "Source". Source suggests source code; an
9//! Eval is the evaluation itself, as a body of material, distinct from the
10//! Card that summarises it.
11//!
12//! # Shape
13//!
14//! ```text
15//! Eval
16//! ├── schema "evalhub.eval/1.0"
17//! ├── title
18//! ├── producer { name, version }
19//! ├── eval_kind run_set | prompt_set | task_set | trace_set
20//! ├── origin live | reconstructed | imported
21//! ├── model / task / harness / generation / trial / env (no grading facet)
22//! ├── runs[] { run_id, started_at, ended_at, outcome, calls, artifacts[] }
23//! │ required when eval_kind = run_set
24//! ├── relations[] derived_from / subset_of → other Evals
25//! ├── attachments[]
26//! ├── redaction
27//! └── ext
28//! ```
29//!
30//! A `run_set` example:
31//!
32//! ```json
33//! {
34//! "schema": "evalhub.eval/1.0",
35//! "title": "single2 K4, 33 runs",
36//! "producer": {"name": "my-harness", "version": "0.4.1"},
37//! "eval_kind": "run_set",
38//! "origin": "live",
39//! "harness": {"name": "my-harness", "version": "0.4.1"},
40//! "model": {"id": "qwen3.6-32b"},
41//! "task": {"id": "single2", "version": "3", "split": "test", "n": 33},
42//! "generation": {"temperature": 0.0},
43//! "trial": {"k": 4},
44//! "env": {"git": {"origin": "https://github.com/x/y", "commit": "abc123", "dirty": false}},
45//! "runs": [{
46//! "run_id": "r1",
47//! "started_at": "2026-09-20T10:00:00Z", "ended_at": "2026-09-20T10:02:11Z",
48//! "outcome": "pass",
49//! "calls": "calls/r1.jsonl",
50//! "artifacts": ["artifacts/r1/diff.patch"]
51//! }],
52//! "relations": [{"type": "core/subset_of", "to": "alice/single2-full@1"}],
53//! "attachments": [
54//! {"path": "calls/r1.jsonl", "sha256": "…", "size": 90210, "media_type": "application/x-ndjson"},
55//! {"path": "artifacts/r1/diff.patch", "sha256": "…", "size": 3311, "media_type": "text/x-diff"}
56//! ],
57//! "redaction": {"applied": false},
58//! "ext": {}
59//! }
60//! ```
61//!
62//! # `eval_kind` and `origin`
63//!
64//! `eval_kind` says what the material is. A `run_set` has `runs[]`, each with
65//! an outcome and a pointer (`calls`) to an attachment holding the model
66//! calls. The other kinds carry their material entirely in attachments.
67//!
68//! `origin` says how the material came to exist: `live` (captured while the
69//! evaluation ran), `reconstructed` (rebuilt afterwards from logs),
70//! `imported` (converted from another format, such as an Inspect `.eval`
71//! file or an EEE datastore entry). A reader comparing Cards can weigh a
72//! reconstructed Eval differently; the hub does not.
73//!
74//! # No grading facet
75//!
76//! An Eval has six of the seven facets. Grading belongs to the Card, because
77//! the same material can be graded several ways, and each grading is a
78//! different Card pointing at the same Eval.
79//!
80//! # Privacy across the relation
81//!
82//! A public Card may point at a private Eval. A reader without access sees
83//! that an Eval exists, its `version_id`, and the `content_hash` commitment —
84//! enough to know the Card is anchored to something specific — and nothing
85//! else. No title, no runs, no attachment URLs.
86
87use chrono::{DateTime, Utc};
88use schemars::JsonSchema;
89use serde::{Deserialize, Serialize};
90
91use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
92use crate::facet::{Env, Generation, Harness, Model, Task, Trial};
93
94/// An Eval: the material a Card was measured from.
95#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
96#[serde(deny_unknown_fields)]
97pub struct Eval {
98 /// Schema identifier; must be `evalhub.eval/1.0`.
99 #[schemars(extend("const" = "evalhub.eval/1.0"))]
100 pub schema: String,
101 /// Human-readable title.
102 pub title: String,
103 /// The software that wrote this record.
104 pub producer: Producer,
105 /// What the material is.
106 pub eval_kind: EvalKind,
107 /// How the material came to exist.
108 pub origin: Origin,
109 /// The model facet.
110 #[serde(default, skip_serializing_if = "Option::is_none")]
111 pub model: Option<Model>,
112 /// The task facet.
113 #[serde(default, skip_serializing_if = "Option::is_none")]
114 pub task: Option<Task>,
115 /// The harness facet.
116 #[serde(default, skip_serializing_if = "Option::is_none")]
117 pub harness: Option<Harness>,
118 /// The generation facet.
119 #[serde(default, skip_serializing_if = "Option::is_none")]
120 pub generation: Option<Generation>,
121 /// The trial facet.
122 #[serde(default, skip_serializing_if = "Option::is_none")]
123 pub trial: Option<Trial>,
124 /// The environment facet.
125 #[serde(default, skip_serializing_if = "Option::is_none")]
126 pub env: Option<Env>,
127 /// The runs. Required when `eval_kind` is `run_set`.
128 #[serde(default, skip_serializing_if = "Vec::is_empty")]
129 pub runs: Vec<Run>,
130 /// Edges to other Evals (`core/derived_from`, `core/subset_of`).
131 #[serde(default, skip_serializing_if = "Vec::is_empty")]
132 pub relations: Vec<Relation>,
133 /// Files this record refers to.
134 #[serde(default, skip_serializing_if = "Vec::is_empty")]
135 pub attachments: Vec<Attachment>,
136 /// What the producer removed before publishing.
137 #[serde(default, skip_serializing_if = "Option::is_none")]
138 pub redaction: Option<Redaction>,
139 /// Producer-private extensions keyed by namespace.
140 #[serde(default)]
141 pub ext: Ext,
142}
143
144/// What an Eval's material is.
145#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
146#[serde(rename_all = "snake_case")]
147pub enum EvalKind {
148 /// A set of runs, each with its model calls.
149 RunSet,
150 /// A set of prompts.
151 PromptSet,
152 /// A set of tasks.
153 TaskSet,
154 /// A set of traces.
155 TraceSet,
156}
157
158/// How an Eval's material came to exist.
159#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
160#[serde(rename_all = "snake_case")]
161pub enum Origin {
162 /// Captured while the evaluation ran.
163 Live,
164 /// Rebuilt afterwards from logs.
165 Reconstructed,
166 /// Converted from another format.
167 Imported,
168}
169
170/// One run in a `run_set`.
171#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
172#[serde(deny_unknown_fields)]
173pub struct Run {
174 /// Run identifier, unique within the Eval.
175 pub run_id: String,
176 /// When the run started.
177 #[serde(default, skip_serializing_if = "Option::is_none")]
178 pub started_at: Option<DateTime<Utc>>,
179 /// When the run ended.
180 #[serde(default, skip_serializing_if = "Option::is_none")]
181 pub ended_at: Option<DateTime<Utc>>,
182 /// How the run ended.
183 pub outcome: Outcome,
184 /// `attachments[].path` of the model calls made during the run.
185 #[serde(default, skip_serializing_if = "Option::is_none")]
186 pub calls: Option<String>,
187 /// `attachments[].path` entries of artifacts the run produced.
188 #[serde(default, skip_serializing_if = "Vec::is_empty")]
189 pub artifacts: Vec<String>,
190}
191
192/// How a run ended.
193#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
194#[serde(rename_all = "snake_case")]
195pub enum Outcome {
196 /// The run completed and passed.
197 Pass,
198 /// The run completed and failed.
199 Fail,
200 /// The run errored before completion.
201 Error,
202 /// The run was skipped.
203 Skipped,
204}