evalhub_schema/eval.rs
1//! The Eval record: the material a Card was measured from.
2//!
3//! An Eval is a named, versioned set of runs, prompts, tasks or traces. It is
4//! what makes two Cards comparable: if both point at the same Eval version
5//! through `core/uses_eval`, they were measured on the same material, and the
6//! comparison view in `evalhub_server::api::relations` can line them up.
7//!
8//! The name is deliberately not "Source". Source suggests source code; an
9//! Eval is the evaluation itself, as a body of material, distinct from the
10//! Card that summarises it.
11//!
12//! # Shape
13//!
14//! ```text
15//! Eval
16//! ├── schema "evalhub.eval/1.0"
17//! ├── title
18//! ├── producer { name, version }
19//! ├── eval_kind run_set | prompt_set | task_set | trace_set
20//! ├── origin live | reconstructed | imported
21//! ├── model / task / harness / generation / trial / env (no grading facet)
22//! ├── runs[] { run_id, started_at, ended_at, outcome, calls, artifacts[] }
23//! │ required when eval_kind = run_set
24//! ├── relations[] derived_from / subset_of → other Evals
25//! ├── attachments[]
26//! ├── redaction
27//! └── ext
28//! ```
29//!
30//! A `run_set` example:
31//!
32//! ```json
33//! {
34//! "schema": "evalhub.eval/1.0",
35//! "title": "single2 K4, 33 runs",
36//! "producer": {"name": "my-harness", "version": "0.4.1"},
37//! "eval_kind": "run_set",
38//! "origin": "live",
39//! "harness": {"name": "my-harness", "version": "0.4.1"},
40//! "model": {"id": "qwen3.6-32b"},
41//! "task": {"id": "single2", "version": "3", "split": "test", "n": 33},
42//! "generation": {"temperature": 0.0},
43//! "trial": {"k": 4},
44//! "env": {"git": {"origin": "https://github.com/x/y", "commit": "abc123", "dirty": false}},
45//! "runs": [{
46//! "run_id": "r1",
47//! "started_at": "2026-09-20T10:00:00Z", "ended_at": "2026-09-20T10:02:11Z",
48//! "outcome": "pass",
49//! "calls": "calls/r1.jsonl",
50//! "artifacts": ["artifacts/r1/diff.patch"]
51//! }],
52//! "relations": [{"type": "core/subset_of", "to": "alice/single2-full@1"}],
53//! "attachments": [
54//! {"path": "calls/r1.jsonl", "sha256": "…", "size": 90210, "media_type": "application/x-ndjson"},
55//! {"path": "artifacts/r1/diff.patch", "sha256": "…", "size": 3311, "media_type": "text/x-diff"}
56//! ],
57//! "redaction": {"applied": false},
58//! "ext": {}
59//! }
60//! ```
61//!
62//! # `eval_kind` and `origin`
63//!
64//! `eval_kind` says what the material is. A `run_set` has `runs[]`, each with
65//! an outcome and a pointer (`calls`) to an attachment holding the model
66//! calls. The other kinds carry their material entirely in attachments.
67//!
68//! `origin` says how the material came to exist: `live` (captured while the
69//! evaluation ran), `reconstructed` (rebuilt afterwards from logs),
70//! `imported` (converted from another format, such as an Inspect `.eval`
71//! file or an EEE datastore entry). A reader comparing Cards can weigh a
72//! reconstructed Eval differently; the hub does not.
73//!
74//! # No grading facet
75//!
76//! An Eval has six of the seven facets. Grading belongs to the Card, because
77//! the same material can be graded several ways, and each grading is a
78//! different Card pointing at the same Eval.
79//!
80//! # Privacy across the relation
81//!
82//! A public Card may point at a private Eval. A reader without access sees
83//! that an Eval exists, its `version_id`, and the `content_hash` commitment —
84//! enough to know the Card is anchored to something specific — and nothing
85//! else. No title, no runs, no attachment URLs.
86//!
87//! The Card's own body does not reveal more than its edges. On every read
88//! that returns a body, a `relations[]` element whose target the reader may
89//! not see is removed and listed under `withheld.relations` as the same
90//! commitment (`type`, `version_id`, `content_hash`). The body returned is
91//! then not the stored one, and its `content_hash` does not match it. A
92//! query's `relations.to` condition matches only targets the caller may
93//! see, so a search cannot confirm a private name either.
94//!
95//! A write learns no more than a read. A target the writer may not see is
96//! stored unresolved, exactly as one that does not exist: the response and
97//! the `refs_resolved` badge are the same in both cases. The edge resolves
98//! for a reader once the target becomes visible to them.
99
100use chrono::{DateTime, Utc};
101use schemars::JsonSchema;
102use serde::{Deserialize, Serialize};
103
104use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
105use crate::facet::{Env, Generation, Harness, Model, Task, Trial};
106
107/// An Eval: the material a Card was measured from.
108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
109#[serde(deny_unknown_fields)]
110pub struct Eval {
111 /// Schema identifier; must be `evalhub.eval/1.0`.
112 #[schemars(extend("const" = "evalhub.eval/1.0"))]
113 pub schema: String,
114 /// Human-readable title.
115 pub title: String,
116 /// The software that wrote this record.
117 pub producer: Producer,
118 /// What the material is.
119 pub eval_kind: EvalKind,
120 /// How the material came to exist.
121 pub origin: Origin,
122 /// The model facet.
123 #[serde(default, skip_serializing_if = "Option::is_none")]
124 pub model: Option<Model>,
125 /// The task facet.
126 #[serde(default, skip_serializing_if = "Option::is_none")]
127 pub task: Option<Task>,
128 /// The harness facet.
129 #[serde(default, skip_serializing_if = "Option::is_none")]
130 pub harness: Option<Harness>,
131 /// The generation facet.
132 #[serde(default, skip_serializing_if = "Option::is_none")]
133 pub generation: Option<Generation>,
134 /// The trial facet.
135 #[serde(default, skip_serializing_if = "Option::is_none")]
136 pub trial: Option<Trial>,
137 /// The environment facet.
138 #[serde(default, skip_serializing_if = "Option::is_none")]
139 pub env: Option<Env>,
140 /// The runs. Required when `eval_kind` is `run_set`.
141 #[serde(default, skip_serializing_if = "Vec::is_empty")]
142 pub runs: Vec<Run>,
143 /// Edges to other Evals (`core/derived_from`, `core/subset_of`).
144 #[serde(default, skip_serializing_if = "Vec::is_empty")]
145 pub relations: Vec<Relation>,
146 /// Files this record refers to.
147 #[serde(default, skip_serializing_if = "Vec::is_empty")]
148 pub attachments: Vec<Attachment>,
149 /// What the producer removed before publishing.
150 #[serde(default, skip_serializing_if = "Option::is_none")]
151 pub redaction: Option<Redaction>,
152 /// Producer-private extensions keyed by namespace.
153 #[serde(default)]
154 pub ext: Ext,
155}
156
157/// What an Eval's material is.
158#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
159#[serde(rename_all = "snake_case")]
160pub enum EvalKind {
161 /// A set of runs, each with its model calls.
162 RunSet,
163 /// A set of prompts.
164 PromptSet,
165 /// A set of tasks.
166 TaskSet,
167 /// A set of traces.
168 TraceSet,
169}
170
171/// How an Eval's material came to exist.
172#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
173#[serde(rename_all = "snake_case")]
174pub enum Origin {
175 /// Captured while the evaluation ran.
176 Live,
177 /// Rebuilt afterwards from logs.
178 Reconstructed,
179 /// Converted from another format.
180 Imported,
181}
182
183/// One run in a `run_set`.
184#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
185#[serde(deny_unknown_fields)]
186pub struct Run {
187 /// Run identifier, unique within the Eval.
188 pub run_id: String,
189 /// When the run started.
190 #[serde(default, skip_serializing_if = "Option::is_none")]
191 pub started_at: Option<DateTime<Utc>>,
192 /// When the run ended.
193 #[serde(default, skip_serializing_if = "Option::is_none")]
194 pub ended_at: Option<DateTime<Utc>>,
195 /// How the run ended.
196 pub outcome: Outcome,
197 /// `attachments[].path` of the model calls made during the run.
198 #[serde(default, skip_serializing_if = "Option::is_none")]
199 pub calls: Option<String>,
200 /// `attachments[].path` entries of artifacts the run produced.
201 #[serde(default, skip_serializing_if = "Vec::is_empty")]
202 pub artifacts: Vec<String>,
203}
204
205/// How a run ended.
206#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
207#[serde(rename_all = "snake_case")]
208pub enum Outcome {
209 /// The run completed and passed.
210 Pass,
211 /// The run completed and failed.
212 Fail,
213 /// The run errored before completion.
214 Error,
215 /// The run was skipped.
216 Skipped,
217}