Skip to main content

evalhub_schema/
eval.rs

1//! The Eval record: the material a Card was measured from.
2//!
3//! An Eval is a named, versioned set of runs, prompts, tasks or traces. It is
4//! what makes two Cards comparable: if both point at the same Eval version
5//! through `core/uses_eval`, they were measured on the same material, and the
6//! comparison view in `evalhub_server::api::relations` can line them up.
7//!
8//! The name is deliberately not "Source". Source suggests source code; an
9//! Eval is the evaluation itself, as a body of material, distinct from the
10//! Card that summarises it.
11//!
12//! # Shape
13//!
14//! ```text
15//! Eval
16//! ├── schema        "evalhub.eval/1.0"
17//! ├── title
18//! ├── producer      { name, version }
19//! ├── eval_kind     run_set | prompt_set | task_set | trace_set
20//! ├── origin        live | reconstructed | imported
21//! ├── model / task / harness / generation / trial / env   (no grading facet)
22//! ├── runs[]        { run_id, started_at, ended_at, outcome, calls, artifacts[] }
23//! │                 required when eval_kind = run_set
24//! ├── relations[]   derived_from / subset_of → other Evals
25//! ├── attachments[]
26//! ├── redaction
27//! └── ext
28//! ```
29//!
30//! A `run_set` example:
31//!
32//! ```json
33//! {
34//!   "schema": "evalhub.eval/1.0",
35//!   "title": "single2 K4, 33 runs",
36//!   "producer": {"name": "my-harness", "version": "0.4.1"},
37//!   "eval_kind": "run_set",
38//!   "origin": "live",
39//!   "harness": {"name": "my-harness", "version": "0.4.1"},
40//!   "model": {"id": "qwen3.6-32b"},
41//!   "task": {"id": "single2", "version": "3", "split": "test", "n": 33},
42//!   "generation": {"temperature": 0.0},
43//!   "trial": {"k": 4},
44//!   "env": {"git": {"origin": "https://github.com/x/y", "commit": "abc123", "dirty": false}},
45//!   "runs": [{
46//!     "run_id": "r1",
47//!     "started_at": "2026-09-20T10:00:00Z", "ended_at": "2026-09-20T10:02:11Z",
48//!     "outcome": "pass",
49//!     "calls": "calls/r1.jsonl",
50//!     "artifacts": ["artifacts/r1/diff.patch"]
51//!   }],
52//!   "relations": [{"type": "core/subset_of", "to": "alice/single2-full@1"}],
53//!   "attachments": [
54//!     {"path": "calls/r1.jsonl", "sha256": "…", "size": 90210, "media_type": "application/x-ndjson"},
55//!     {"path": "artifacts/r1/diff.patch", "sha256": "…", "size": 3311, "media_type": "text/x-diff"}
56//!   ],
57//!   "redaction": {"applied": false},
58//!   "ext": {}
59//! }
60//! ```
61//!
62//! # `eval_kind` and `origin`
63//!
64//! `eval_kind` says what the material is. A `run_set` has `runs[]`, each with
65//! an outcome and a pointer (`calls`) to an attachment holding the model
66//! calls. The other kinds carry their material entirely in attachments.
67//!
68//! `origin` says how the material came to exist: `live` (captured while the
69//! evaluation ran), `reconstructed` (rebuilt afterwards from logs),
70//! `imported` (converted from another format, such as an Inspect `.eval`
71//! file or an EEE datastore entry). A reader comparing Cards can weigh a
72//! reconstructed Eval differently; the hub does not.
73//!
74//! # No grading facet
75//!
76//! An Eval has six of the seven facets. Grading belongs to the Card, because
77//! the same material can be graded several ways, and each grading is a
78//! different Card pointing at the same Eval.
79//!
80//! # Privacy across the relation
81//!
82//! A public Card may point at a private Eval. A reader without access sees
83//! that an Eval exists, its `version_id`, and the `content_hash` commitment —
84//! enough to know the Card is anchored to something specific — and nothing
85//! else. No title, no runs, no attachment URLs.
86//!
87//! The Card's own body does not reveal more than its edges. On every read
88//! that returns a body, a `relations[]` element whose target the reader may
89//! not see is removed and listed under `withheld.relations` as the same
90//! commitment (`type`, `version_id`, `content_hash`). The body returned is
91//! then not the stored one, and its `content_hash` does not match it. A
92//! query's `relations.to` condition matches only targets the caller may
93//! see, so a search cannot confirm a private name either.
94//!
95//! A write learns no more than a read. A target the writer may not see is
96//! stored unresolved, exactly as one that does not exist: the response and
97//! the `refs_resolved` badge are the same in both cases. The edge resolves
98//! for a reader once the target becomes visible to them.
99
100use chrono::{DateTime, Utc};
101use schemars::JsonSchema;
102use serde::{Deserialize, Serialize};
103
104use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
105use crate::facet::{Env, Generation, Harness, Model, Task, Trial};
106
107/// An Eval: the material a Card was measured from.
108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
109#[serde(deny_unknown_fields)]
110pub struct Eval {
111    /// Schema identifier; must be `evalhub.eval/1.0`.
112    #[schemars(extend("const" = "evalhub.eval/1.0"))]
113    pub schema: String,
114    /// Human-readable title.
115    pub title: String,
116    /// The software that wrote this record.
117    pub producer: Producer,
118    /// What the material is.
119    pub eval_kind: EvalKind,
120    /// How the material came to exist.
121    pub origin: Origin,
122    /// The model facet.
123    #[serde(default, skip_serializing_if = "Option::is_none")]
124    pub model: Option<Model>,
125    /// The task facet.
126    #[serde(default, skip_serializing_if = "Option::is_none")]
127    pub task: Option<Task>,
128    /// The harness facet.
129    #[serde(default, skip_serializing_if = "Option::is_none")]
130    pub harness: Option<Harness>,
131    /// The generation facet.
132    #[serde(default, skip_serializing_if = "Option::is_none")]
133    pub generation: Option<Generation>,
134    /// The trial facet.
135    #[serde(default, skip_serializing_if = "Option::is_none")]
136    pub trial: Option<Trial>,
137    /// The environment facet.
138    #[serde(default, skip_serializing_if = "Option::is_none")]
139    pub env: Option<Env>,
140    /// The runs. Required when `eval_kind` is `run_set`.
141    #[serde(default, skip_serializing_if = "Vec::is_empty")]
142    pub runs: Vec<Run>,
143    /// Edges to other Evals (`core/derived_from`, `core/subset_of`).
144    #[serde(default, skip_serializing_if = "Vec::is_empty")]
145    pub relations: Vec<Relation>,
146    /// Files this record refers to.
147    #[serde(default, skip_serializing_if = "Vec::is_empty")]
148    pub attachments: Vec<Attachment>,
149    /// What the producer removed before publishing.
150    #[serde(default, skip_serializing_if = "Option::is_none")]
151    pub redaction: Option<Redaction>,
152    /// Producer-private extensions keyed by namespace.
153    #[serde(default)]
154    pub ext: Ext,
155}
156
157/// What an Eval's material is.
158#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
159#[serde(rename_all = "snake_case")]
160pub enum EvalKind {
161    /// A set of runs, each with its model calls.
162    RunSet,
163    /// A set of prompts.
164    PromptSet,
165    /// A set of tasks.
166    TaskSet,
167    /// A set of traces.
168    TraceSet,
169}
170
171/// How an Eval's material came to exist.
172#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
173#[serde(rename_all = "snake_case")]
174pub enum Origin {
175    /// Captured while the evaluation ran.
176    Live,
177    /// Rebuilt afterwards from logs.
178    Reconstructed,
179    /// Converted from another format.
180    Imported,
181}
182
183/// One run in a `run_set`.
184#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
185#[serde(deny_unknown_fields)]
186pub struct Run {
187    /// Run identifier, unique within the Eval.
188    pub run_id: String,
189    /// When the run started.
190    #[serde(default, skip_serializing_if = "Option::is_none")]
191    pub started_at: Option<DateTime<Utc>>,
192    /// When the run ended.
193    #[serde(default, skip_serializing_if = "Option::is_none")]
194    pub ended_at: Option<DateTime<Utc>>,
195    /// How the run ended.
196    pub outcome: Outcome,
197    /// `attachments[].path` of the model calls made during the run.
198    #[serde(default, skip_serializing_if = "Option::is_none")]
199    pub calls: Option<String>,
200    /// `attachments[].path` entries of artifacts the run produced.
201    #[serde(default, skip_serializing_if = "Vec::is_empty")]
202    pub artifacts: Vec<String>,
203}
204
205/// How a run ended.
206#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
207#[serde(rename_all = "snake_case")]
208pub enum Outcome {
209    /// The run completed and passed.
210    Pass,
211    /// The run completed and failed.
212    Fail,
213    /// The run errored before completion.
214    Error,
215    /// The run was skipped.
216    Skipped,
217}