evalhub_schema/run.rs
1//! The Run: one execution inside an Eval, as a row of the record.
2//!
3//! A run is what a harness did once: one attempt at one task (or at a whole
4//! session of tasks, if that is the harness's unit), under stated
5//! conditions, with its log, what it produced, and what was measured
6//! directly from it. Runs belong to the Eval *record*, not to a header
7//! version (see [`crate::eval`]): a run is written on its own, keyed by
8//! `(record, run_id)`, and writing one does not append a header version.
9//!
10//! # Shape
11//!
12//! ```text
13//! Run
14//! ├── run_id URL segment, unique within the Eval record
15//! ├── status ok | error | skipped
16//! ├── error { kind, message?, log? } exactly when status = error
17//! ├── skip_reason free text exactly when status = skipped
18//! ├── started_at / ended_at
19//! ├── model / task / harness / generation / trial / env
20//! │ the conditions of this run; the header's facets are defaults
21//! ├── calls attachments[].path of the model calls
22//! ├── artifacts[] attachments[].path of what the run produced
23//! ├── attachments[] { path, sha256, size, media_type } this run's own files
24//! ├── metrics { "{ns}/{name}": number } measured directly from the run
25//! ├── ext { "{ns}/{name}": { ... } } producer-private, namespaced
26//! └── meta any JSON never interpreted
27//! ```
28//!
29//! An example:
30//!
31//! ```json
32//! {
33//! "run_id": "r1",
34//! "status": "ok",
35//! "started_at": "2026-09-20T10:00:00Z", "ended_at": "2026-09-20T10:02:11Z",
36//! "model": {"id": "qwen3.6-32b"},
37//! "generation": {"temperature": 0.0},
38//! "calls": "calls/r1.jsonl",
39//! "artifacts": ["artifacts/r1/diff.patch"],
40//! "attachments": [
41//! {"path": "calls/r1.jsonl", "sha256": "…", "size": 90210, "media_type": "application/x-ndjson"},
42//! {"path": "artifacts/r1/diff.patch", "sha256": "…", "size": 3311, "media_type": "text/x-diff"}
43//! ],
44//! "metrics": {"core/tokens_out": 1834, "core/duration_ms": 131000},
45//! "ext": {},
46//! "meta": {"worker": "gpu-3"}
47//! }
48//! ```
49//!
50//! # `run_id`
51//!
52//! `run_id` is the run's address inside its Eval and appears as one path
53//! segment of a URL. It is non-empty, contains no `/`, and is at most 200
54//! bytes of UTF-8 (`run_id_invalid` otherwise). Nothing else is imposed:
55//! every `run_id` a 0.1.x producer wrote into `runs[]` meets these rules.
56//! Writing a run whose `run_id` already exists in the record overwrites it;
57//! repeated attempts at the same task are separate runs with separate ids.
58//!
59//! # `status`, `error`, `skip_reason`
60//!
61//! `status` is the state of the execution and nothing more:
62//!
63//! - `ok`: the run went to completion.
64//! - `error`: the run was executed and failed along the way (budget
65//! exhausted, timeout, crash, an API error). `error` is required and
66//! says how.
67//! - `skipped`: the producer deliberately did not execute it (left out by a
68//! limit, already done when a session resumed). `skip_reason` is
69//! required.
70//!
71//! `error` is present exactly when `status` is `error`, and `skip_reason`
72//! exactly when `status` is `skipped` (`run_status_detail` otherwise).
73//!
74//! A run carries no verdict. Whether it passed depends on a grader and a
75//! threshold, and the same run can be graded several ways, so pass and fail
76//! are the Card's claim, in [`crate::card::Card::run_results`].
77//!
78//! `error.kind` is a short slug the producer chooses (`budget_exhausted`,
79//! `timeout`, `crash`), of the form `[a-z0-9][a-z0-9._-]*`. It is not an
80//! enum: harnesses fail in more ways than a closed list can name, and the
81//! value is for grouping and filtering, not for the hub to act on. The
82//! value `unrecorded` is reserved in practice for runs converted from
83//! `evalhub.eval/1.0`, whose `outcome` said that a run errored or was
84//! skipped but not why; the conversion writes `unrecorded` as the
85//! `error.kind` or the `skip_reason` rather than guessing.
86//!
87//! # Facets
88//!
89//! A run has the six facets of an Eval header, with the same types
90//! ([`crate::facet`]). They are the conditions this run was measured
91//! under. A facet the run omits is filled from the header's default when
92//! the run is written, and stored with the run, so a later change to the
93//! header does not change the conditions of a run already written. The
94//! fingerprint of each facet is computed per run.
95//!
96//! # Attachments, `calls`, `artifacts`
97//!
98//! A run has its own `attachments[]`, of the same type as a record's
99//! ([`crate::common::Attachment`]). `calls`, `artifacts[]` and
100//! `error.log` name a `path` in *this run's* `attachments[]`, never the
101//! header's: a run's files change with the run, and pointing into the
102//! header would make adding a run append a header version. The same checks
103//! apply as on a record (`attachment_ref_unknown`,
104//! `attachment_path_invalid`, `attachment_missing`), per run.
105//!
106//! # What the hub interprets and what it does not
107//!
108//! | Key | The hub |
109//! | --------------------------- | ---------------------------------------------------------------- |
110//! | `run_id` | checks the form; addresses the run by it |
111//! | `status`, `error.kind` | checks the status/detail pairing; filters and counts by them |
112//! | facets | fills defaults from the header; fingerprints them |
113//! | `metrics` | checks each key is `{ns}/{name}`; indexes the numbers |
114//! | `attachments[]` and pointers| checks references resolve and the objects are uploaded |
115//! | `ext` | stores; searchable, typed once the namespace registers a schema |
116//! | `error.message`, `skip_reason`, `started_at`, `ended_at` | stores and returns |
117//! | `meta` | stores and returns; never validates, indexes or queries it |
118//!
119//! `metrics` holds values measured directly from the run: token counts,
120//! durations, cost, an exit code, the length of an output, basic statistics
121//! within the run. A key is a registry id of the same form as a Card's
122//! `results[].metric` (`metric_id_invalid` otherwise); a metric the
123//! registry does not know is accepted. A value that depends on how the run
124//! is graded does not belong here; it is the Card's, in `run_results`,
125//! which may name the metrics it was derived from (`from`).
126//!
127//! `meta` is the one place the hub never looks into. It is for whatever the
128//! producer wants kept with the run and returned verbatim, and it is where
129//! the 1.0 conversion keeps the values it had to drop (the old `outcome`).
130//! `ext` is different: it is namespaced, and a namespace can register a
131//! schema to make its keys typed and queryable.
132//!
133//! As with a record, the run is the producer's claim and the hub's facts
134//! (its content hash, its fingerprints, whether it was archived) are
135//! returned beside it, not inside it.
136
137use std::collections::BTreeMap;
138
139use chrono::{DateTime, Utc};
140use schemars::JsonSchema;
141use serde::{Deserialize, Serialize};
142
143use crate::common::{Attachment, Ext};
144use crate::facet::{Env, Generation, Harness, Model, Task, Trial};
145
146/// One run of an Eval: one execution, its conditions, its files and what
147/// was measured directly from it. A run records what happened, never
148/// whether it passed; the verdict is a Card's `run_results`.
149#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
150#[serde(deny_unknown_fields)]
151pub struct Run {
152 /// Run identifier, unique within the Eval record. One URL segment:
153 /// non-empty, no `/`, at most 200 bytes.
154 pub run_id: String,
155 /// The state of the execution: `ok`, `error` or `skipped`.
156 pub status: RunStatus,
157 /// How the run failed. Present exactly when `status` is `error`.
158 #[serde(default, skip_serializing_if = "Option::is_none")]
159 pub error: Option<RunError>,
160 /// Why the producer did not execute the run. Present exactly when
161 /// `status` is `skipped`.
162 #[serde(default, skip_serializing_if = "Option::is_none")]
163 pub skip_reason: Option<String>,
164 /// When the run started.
165 #[serde(default, skip_serializing_if = "Option::is_none")]
166 pub started_at: Option<DateTime<Utc>>,
167 /// When the run ended.
168 #[serde(default, skip_serializing_if = "Option::is_none")]
169 pub ended_at: Option<DateTime<Utc>>,
170 /// The model this run used. Filled from the Eval header when omitted.
171 #[serde(default, skip_serializing_if = "Option::is_none")]
172 pub model: Option<Model>,
173 /// The task this run attempted. Filled from the Eval header when omitted.
174 #[serde(default, skip_serializing_if = "Option::is_none")]
175 pub task: Option<Task>,
176 /// The harness that drove this run. Filled from the Eval header when omitted.
177 #[serde(default, skip_serializing_if = "Option::is_none")]
178 pub harness: Option<Harness>,
179 /// The sampling parameters of this run. Filled from the Eval header when omitted.
180 #[serde(default, skip_serializing_if = "Option::is_none")]
181 pub generation: Option<Generation>,
182 /// How this run's attempt was made. Filled from the Eval header when omitted.
183 #[serde(default, skip_serializing_if = "Option::is_none")]
184 pub trial: Option<Trial>,
185 /// The environment this run executed in. Filled from the Eval header when omitted.
186 #[serde(default, skip_serializing_if = "Option::is_none")]
187 pub env: Option<Env>,
188 /// This run's `attachments[].path` holding the model calls made during the run.
189 #[serde(default, skip_serializing_if = "Option::is_none")]
190 pub calls: Option<String>,
191 /// This run's `attachments[].path` entries of artifacts the run produced.
192 #[serde(default, skip_serializing_if = "Vec::is_empty")]
193 pub artifacts: Vec<String>,
194 /// Files this run refers to. `calls`, `artifacts[]` and `error.log` point here.
195 #[serde(default, skip_serializing_if = "Vec::is_empty")]
196 pub attachments: Vec<Attachment>,
197 /// Values measured directly from the run, keyed by metric registry id
198 /// `{ns}/{name}` (for example `core/tokens_out`). Grader-dependent
199 /// values belong to a Card's `run_results`, not here.
200 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
201 pub metrics: BTreeMap<String, f64>,
202 /// Producer-private extensions keyed by namespace.
203 #[serde(default)]
204 pub ext: Ext,
205 /// Free-form JSON kept with the run and returned verbatim. The hub never
206 /// validates, indexes or queries it.
207 #[serde(default, skip_serializing_if = "Option::is_none")]
208 pub meta: Option<serde_json::Value>,
209}
210
211/// The state of a run's execution. Not a verdict: pass and fail are a Card's.
212#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
213#[serde(rename_all = "snake_case")]
214pub enum RunStatus {
215 /// The run went to completion.
216 Ok,
217 /// The run was executed and failed along the way; `error` says how.
218 Error,
219 /// The producer deliberately did not execute the run; `skip_reason` says why.
220 Skipped,
221}
222
223/// How a run failed.
224#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
225#[serde(deny_unknown_fields)]
226pub struct RunError {
227 /// Short slug naming the kind of failure, `[a-z0-9][a-z0-9._-]*`
228 /// (for example `timeout`, `budget_exhausted`, `crash`). Chosen by the
229 /// producer; not a closed list. `unrecorded` means the kind is not known.
230 pub kind: String,
231 /// What went wrong, for a human.
232 #[serde(default, skip_serializing_if = "Option::is_none")]
233 pub message: Option<String>,
234 /// This run's `attachments[].path` holding the failure's log (stderr, a trace).
235 #[serde(default, skip_serializing_if = "Option::is_none")]
236 pub log: Option<String>,
237}