evalhub_schema/facet.rs
1//! The seven facets: the parameters that decide comparability.
2//!
3//! A facet groups the parameters that must match for two records to have
4//! been measured "the same way" along one axis. Each facet is fingerprinted
5//! independently (see `evalhub_core::fingerprint`), so the hub can say "same
6//! model, same task, different generation settings" without a rule for what
7//! counts as "the same evaluation" overall. That judgement is left to the
8//! reader; the hub only exposes the axes.
9//!
10//! | Facet | Core keys | Recorded, not fingerprinted |
11//! | ------------ | ------------------------------------------------------------------------------------------- | --------------------------- |
12//! | `model` | `id` (required), `provider`, `revision`, `endpoint`, `engine{name,version}`, `quantization{method,dtype}`, `availability` | `context_window` |
13//! | `task` | `id` (required), `version`, `revision`, `split`, `config`, `n`, `seed`, `shuffled`, `source_data`, `sample_ids_sha256` | — |
14//! | `harness` | `name` (required), `version` (required), `config_sha256`, `prompt_template_sha256`, `system_prompt_sha256`, `few_shot` | — |
15//! | `generation` | `temperature`, `top_p`, `top_k`, `max_tokens`, `seed`, `n`, `stop[]`, `reasoning{effort,budget_tokens}` | — |
16//! | `trial` | `k`, `pass_at_k`, `max_turns`, `timeout_ms`, `retries`, `sandbox{image,digest}` | — |
17//! | `grading` | `graders[]{name,kind}`, `judge{id,provider,revision}`, `rubric_sha256`, `aggregation`, `threshold` | — |
18//! | `env` | `git{origin,commit,dirty}`, `packages[]{name,version}` | `os`, `hardware` |
19//!
20//! # The admission rule for core keys
21//!
22//! A key goes into a facet's core only if it meets both tests:
23//!
24//! 1. At least two of the major harnesses or schemas (EEE, Inspect, the
25//! Hugging Face model-index, the OpenAI generation parameters) have a key
26//! with the same meaning.
27//! 2. The key is used by a fingerprint or a badge.
28//!
29//! Anything else goes in that facet's `ext`. The first test keeps the core
30//! from becoming one harness's private vocabulary; the second keeps it from
31//! collecting keys that are merely interesting. Interoperability is by
32//! converter, not by absorbing every format's fields.
33//!
34//! # `x-fingerprint: false`
35//!
36//! A key that is recorded but must not influence the fingerprint is marked
37//! in the generated schema with `x-fingerprint: false`, written on the Rust
38//! field with `#[schemars(extend("x-fingerprint" = false))]`. `context_window`
39//! is the canonical example: it describes the model but does not change what
40//! was measured. The fingerprint code reads this annotation from the schema;
41//! there is no second list to keep in sync.
42//!
43//! # Facet `ext`
44//!
45//! Each facet has its own `ext` map, keyed by `{ns}/{name}`, for parameters
46//! that fail the admission rule. By default `ext` is excluded from the
47//! fingerprint. A namespace that wants its extension to participate registers
48//! an `ext_schema` in the registry with `fingerprint: true`; from then on the
49//! hub types those paths, indexes them, and folds them into the facet's
50//! fingerprint.
51//!
52//! # Absent versus `null` in these types
53//!
54//! Every optional key below is an `Option<T>`; `null` and an absent key both
55//! deserialise to `None` when a record is read *through these types*. The
56//! hub does not lose the distinction: it validates and fingerprints the
57//! record as a JSON value (see `evalhub_core`), where `null` and absence are
58//! different, and stores the canonical bytes. The generated schema allows
59//! `null` for every optional key.
60
61use schemars::JsonSchema;
62use serde::{Deserialize, Serialize};
63
64use crate::common::{Ext, ext_is_empty};
65
66/// The model that was evaluated.
67#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
68#[serde(deny_unknown_fields)]
69pub struct Model {
70 /// Model identifier as the producer names it (for example `qwen3.6-32b`).
71 pub id: String,
72 /// Who served the model (`openai`, `anthropic`, `local`, …).
73 #[serde(default, skip_serializing_if = "Option::is_none")]
74 pub provider: Option<String>,
75 /// Exact revision: a commit, a checkpoint tag, or a provider snapshot date.
76 #[serde(default, skip_serializing_if = "Option::is_none")]
77 pub revision: Option<String>,
78 /// Endpoint the model was reached through, when that changes the result.
79 #[serde(default, skip_serializing_if = "Option::is_none")]
80 pub endpoint: Option<String>,
81 /// Inference engine that ran the model.
82 #[serde(default, skip_serializing_if = "Option::is_none")]
83 pub engine: Option<Engine>,
84 /// Quantisation applied to the weights.
85 #[serde(default, skip_serializing_if = "Option::is_none")]
86 pub quantization: Option<Quantization>,
87 /// Availability of the weights (`open`, `api`, `private`, …).
88 #[serde(default, skip_serializing_if = "Option::is_none")]
89 pub availability: Option<String>,
90 /// Context window in tokens. Recorded; does not affect the fingerprint.
91 #[serde(default, skip_serializing_if = "Option::is_none")]
92 #[schemars(extend("x-fingerprint" = false))]
93 pub context_window: Option<u64>,
94 /// Extensions keyed by namespace; not fingerprinted unless registered.
95 #[serde(default, skip_serializing_if = "ext_is_empty")]
96 pub ext: Ext,
97}
98
99/// Inference engine name and version.
100#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
101#[serde(deny_unknown_fields)]
102pub struct Engine {
103 /// Engine name (`vllm`, `llama.cpp`, `transformers`, …).
104 pub name: String,
105 /// Engine version.
106 #[serde(default, skip_serializing_if = "Option::is_none")]
107 pub version: Option<String>,
108}
109
110/// Weight quantisation.
111#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
112#[serde(deny_unknown_fields)]
113pub struct Quantization {
114 /// Method (`awq`, `gptq`, `gguf`, …).
115 pub method: String,
116 /// Storage type (`int4`, `int8`, `fp8`, …).
117 #[serde(default, skip_serializing_if = "Option::is_none")]
118 pub dtype: Option<String>,
119}
120
121/// The task or benchmark that was run.
122#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
123#[serde(deny_unknown_fields)]
124pub struct Task {
125 /// Task identifier as the producer names it (for example `single2`).
126 pub id: String,
127 /// Task version, as published by the task's author.
128 #[serde(default, skip_serializing_if = "Option::is_none")]
129 pub version: Option<String>,
130 /// Exact revision of the task material (a commit or dataset revision).
131 #[serde(default, skip_serializing_if = "Option::is_none")]
132 pub revision: Option<String>,
133 /// Split that was evaluated (`test`, `validation`, …).
134 #[serde(default, skip_serializing_if = "Option::is_none")]
135 pub split: Option<String>,
136 /// Named task configuration, when the task has more than one.
137 #[serde(default, skip_serializing_if = "Option::is_none")]
138 pub config: Option<String>,
139 /// Number of items evaluated.
140 #[serde(default, skip_serializing_if = "Option::is_none")]
141 pub n: Option<u64>,
142 /// Seed used to select or order items.
143 #[serde(default, skip_serializing_if = "Option::is_none")]
144 pub seed: Option<i64>,
145 /// Whether items were shuffled before evaluation.
146 #[serde(default, skip_serializing_if = "Option::is_none")]
147 pub shuffled: Option<bool>,
148 /// Where the task data came from (a URL or dataset id).
149 #[serde(default, skip_serializing_if = "Option::is_none")]
150 pub source_data: Option<String>,
151 /// sha256 over the sorted list of sample ids that were evaluated.
152 #[serde(default, skip_serializing_if = "Option::is_none")]
153 pub sample_ids_sha256: Option<String>,
154 /// Extensions keyed by namespace; not fingerprinted unless registered.
155 #[serde(default, skip_serializing_if = "ext_is_empty")]
156 pub ext: Ext,
157}
158
159/// The harness that drove the evaluation.
160#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
161#[serde(deny_unknown_fields)]
162pub struct Harness {
163 /// Harness name.
164 pub name: String,
165 /// Harness version.
166 pub version: String,
167 /// sha256 of the harness configuration that was in effect.
168 #[serde(default, skip_serializing_if = "Option::is_none")]
169 pub config_sha256: Option<String>,
170 /// sha256 of the prompt template.
171 #[serde(default, skip_serializing_if = "Option::is_none")]
172 pub prompt_template_sha256: Option<String>,
173 /// sha256 of the system prompt.
174 #[serde(default, skip_serializing_if = "Option::is_none")]
175 pub system_prompt_sha256: Option<String>,
176 /// Number of few-shot examples in the prompt.
177 #[serde(default, skip_serializing_if = "Option::is_none")]
178 pub few_shot: Option<u32>,
179 /// Extensions keyed by namespace; not fingerprinted unless registered.
180 #[serde(default, skip_serializing_if = "ext_is_empty")]
181 pub ext: Ext,
182}
183
184/// Sampling parameters.
185#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
186#[serde(deny_unknown_fields)]
187pub struct Generation {
188 /// Sampling temperature.
189 #[serde(default, skip_serializing_if = "Option::is_none")]
190 pub temperature: Option<f64>,
191 /// Nucleus sampling threshold.
192 #[serde(default, skip_serializing_if = "Option::is_none")]
193 pub top_p: Option<f64>,
194 /// Top-k sampling cutoff.
195 #[serde(default, skip_serializing_if = "Option::is_none")]
196 pub top_k: Option<u32>,
197 /// Maximum tokens generated per response.
198 #[serde(default, skip_serializing_if = "Option::is_none")]
199 pub max_tokens: Option<u64>,
200 /// Sampling seed.
201 #[serde(default, skip_serializing_if = "Option::is_none")]
202 pub seed: Option<i64>,
203 /// Number of completions requested per prompt.
204 #[serde(default, skip_serializing_if = "Option::is_none")]
205 pub n: Option<u32>,
206 /// Stop sequences.
207 #[serde(default, skip_serializing_if = "Vec::is_empty")]
208 pub stop: Vec<String>,
209 /// Reasoning (extended thinking) settings.
210 #[serde(default, skip_serializing_if = "Option::is_none")]
211 pub reasoning: Option<Reasoning>,
212 /// Extensions keyed by namespace; not fingerprinted unless registered.
213 #[serde(default, skip_serializing_if = "ext_is_empty")]
214 pub ext: Ext,
215}
216
217/// Reasoning settings passed to the model.
218#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
219#[serde(deny_unknown_fields)]
220pub struct Reasoning {
221 /// Provider-defined effort level (`low`, `medium`, `high`, …).
222 #[serde(default, skip_serializing_if = "Option::is_none")]
223 pub effort: Option<String>,
224 /// Token budget for reasoning.
225 #[serde(default, skip_serializing_if = "Option::is_none")]
226 pub budget_tokens: Option<u64>,
227}
228
229/// How each item was attempted.
230#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
231#[serde(deny_unknown_fields)]
232pub struct Trial {
233 /// Attempts per item.
234 #[serde(default, skip_serializing_if = "Option::is_none")]
235 pub k: Option<u32>,
236 /// Whether the score is pass@k over the attempts.
237 #[serde(default, skip_serializing_if = "Option::is_none")]
238 pub pass_at_k: Option<bool>,
239 /// Maximum agent turns per attempt.
240 #[serde(default, skip_serializing_if = "Option::is_none")]
241 pub max_turns: Option<u32>,
242 /// Timeout per attempt in milliseconds.
243 #[serde(default, skip_serializing_if = "Option::is_none")]
244 pub timeout_ms: Option<u64>,
245 /// Retries on error per attempt.
246 #[serde(default, skip_serializing_if = "Option::is_none")]
247 pub retries: Option<u32>,
248 /// Sandbox the attempt ran in.
249 #[serde(default, skip_serializing_if = "Option::is_none")]
250 pub sandbox: Option<Sandbox>,
251 /// Extensions keyed by namespace; not fingerprinted unless registered.
252 #[serde(default, skip_serializing_if = "ext_is_empty")]
253 pub ext: Ext,
254}
255
256/// Container image the attempts ran in.
257#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
258#[serde(deny_unknown_fields)]
259pub struct Sandbox {
260 /// Image reference (for example `ghcr.io/org/image:tag`).
261 pub image: String,
262 /// Content digest of the image (`sha256:…`).
263 #[serde(default, skip_serializing_if = "Option::is_none")]
264 pub digest: Option<String>,
265}
266
267/// How responses were scored. Cards only.
268#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
269#[serde(deny_unknown_fields)]
270pub struct Grading {
271 /// Graders that produced the scores.
272 #[serde(default, skip_serializing_if = "Vec::is_empty")]
273 pub graders: Vec<Grader>,
274 /// Judge model, when a grader is a model.
275 #[serde(default, skip_serializing_if = "Option::is_none")]
276 pub judge: Option<Judge>,
277 /// sha256 of the rubric given to the graders.
278 #[serde(default, skip_serializing_if = "Option::is_none")]
279 pub rubric_sha256: Option<String>,
280 /// How several graders' verdicts were combined (`majority`, `all`, …).
281 #[serde(default, skip_serializing_if = "Option::is_none")]
282 pub aggregation: Option<String>,
283 /// Threshold a score had to reach to count as a pass.
284 #[serde(default, skip_serializing_if = "Option::is_none")]
285 pub threshold: Option<f64>,
286 /// Extensions keyed by namespace; not fingerprinted unless registered.
287 #[serde(default, skip_serializing_if = "ext_is_empty")]
288 pub ext: Ext,
289}
290
291/// One grader.
292#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
293#[serde(deny_unknown_fields)]
294pub struct Grader {
295 /// Grader name (`exact_match`, `llm_judge`, …).
296 pub name: String,
297 /// Grader kind (`deterministic`, `model`, `human`).
298 #[serde(default, skip_serializing_if = "Option::is_none")]
299 pub kind: Option<String>,
300}
301
302/// The judge model used by a model grader.
303#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
304#[serde(deny_unknown_fields)]
305pub struct Judge {
306 /// Judge model identifier.
307 pub id: String,
308 /// Who served the judge.
309 #[serde(default, skip_serializing_if = "Option::is_none")]
310 pub provider: Option<String>,
311 /// Exact revision of the judge.
312 #[serde(default, skip_serializing_if = "Option::is_none")]
313 pub revision: Option<String>,
314}
315
316/// The software environment the evaluation ran in.
317#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
318#[serde(deny_unknown_fields)]
319pub struct Env {
320 /// Git state of the code that ran.
321 #[serde(default, skip_serializing_if = "Option::is_none")]
322 pub git: Option<Git>,
323 /// Pinned packages that matter to the result.
324 #[serde(default, skip_serializing_if = "Vec::is_empty")]
325 pub packages: Vec<Package>,
326 /// Operating system description. Recorded; does not affect the fingerprint.
327 #[serde(default, skip_serializing_if = "Option::is_none")]
328 #[schemars(extend("x-fingerprint" = false))]
329 pub os: Option<String>,
330 /// Hardware description. Recorded; does not affect the fingerprint.
331 #[serde(default, skip_serializing_if = "Option::is_none")]
332 #[schemars(extend("x-fingerprint" = false))]
333 pub hardware: Option<Hardware>,
334 /// Extensions keyed by namespace; not fingerprinted unless registered.
335 #[serde(default, skip_serializing_if = "ext_is_empty")]
336 pub ext: Ext,
337}
338
339/// Git state of the evaluating code.
340#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
341#[serde(deny_unknown_fields)]
342pub struct Git {
343 /// Remote the code came from.
344 #[serde(default, skip_serializing_if = "Option::is_none")]
345 pub origin: Option<String>,
346 /// Commit that ran.
347 pub commit: String,
348 /// Whether the working tree had uncommitted changes.
349 #[serde(default, skip_serializing_if = "Option::is_none")]
350 pub dirty: Option<bool>,
351}
352
353/// A pinned package.
354#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
355#[serde(deny_unknown_fields)]
356pub struct Package {
357 /// Package name.
358 pub name: String,
359 /// Package version.
360 pub version: String,
361}
362
363/// Hardware the evaluation ran on.
364#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
365#[serde(deny_unknown_fields)]
366pub struct Hardware {
367 /// Accelerator model (for example `A100`).
368 #[serde(default, skip_serializing_if = "Option::is_none")]
369 pub gpu: Option<String>,
370 /// Number of accelerators.
371 #[serde(default, skip_serializing_if = "Option::is_none")]
372 pub gpu_count: Option<u32>,
373 /// CPU model.
374 #[serde(default, skip_serializing_if = "Option::is_none")]
375 pub cpu: Option<String>,
376 /// Memory in gigabytes.
377 #[serde(default, skip_serializing_if = "Option::is_none")]
378 pub memory_gb: Option<f64>,
379}