Skip to main content

evalhub_schema/
facet.rs

1//! The seven facets: the parameters that decide comparability.
2//!
3//! A facet groups the parameters that must match for two records to have
4//! been measured "the same way" along one axis. Each facet is fingerprinted
5//! independently (see `evalhub_core::fingerprint`), so the hub can say "same
6//! model, same task, different generation settings" without a rule for what
7//! counts as "the same evaluation" overall. That judgement is left to the
8//! reader; the hub only exposes the axes.
9//!
10//! | Facet        | Core keys                                                                                   | Recorded, not fingerprinted |
11//! | ------------ | ------------------------------------------------------------------------------------------- | --------------------------- |
12//! | `model`      | `id` (required), `provider`, `revision`, `endpoint`, `engine{name,version}`, `quantization{method,dtype}`, `availability` | `context_window` |
13//! | `task`       | `id` (required), `version`, `revision`, `split`, `config`, `n`, `seed`, `shuffled`, `source_data`, `sample_ids_sha256` | — |
14//! | `harness`    | `name` (required), `version` (required), `config_sha256`, `prompt_template_sha256`, `system_prompt_sha256`, `few_shot` | — |
15//! | `generation` | `temperature`, `top_p`, `top_k`, `max_tokens`, `seed`, `n`, `stop[]`, `reasoning{effort,budget_tokens}` | — |
16//! | `trial`      | `k`, `pass_at_k`, `max_turns`, `timeout_ms`, `retries`, `sandbox{image,digest}`             | — |
17//! | `grading`    | `graders[]{name,kind}`, `judge{id,provider,revision}`, `rubric_sha256`, `aggregation`, `threshold` | — |
18//! | `env`        | `git{origin,commit,dirty}`, `packages[]{name,version}`                                      | `os`, `hardware` |
19//!
20//! # The admission rule for core keys
21//!
22//! A key goes into a facet's core only if it meets both tests:
23//!
24//! 1. At least two of the major harnesses or schemas (EEE, Inspect, the
25//!    Hugging Face model-index, the OpenAI generation parameters) have a key
26//!    with the same meaning.
27//! 2. The key is used by a fingerprint or a badge.
28//!
29//! Anything else goes in that facet's `ext`. The first test keeps the core
30//! from becoming one harness's private vocabulary; the second keeps it from
31//! collecting keys that are merely interesting. Interoperability is by
32//! converter, not by absorbing every format's fields.
33//!
34//! # `x-fingerprint: false`
35//!
36//! A key that is recorded but must not influence the fingerprint is marked
37//! in the generated schema with `x-fingerprint: false`, written on the Rust
38//! field with `#[schemars(extend("x-fingerprint" = false))]`. `context_window`
39//! is the canonical example: it describes the model but does not change what
40//! was measured. The fingerprint code reads this annotation from the schema;
41//! there is no second list to keep in sync.
42//!
43//! # Facet `ext`
44//!
45//! Each facet has its own `ext` map, keyed by `{ns}/{name}`, for parameters
46//! that fail the admission rule. By default `ext` is excluded from the
47//! fingerprint. A namespace that wants its extension to participate registers
48//! an `ext_schema` in the registry with `fingerprint: true`; from then on the
49//! hub types those paths, indexes them, and folds them into the facet's
50//! fingerprint.
51//!
52//! # Absent versus `null` in these types
53//!
54//! Every optional key below is an `Option<T>`; `null` and an absent key both
55//! deserialise to `None` when a record is read *through these types*. The
56//! hub does not lose the distinction: it validates and fingerprints the
57//! record as a JSON value (see `evalhub_core`), where `null` and absence are
58//! different, and stores the canonical bytes. The generated schema allows
59//! `null` for every optional key.
60
61use schemars::JsonSchema;
62use serde::{Deserialize, Serialize};
63
64use crate::common::{Ext, ext_is_empty};
65
66/// The model that was evaluated.
67#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
68#[serde(deny_unknown_fields)]
69pub struct Model {
70    /// Model identifier as the producer names it (for example `qwen3.6-32b`).
71    pub id: String,
72    /// Who served the model (`openai`, `anthropic`, `local`, …).
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    pub provider: Option<String>,
75    /// Exact revision: a commit, a checkpoint tag, or a provider snapshot date.
76    #[serde(default, skip_serializing_if = "Option::is_none")]
77    pub revision: Option<String>,
78    /// Endpoint the model was reached through, when that changes the result.
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub endpoint: Option<String>,
81    /// Inference engine that ran the model.
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub engine: Option<Engine>,
84    /// Quantisation applied to the weights.
85    #[serde(default, skip_serializing_if = "Option::is_none")]
86    pub quantization: Option<Quantization>,
87    /// Availability of the weights (`open`, `api`, `private`, …).
88    #[serde(default, skip_serializing_if = "Option::is_none")]
89    pub availability: Option<String>,
90    /// Context window in tokens. Recorded; does not affect the fingerprint.
91    #[serde(default, skip_serializing_if = "Option::is_none")]
92    #[schemars(extend("x-fingerprint" = false))]
93    pub context_window: Option<u64>,
94    /// Extensions keyed by namespace; not fingerprinted unless registered.
95    #[serde(default, skip_serializing_if = "ext_is_empty")]
96    pub ext: Ext,
97}
98
99/// Inference engine name and version.
100#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
101#[serde(deny_unknown_fields)]
102pub struct Engine {
103    /// Engine name (`vllm`, `llama.cpp`, `transformers`, …).
104    pub name: String,
105    /// Engine version.
106    #[serde(default, skip_serializing_if = "Option::is_none")]
107    pub version: Option<String>,
108}
109
110/// Weight quantisation.
111#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
112#[serde(deny_unknown_fields)]
113pub struct Quantization {
114    /// Method (`awq`, `gptq`, `gguf`, …).
115    pub method: String,
116    /// Storage type (`int4`, `int8`, `fp8`, …).
117    #[serde(default, skip_serializing_if = "Option::is_none")]
118    pub dtype: Option<String>,
119}
120
121/// The task or benchmark that was run.
122#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
123#[serde(deny_unknown_fields)]
124pub struct Task {
125    /// Task identifier as the producer names it (for example `single2`).
126    pub id: String,
127    /// Task version, as published by the task's author.
128    #[serde(default, skip_serializing_if = "Option::is_none")]
129    pub version: Option<String>,
130    /// Exact revision of the task material (a commit or dataset revision).
131    #[serde(default, skip_serializing_if = "Option::is_none")]
132    pub revision: Option<String>,
133    /// Split that was evaluated (`test`, `validation`, …).
134    #[serde(default, skip_serializing_if = "Option::is_none")]
135    pub split: Option<String>,
136    /// Named task configuration, when the task has more than one.
137    #[serde(default, skip_serializing_if = "Option::is_none")]
138    pub config: Option<String>,
139    /// Number of items evaluated.
140    #[serde(default, skip_serializing_if = "Option::is_none")]
141    pub n: Option<u64>,
142    /// Seed used to select or order items.
143    #[serde(default, skip_serializing_if = "Option::is_none")]
144    pub seed: Option<i64>,
145    /// Whether items were shuffled before evaluation.
146    #[serde(default, skip_serializing_if = "Option::is_none")]
147    pub shuffled: Option<bool>,
148    /// Where the task data came from (a URL or dataset id).
149    #[serde(default, skip_serializing_if = "Option::is_none")]
150    pub source_data: Option<String>,
151    /// sha256 over the sorted list of sample ids that were evaluated.
152    #[serde(default, skip_serializing_if = "Option::is_none")]
153    pub sample_ids_sha256: Option<String>,
154    /// Extensions keyed by namespace; not fingerprinted unless registered.
155    #[serde(default, skip_serializing_if = "ext_is_empty")]
156    pub ext: Ext,
157}
158
159/// The harness that drove the evaluation.
160#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
161#[serde(deny_unknown_fields)]
162pub struct Harness {
163    /// Harness name.
164    pub name: String,
165    /// Harness version.
166    pub version: String,
167    /// sha256 of the harness configuration that was in effect.
168    #[serde(default, skip_serializing_if = "Option::is_none")]
169    pub config_sha256: Option<String>,
170    /// sha256 of the prompt template.
171    #[serde(default, skip_serializing_if = "Option::is_none")]
172    pub prompt_template_sha256: Option<String>,
173    /// sha256 of the system prompt.
174    #[serde(default, skip_serializing_if = "Option::is_none")]
175    pub system_prompt_sha256: Option<String>,
176    /// Number of few-shot examples in the prompt.
177    #[serde(default, skip_serializing_if = "Option::is_none")]
178    pub few_shot: Option<u32>,
179    /// Extensions keyed by namespace; not fingerprinted unless registered.
180    #[serde(default, skip_serializing_if = "ext_is_empty")]
181    pub ext: Ext,
182}
183
184/// Sampling parameters.
185#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
186#[serde(deny_unknown_fields)]
187pub struct Generation {
188    /// Sampling temperature.
189    #[serde(default, skip_serializing_if = "Option::is_none")]
190    pub temperature: Option<f64>,
191    /// Nucleus sampling threshold.
192    #[serde(default, skip_serializing_if = "Option::is_none")]
193    pub top_p: Option<f64>,
194    /// Top-k sampling cutoff.
195    #[serde(default, skip_serializing_if = "Option::is_none")]
196    pub top_k: Option<u32>,
197    /// Maximum tokens generated per response.
198    #[serde(default, skip_serializing_if = "Option::is_none")]
199    pub max_tokens: Option<u64>,
200    /// Sampling seed.
201    #[serde(default, skip_serializing_if = "Option::is_none")]
202    pub seed: Option<i64>,
203    /// Number of completions requested per prompt.
204    #[serde(default, skip_serializing_if = "Option::is_none")]
205    pub n: Option<u32>,
206    /// Stop sequences.
207    #[serde(default, skip_serializing_if = "Vec::is_empty")]
208    pub stop: Vec<String>,
209    /// Reasoning (extended thinking) settings.
210    #[serde(default, skip_serializing_if = "Option::is_none")]
211    pub reasoning: Option<Reasoning>,
212    /// Extensions keyed by namespace; not fingerprinted unless registered.
213    #[serde(default, skip_serializing_if = "ext_is_empty")]
214    pub ext: Ext,
215}
216
217/// Reasoning settings passed to the model.
218#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
219#[serde(deny_unknown_fields)]
220pub struct Reasoning {
221    /// Provider-defined effort level (`low`, `medium`, `high`, …).
222    #[serde(default, skip_serializing_if = "Option::is_none")]
223    pub effort: Option<String>,
224    /// Token budget for reasoning.
225    #[serde(default, skip_serializing_if = "Option::is_none")]
226    pub budget_tokens: Option<u64>,
227}
228
229/// How each item was attempted.
230#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
231#[serde(deny_unknown_fields)]
232pub struct Trial {
233    /// Attempts per item.
234    #[serde(default, skip_serializing_if = "Option::is_none")]
235    pub k: Option<u32>,
236    /// Whether the score is pass@k over the attempts.
237    #[serde(default, skip_serializing_if = "Option::is_none")]
238    pub pass_at_k: Option<bool>,
239    /// Maximum agent turns per attempt.
240    #[serde(default, skip_serializing_if = "Option::is_none")]
241    pub max_turns: Option<u32>,
242    /// Timeout per attempt in milliseconds.
243    #[serde(default, skip_serializing_if = "Option::is_none")]
244    pub timeout_ms: Option<u64>,
245    /// Retries on error per attempt.
246    #[serde(default, skip_serializing_if = "Option::is_none")]
247    pub retries: Option<u32>,
248    /// Sandbox the attempt ran in.
249    #[serde(default, skip_serializing_if = "Option::is_none")]
250    pub sandbox: Option<Sandbox>,
251    /// Extensions keyed by namespace; not fingerprinted unless registered.
252    #[serde(default, skip_serializing_if = "ext_is_empty")]
253    pub ext: Ext,
254}
255
256/// Container image the attempts ran in.
257#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
258#[serde(deny_unknown_fields)]
259pub struct Sandbox {
260    /// Image reference (for example `ghcr.io/org/image:tag`).
261    pub image: String,
262    /// Content digest of the image (`sha256:…`).
263    #[serde(default, skip_serializing_if = "Option::is_none")]
264    pub digest: Option<String>,
265}
266
267/// How responses were scored. Cards only.
268#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
269#[serde(deny_unknown_fields)]
270pub struct Grading {
271    /// Graders that produced the scores.
272    #[serde(default, skip_serializing_if = "Vec::is_empty")]
273    pub graders: Vec<Grader>,
274    /// Judge model, when a grader is a model.
275    #[serde(default, skip_serializing_if = "Option::is_none")]
276    pub judge: Option<Judge>,
277    /// sha256 of the rubric given to the graders.
278    #[serde(default, skip_serializing_if = "Option::is_none")]
279    pub rubric_sha256: Option<String>,
280    /// How several graders' verdicts were combined (`majority`, `all`, …).
281    #[serde(default, skip_serializing_if = "Option::is_none")]
282    pub aggregation: Option<String>,
283    /// Threshold a score had to reach to count as a pass.
284    #[serde(default, skip_serializing_if = "Option::is_none")]
285    pub threshold: Option<f64>,
286    /// Extensions keyed by namespace; not fingerprinted unless registered.
287    #[serde(default, skip_serializing_if = "ext_is_empty")]
288    pub ext: Ext,
289}
290
291/// One grader.
292#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
293#[serde(deny_unknown_fields)]
294pub struct Grader {
295    /// Grader name (`exact_match`, `llm_judge`, …).
296    pub name: String,
297    /// Grader kind (`deterministic`, `model`, `human`).
298    #[serde(default, skip_serializing_if = "Option::is_none")]
299    pub kind: Option<String>,
300}
301
302/// The judge model used by a model grader.
303#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
304#[serde(deny_unknown_fields)]
305pub struct Judge {
306    /// Judge model identifier.
307    pub id: String,
308    /// Who served the judge.
309    #[serde(default, skip_serializing_if = "Option::is_none")]
310    pub provider: Option<String>,
311    /// Exact revision of the judge.
312    #[serde(default, skip_serializing_if = "Option::is_none")]
313    pub revision: Option<String>,
314}
315
316/// The software environment the evaluation ran in.
317#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, JsonSchema)]
318#[serde(deny_unknown_fields)]
319pub struct Env {
320    /// Git state of the code that ran.
321    #[serde(default, skip_serializing_if = "Option::is_none")]
322    pub git: Option<Git>,
323    /// Pinned packages that matter to the result.
324    #[serde(default, skip_serializing_if = "Vec::is_empty")]
325    pub packages: Vec<Package>,
326    /// Operating system description. Recorded; does not affect the fingerprint.
327    #[serde(default, skip_serializing_if = "Option::is_none")]
328    #[schemars(extend("x-fingerprint" = false))]
329    pub os: Option<String>,
330    /// Hardware description. Recorded; does not affect the fingerprint.
331    #[serde(default, skip_serializing_if = "Option::is_none")]
332    #[schemars(extend("x-fingerprint" = false))]
333    pub hardware: Option<Hardware>,
334    /// Extensions keyed by namespace; not fingerprinted unless registered.
335    #[serde(default, skip_serializing_if = "ext_is_empty")]
336    pub ext: Ext,
337}
338
339/// Git state of the evaluating code.
340#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
341#[serde(deny_unknown_fields)]
342pub struct Git {
343    /// Remote the code came from.
344    #[serde(default, skip_serializing_if = "Option::is_none")]
345    pub origin: Option<String>,
346    /// Commit that ran.
347    pub commit: String,
348    /// Whether the working tree had uncommitted changes.
349    #[serde(default, skip_serializing_if = "Option::is_none")]
350    pub dirty: Option<bool>,
351}
352
353/// A pinned package.
354#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
355#[serde(deny_unknown_fields)]
356pub struct Package {
357    /// Package name.
358    pub name: String,
359    /// Package version.
360    pub version: String,
361}
362
363/// Hardware the evaluation ran on.
364#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
365#[serde(deny_unknown_fields)]
366pub struct Hardware {
367    /// Accelerator model (for example `A100`).
368    #[serde(default, skip_serializing_if = "Option::is_none")]
369    pub gpu: Option<String>,
370    /// Number of accelerators.
371    #[serde(default, skip_serializing_if = "Option::is_none")]
372    pub gpu_count: Option<u32>,
373    /// CPU model.
374    #[serde(default, skip_serializing_if = "Option::is_none")]
375    pub cpu: Option<String>,
376    /// Memory in gigabytes.
377    #[serde(default, skip_serializing_if = "Option::is_none")]
378    pub memory_gb: Option<f64>,
379}