Skip to main content

evalhub_core/
eval.rs

1//! The two Eval schema arms, and the one 1.0 → 2.0 conversion.
2//!
3//! Release 0.2.0 accepts two majors of the Eval record schema (see
4//! "Versioning of the schema itself" in `evalhub_schema`):
5//!
6//! ```text
7//! evalhub.eval/1.0   header + runs[] in one body          (accepted by 0.2.0 only)
8//! evalhub.eval/2.0   header only; runs are rows of the record, written apart
9//! ```
10//!
11//! [`declared_schema`] says which arm a body declares, so that the
12//! validator, the store and the server agree on it without each parsing
13//! `schema` for themselves. [`split_v1`] turns a 1.0 body into what a 2.0
14//! producer would have sent: a header and one run per `runs[]` element. It
15//! is the only implementation of that conversion; the ingest of a 1.0 body
16//! and the one-shot data migration of stored 1.0 versions both call it, so
17//! a run converted at ingest and a run converted by the migration are the
18//! same run, byte for byte, and have the same content hash.
19//!
20//! # The conversion
21//!
22//! ```text
23//! 1.0 body ──▶ header: body − runs, schema := "evalhub.eval/2.0"
24//!          └─▶ runs[i] ──▶ run: run_id, status (+ error / skip_reason),
25//!                               started_at, ended_at, calls, artifacts   (verbatim)
26//!                               attachments   the header entries calls/artifacts name
27//!                               meta          { "v0.2.0_migration": { "outcome": … } }
28//!                               facets        materialised from the header
29//! ```
30//!
31//! Nothing else in the header changes: its `attachments[]` keep the
32//! entries the runs point at (they are copied, not moved), so the header
33//! is exactly the 1.0 body minus `runs` under the 2.0 identifier, and its
34//! content hash is the hash of that header posted as 2.0.
35//!
36//! `outcome` mixed the execution state with a verdict; 2.0 keeps only the
37//! state:
38//!
39//! | 1.0 `outcome` | 2.0 `status` | detail                                  |
40//! | ------------- | ------------ | --------------------------------------- |
41//! | `pass`        | `ok`         |                                         |
42//! | `fail`        | `ok`         |                                         |
43//! | `error`       | `error`      | `error: { "kind": "unrecorded" }`       |
44//! | `skipped`     | `skipped`    | `skip_reason: "unrecorded"`             |
45//!
46//! A 1.0 run says that it errored or was skipped but never why, so the
47//! conversion writes `unrecorded` rather than a guess. The dropped
48//! `outcome` is kept verbatim in the run's `meta`, under
49//! [`V1_MIGRATION_META_KEY`] (`v0.2.0_migration`), so the verdict a 0.1.x
50//! producer wrote is not lost; it is simply no longer the hub's to
51//! interpret. A later release that has to drop a meaning again keeps it the
52//! same way, under its own `v{version}_migration` key.
53//!
54//! The run's facets are the header's, copied onto the run
55//! ([`crate::run::materialise`]); `split_v1` owns materialisation on this
56//! path, so the run write path's own `materialise`, which is idempotent,
57//! finds nothing left to copy.
58//!
59//! A 1.0 run's `calls` and `artifacts[]` named paths in the *body's*
60//! `attachments[]`. A 2.0 run points only into its own, so each header
61//! entry a run names is copied into that run's `attachments[]`, in the
62//! header's order, once.
63
64use serde_json::{Map, Value};
65
66use evalhub_schema::EVAL_SCHEMA;
67
68use crate::run::materialise;
69
70/// The `meta` key under which the 1.0 → 2.0 conversion keeps what it
71/// dropped: `{ "v0.2.0_migration": { "outcome": … } }`.
72pub const V1_MIGRATION_META_KEY: &str = "v0.2.0_migration";
73
74/// The value the conversion writes where 1.0 recorded no reason: the
75/// `error.kind` of an `error` run, the `skip_reason` of a `skipped` one.
76pub const UNRECORDED: &str = "unrecorded";
77
78/// The `evalhub.eval/1.0` identifier.
79const EVAL_SCHEMA_1_0: &str = "evalhub.eval/1.0";
80
81/// Which major of the Eval schema a body declares.
82#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
83pub enum EvalSchema {
84    /// `evalhub.eval/1.0`: `runs[]` in the body. Accepted by release 0.2.0
85    /// only; converted by [`split_v1`].
86    V1,
87    /// `evalhub.eval/2.0`: the header alone.
88    V2,
89}
90
91impl EvalSchema {
92    /// The identifier a body declares for this arm.
93    pub const fn id(self) -> &'static str {
94        match self {
95            EvalSchema::V1 => EVAL_SCHEMA_1_0,
96            EvalSchema::V2 => EVAL_SCHEMA,
97        }
98    }
99}
100
101/// The Eval arm `body` declares in its `schema` key, or `None` when the key
102/// is absent, not a string, or not an accepted Eval identifier.
103///
104/// This reads one key and checks nothing else; a body that declares an arm
105/// can still fail that arm's validation. `None` means the body is not an
106/// Eval the hub accepts, and [`crate::validate::validate`] reports why.
107pub fn declared_schema(body: &Value) -> Option<EvalSchema> {
108    match body.get("schema").and_then(Value::as_str)? {
109        EVAL_SCHEMA => Some(EvalSchema::V2),
110        EVAL_SCHEMA_1_0 => Some(EvalSchema::V1),
111        _ => None,
112    }
113}
114
115/// A 1.0 body split into its 2.0 parts. See [`split_v1`].
116#[derive(Debug, Clone, PartialEq)]
117pub struct SplitV1 {
118    /// The 2.0 header: the body without `runs`, `schema` set to
119    /// `evalhub.eval/2.0`.
120    pub header: Value,
121    /// One `(run_id, run)` per distinct `run_id`, in the order each id first
122    /// appears in `runs[]`; for a repeated id, the run is the *last*
123    /// element carrying it. Each run is a 2.0 run body, materialised.
124    pub runs: Vec<(String, Value)>,
125    /// Every `run_id` that appeared more than once in `runs[]`, once each,
126    /// in the order of first appearance. Empty for a body without
127    /// duplicates. The conversion does not fail on them (0.1.x accepted
128    /// such bodies, and the migration must convert what is stored); the
129    /// caller decides whether to report, log or audit them.
130    pub duplicate_run_ids: Vec<String>,
131}
132
133/// Split an `evalhub.eval/1.0` body into a 2.0 header and its runs.
134///
135/// Pure: no I/O, the input is not modified, and the same body always gives
136/// the same result. The mapping is in the module doc. Duplicate `run_id`s
137/// do not fail the call: the last element with an id wins, and the ids are
138/// listed in [`SplitV1::duplicate_run_ids`].
139///
140/// Meant for a body [`crate::validate::validate`] accepted as 1.0. On
141/// anything else it still returns, without guessing more than it must: a
142/// body that is not an object is returned as the header with no runs; a
143/// `runs[]` element without a string `run_id` is dropped; an `outcome`
144/// other than the four 1.0 values is treated as `error` (`unrecorded`),
145/// and kept verbatim in `meta` like any other.
146///
147/// Cost: one clone of the body plus one pass over `runs[]` and the header's
148/// `attachments[]` per run.
149pub fn split_v1(body: &Value) -> SplitV1 {
150    let Some(obj) = body.as_object() else {
151        return SplitV1 {
152            header: body.clone(),
153            runs: Vec::new(),
154            duplicate_run_ids: Vec::new(),
155        };
156    };
157
158    let mut header_obj = obj.clone();
159    let runs_v1 = header_obj.remove("runs");
160    header_obj.insert("schema".to_string(), Value::String(EVAL_SCHEMA.to_string()));
161    let header = Value::Object(header_obj);
162
163    let mut runs: Vec<(String, Value)> = Vec::new();
164    let mut duplicate_run_ids: Vec<String> = Vec::new();
165    let elements = runs_v1
166        .as_ref()
167        .and_then(Value::as_array)
168        .map(Vec::as_slice)
169        .unwrap_or_default();
170    for element in elements {
171        let Some(element) = element.as_object() else {
172            continue;
173        };
174        let Some(run_id) = element.get("run_id").and_then(Value::as_str) else {
175            continue;
176        };
177        let run = convert_run(run_id, element, &header);
178        match runs.iter_mut().find(|(id, _)| id == run_id) {
179            Some(slot) => {
180                slot.1 = run;
181                if !duplicate_run_ids.iter().any(|d| d == run_id) {
182                    duplicate_run_ids.push(run_id.to_string());
183                }
184            }
185            None => runs.push((run_id.to_string(), run)),
186        }
187    }
188
189    SplitV1 {
190        header,
191        runs,
192        duplicate_run_ids,
193    }
194}
195
196/// One 1.0 `runs[]` element as a 2.0 run body.
197fn convert_run(run_id: &str, v1: &Map<String, Value>, header: &Value) -> Value {
198    let mut run = Map::new();
199    run.insert("run_id".to_string(), Value::String(run_id.to_string()));
200
201    let outcome = v1.get("outcome");
202    match outcome.and_then(Value::as_str) {
203        Some("pass" | "fail") => {
204            run.insert("status".to_string(), "ok".into());
205        }
206        Some("skipped") => {
207            run.insert("status".to_string(), "skipped".into());
208            run.insert("skip_reason".to_string(), UNRECORDED.into());
209        }
210        // `error`, and anything a validated body cannot carry.
211        _ => {
212            run.insert("status".to_string(), "error".into());
213            let mut error = Map::new();
214            error.insert("kind".to_string(), UNRECORDED.into());
215            run.insert("error".to_string(), Value::Object(error));
216        }
217    }
218
219    for key in ["started_at", "ended_at", "calls", "artifacts"] {
220        if let Some(v) = v1.get(key) {
221            run.insert(key.to_string(), v.clone());
222        }
223    }
224
225    // The header entries this run points at, in header order, once each.
226    let mut named: Vec<&str> = Vec::new();
227    if let Some(c) = v1.get("calls").and_then(Value::as_str) {
228        named.push(c);
229    }
230    if let Some(artifacts) = v1.get("artifacts").and_then(Value::as_array) {
231        named.extend(artifacts.iter().filter_map(Value::as_str));
232    }
233    let mut copied: Vec<&str> = Vec::new();
234    let mut attachments: Vec<Value> = Vec::new();
235    if let Some(entries) = header.get("attachments").and_then(Value::as_array) {
236        for entry in entries {
237            let Some(path) = entry.get("path").and_then(Value::as_str) else {
238                continue;
239            };
240            if named.contains(&path) && !copied.contains(&path) {
241                copied.push(path);
242                attachments.push(entry.clone());
243            }
244        }
245    }
246    if !attachments.is_empty() {
247        run.insert("attachments".to_string(), Value::Array(attachments));
248    }
249
250    if let Some(outcome) = outcome {
251        let mut dropped = Map::new();
252        dropped.insert("outcome".to_string(), outcome.clone());
253        let mut meta = Map::new();
254        meta.insert(V1_MIGRATION_META_KEY.to_string(), Value::Object(dropped));
255        run.insert("meta".to_string(), Value::Object(meta));
256    }
257
258    let mut run = Value::Object(run);
259    materialise(&mut run, header);
260    run
261}