evalhub_schema/common.rs
1//! Pieces shared by Card, Eval and Run: producer, attachments, relations, `ext`.
2//!
3//! # Producer
4//!
5//! `producer { name, version }` identifies the software that wrote the
6//! record. It is informational and is not a facet: two harnesses can produce
7//! comparable records, and one harness can produce incomparable ones.
8//!
9//! # Attachments
10//!
11//! ```text
12//! attachments[] { path, sha256, size, media_type }
13//! ```
14//!
15//! An attachment is a file in object storage, addressed by its sha256. The
16//! record refers to it by `path`, a relative, `..`-free, unique-within-record
17//! name that other fields (a Card's `results[].samples_ref`, a run's
18//! `calls`, `artifacts[]` and `error.log`) point at. A run has its own
19//! `attachments[]` and its pointers name paths there, not in the Eval
20//! header's (see `crate::run`). The hub checks that referenced paths
21//! exist in `attachments[]` and that every `sha256` has been uploaded and
22//! confirmed (see `evalhub_store::objects`); it does not open the file.
23//! `media_type` is a hint for display and download, nothing more.
24//!
25//! Content addressing means two records attaching the same file share one
26//! object, and a record cannot be published against an object that does not
27//! exist. It also means the record commits to the exact bytes: a different
28//! file is a different sha256 and therefore a different record.
29//!
30//! # Relations
31//!
32//! ```text
33//! relations[] { type, to, attrs }
34//! ```
35//!
36//! `type` is a registry id (`core/uses_eval`, `core/retry_of`, …). `to` is a
37//! pinned reference, `{ns}/{name}@{seq}`, which the hub resolves to a
38//! `version_id` on ingest; an unresolvable reference is accepted and simply
39//! does not earn the `refs_resolved` badge. Targets outside the hub are
40//! written `external:https://…` or `hf:org/repo@sha` and stored with no
41//! `version_id`. `attrs` is free-form per relation type.
42//!
43//! For `core/uses_eval`, `attrs.runs` is a list of `run_id`s: the runs of
44//! the cited Eval this Card used. It feeds the Card's *used set*: per Eval
45//! record, the union of `attrs.runs` over the Card's `core/uses_eval`
46//! relations resolved to that record, or, when none of them carries
47//! `attrs.runs`, every run of that record that is neither archived nor
48//! deleted when the Card is posted. The used set bounds the Card's
49//! `run_results` and is defined in full in `crate::card` ("The used set").
50//! `attrs.runs` names runs of the *record*, not of the pinned version:
51//! runs are not versioned with the Eval header.
52//!
53//! Relations are edges between *versions*. Publishing a new version of a
54//! Card does not move its edges; a reader who wants "the latest Card that
55//! used this Eval" asks the API with `follow_latest=true`.
56//!
57//! # `ext`
58//!
59//! The one open door. `ext` is a map from namespace (`{ns}/{name}`) to an
60//! arbitrary JSON object. Namespacing is mandatory so that two producers'
61//! extensions cannot collide, and so that a namespace can later register an
62//! `ext_schema` to make its keys typed and indexable. Until it does, `ext`
63//! values can be searched with `eq` and `exists` only. The hub checks the
64//! key form on every write, at the top level and inside each facet, and
65//! refuses a key that is not `{ns}/{name}` with `ext_key_invalid`
66//! (`evalhub_core::validate`); the values are not looked into.
67
68use std::collections::BTreeMap;
69
70use schemars::JsonSchema;
71use serde::{Deserialize, Serialize};
72
73/// Producer-private extensions, keyed by namespace (`{ns}/{name}`). Each value
74/// is an arbitrary JSON object owned by that namespace.
75pub type Ext = BTreeMap<String, serde_json::Value>;
76
77/// Free-form attributes carried by a relation, keyed by attribute name.
78pub type Attrs = serde_json::Map<String, serde_json::Value>;
79
80/// True when an extension map is empty; used to omit it on output.
81pub(crate) fn ext_is_empty(ext: &Ext) -> bool {
82 ext.is_empty()
83}
84
85/// The software that wrote the record. Informational; not a facet.
86#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
87#[serde(deny_unknown_fields)]
88pub struct Producer {
89 /// Name of the producing software (a harness, a converter, a script).
90 pub name: String,
91 /// Version of the producing software.
92 pub version: String,
93}
94
95/// A file in object storage that the record refers to by `path`.
96#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
97#[serde(deny_unknown_fields)]
98pub struct Attachment {
99 /// Relative, `..`-free name, unique within the record; other fields point at it.
100 pub path: String,
101 /// Lower-case hex sha256 of the file's bytes; the object's address in storage.
102 pub sha256: String,
103 /// Size of the file in bytes.
104 pub size: u64,
105 /// Media type of the file, for display and download only.
106 #[serde(default, skip_serializing_if = "Option::is_none")]
107 pub media_type: Option<String>,
108}
109
110/// An edge from this version to another record version or an external target.
111#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
112#[serde(deny_unknown_fields)]
113pub struct Relation {
114 /// Relation type as a registry id, `{ns}/{name}` (for example `core/uses_eval`).
115 #[serde(rename = "type")]
116 pub relation_type: String,
117 /// Target: `{ns}/{name}@{seq}` on the hub, or `external:<url>` / `hf:<org>/<repo>@<sha>` outside it.
118 pub to: String,
119 /// Attributes specific to the relation type (for `uses_eval`, which runs were used).
120 #[serde(default, skip_serializing_if = "Option::is_none")]
121 pub attrs: Option<Attrs>,
122}
123
124/// The producer's statement about what it removed before publishing.
125#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
126#[serde(deny_unknown_fields)]
127pub struct Redaction {
128 /// Whether any redaction was applied.
129 pub applied: bool,
130 /// How it was applied (for example `regex+llm`).
131 #[serde(default, skip_serializing_if = "Option::is_none")]
132 pub method: Option<String>,
133 /// Which fields were redacted, as `attachment path:field` or a record path.
134 #[serde(default, skip_serializing_if = "Vec::is_empty")]
135 pub fields: Vec<String>,
136}