Skip to main content

evalhub_schema/
common.rs

1//! Pieces shared by Card, Eval and Run: producer, attachments, relations, `ext`.
2//!
3//! # Producer
4//!
5//! `producer { name, version }` identifies the software that wrote the
6//! record. It is informational and is not a facet: two harnesses can produce
7//! comparable records, and one harness can produce incomparable ones.
8//!
9//! # Attachments
10//!
11//! ```text
12//! attachments[] { path, sha256, size, media_type }
13//! ```
14//!
15//! An attachment is a file in object storage, addressed by its sha256. The
16//! record refers to it by `path`, a relative, `..`-free, unique-within-record
17//! name that other fields (a Card's `results[].samples_ref`, a run's
18//! `calls`, `artifacts[]` and `error.log`) point at. A run has its own
19//! `attachments[]` and its pointers name paths there, not in the Eval
20//! header's (see `crate::run`). The hub checks that referenced paths
21//! exist in `attachments[]` and that every `sha256` has been uploaded and
22//! confirmed (see `evalhub_store::objects`); it does not open the file.
23//! `media_type` is a hint for display and download, nothing more.
24//!
25//! Content addressing means two records attaching the same file share one
26//! object, and a record cannot be published against an object that does not
27//! exist. It also means the record commits to the exact bytes: a different
28//! file is a different sha256 and therefore a different record.
29//!
30//! # Relations
31//!
32//! ```text
33//! relations[] { type, to, attrs }
34//! ```
35//!
36//! `type` is a registry id (`core/uses_eval`, `core/retry_of`, …). `to` is a
37//! pinned reference, `{ns}/{name}@{seq}`, which the hub resolves to a
38//! `version_id` on ingest; an unresolvable reference is accepted and simply
39//! does not earn the `refs_resolved` badge. Targets outside the hub are
40//! written `external:https://…` or `hf:org/repo@sha` and stored with no
41//! `version_id`. `attrs` is free-form per relation type.
42//!
43//! For `core/uses_eval`, `attrs.runs` is a list of `run_id`s: the runs of
44//! the cited Eval this Card used. It feeds the Card's *used set*: per Eval
45//! record, the union of `attrs.runs` over the Card's `core/uses_eval`
46//! relations resolved to that record, or, when none of them carries
47//! `attrs.runs`, every run of that record that is neither archived nor
48//! deleted when the Card is posted. The used set bounds the Card's
49//! `run_results` and is defined in full in `crate::card` ("The used set").
50//! `attrs.runs` names runs of the *record*, not of the pinned version:
51//! runs are not versioned with the Eval header.
52//!
53//! Relations are edges between *versions*. Publishing a new version of a
54//! Card does not move its edges; a reader who wants "the latest Card that
55//! used this Eval" asks the API with `follow_latest=true`.
56//!
57//! # `ext`
58//!
59//! The one open door. `ext` is a map from namespace (`{ns}/{name}`) to an
60//! arbitrary JSON object. Namespacing is mandatory so that two producers'
61//! extensions cannot collide, and so that a namespace can later register an
62//! `ext_schema` to make its keys typed and indexable. Until it does, `ext`
63//! values can be searched with `eq` and `exists` only.
64
65use std::collections::BTreeMap;
66
67use schemars::JsonSchema;
68use serde::{Deserialize, Serialize};
69
70/// Producer-private extensions, keyed by namespace (`{ns}/{name}`). Each value
71/// is an arbitrary JSON object owned by that namespace.
72pub type Ext = BTreeMap<String, serde_json::Value>;
73
74/// Free-form attributes carried by a relation, keyed by attribute name.
75pub type Attrs = serde_json::Map<String, serde_json::Value>;
76
77/// True when an extension map is empty; used to omit it on output.
78pub(crate) fn ext_is_empty(ext: &Ext) -> bool {
79    ext.is_empty()
80}
81
82/// The software that wrote the record. Informational; not a facet.
83#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
84#[serde(deny_unknown_fields)]
85pub struct Producer {
86    /// Name of the producing software (a harness, a converter, a script).
87    pub name: String,
88    /// Version of the producing software.
89    pub version: String,
90}
91
92/// A file in object storage that the record refers to by `path`.
93#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
94#[serde(deny_unknown_fields)]
95pub struct Attachment {
96    /// Relative, `..`-free name, unique within the record; other fields point at it.
97    pub path: String,
98    /// Lower-case hex sha256 of the file's bytes; the object's address in storage.
99    pub sha256: String,
100    /// Size of the file in bytes.
101    pub size: u64,
102    /// Media type of the file, for display and download only.
103    #[serde(default, skip_serializing_if = "Option::is_none")]
104    pub media_type: Option<String>,
105}
106
107/// An edge from this version to another record version or an external target.
108#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
109#[serde(deny_unknown_fields)]
110pub struct Relation {
111    /// Relation type as a registry id, `{ns}/{name}` (for example `core/uses_eval`).
112    #[serde(rename = "type")]
113    pub relation_type: String,
114    /// Target: `{ns}/{name}@{seq}` on the hub, or `external:<url>` / `hf:<org>/<repo>@<sha>` outside it.
115    pub to: String,
116    /// Attributes specific to the relation type (for `uses_eval`, which runs were used).
117    #[serde(default, skip_serializing_if = "Option::is_none")]
118    pub attrs: Option<Attrs>,
119}
120
121/// The producer's statement about what it removed before publishing.
122#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
123#[serde(deny_unknown_fields)]
124pub struct Redaction {
125    /// Whether any redaction was applied.
126    pub applied: bool,
127    /// How it was applied (for example `regex+llm`).
128    #[serde(default, skip_serializing_if = "Option::is_none")]
129    pub method: Option<String>,
130    /// Which fields were redacted, as `attachment path:field` or a record path.
131    #[serde(default, skip_serializing_if = "Vec::is_empty")]
132    pub fields: Vec<String>,
133}