rustev_eval/
dataset.rs

1//! Dataset manifests, `rustev.dataset/1` (spec 004, 3.5.1).
2//!
3//! A manifest names each case once, with its source and snapshot identity,
4//! one split, a label or an explicit missing label, and its subgroups. Its
5//! identity is content-derived. Training, model-selection, calibration and
6//! final-test membership are disjoint by construction, and a source or
7//! snapshot identity repeated across splits is refused as leakage. Semantic
8//! duplicates the manifest cannot identify (two sources with the same
9//! content) remain a stated limit: Rustev does not detect them.
10
11use std::collections::{BTreeMap, BTreeSet};
12use std::fmt;
13
14use rustev_contract::Identified;
15use rustev_contract::decimal::Decimal;
16use rustev_contract::eval_report::{DatasetProvenance, Split};
17use rustev_contract::ids::{ContentDigest, DatasetId, SnapshotId};
18use rustev_contract::limits::REPLAY_V1;
19use serde::{Deserialize, Serialize};
20
21use crate::doc::eval_document;
22
23pub const DATASET: &str = "rustev.dataset/1";
24
25#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
26#[serde(deny_unknown_fields)]
27pub struct DatasetManifest {
28    pub schema: String,
29    pub name: String,
30    /// Synthetic provenance always prevents empirical quality claims (R-04).
31    pub provenance: DatasetProvenance,
32    /// The task adapter whose label shape the cases use.
33    pub adapter: AdapterRef,
34    pub cases: Vec<DatasetCase>,
35}
36
37eval_document!(DatasetManifest, DATASET, REPLAY_V1, DatasetId);
38
39/// A task adapter, by name and version.
40#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
41#[serde(deny_unknown_fields)]
42pub struct AdapterRef {
43    pub name: String,
44    pub version: String,
45}
46
47#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
48#[serde(deny_unknown_fields)]
49pub struct DatasetCase {
50    pub id: String,
51    /// Opaque identity of where the case came from.
52    pub source: String,
53    pub snapshot: SnapshotId,
54    pub split: Split,
55    pub label: Label,
56    pub subgroups: Vec<String>,
57}
58
59/// A label, or an explicit statement that there is none.
60#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
61#[serde(rename_all = "snake_case", deny_unknown_fields)]
62pub enum Label {
63    Missing,
64    Value(String),
65}
66
67impl Label {
68    pub fn value(&self) -> Option<&str> {
69        match self {
70            Label::Value(v) => Some(v),
71            Label::Missing => None,
72        }
73    }
74}
75
76/// Why a manifest was refused.
77#[derive(Debug, Clone, PartialEq, Eq)]
78pub enum DatasetError {
79    Schema,
80    Empty,
81    /// Real provenance needs source, license, labeling method and limits.
82    Provenance,
83    DuplicateCase {
84        case: String,
85    },
86    /// The same source or snapshot appears in more than one split.
87    Leakage {
88        case: String,
89        identity: String,
90    },
91    /// The adapter refused the label's shape.
92    Label {
93        case: String,
94        detail: String,
95    },
96    /// The manifest names another adapter than the one checking it.
97    Adapter,
98}
99
100impl fmt::Display for DatasetError {
101    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
102        write!(f, "{self:?}")
103    }
104}
105
106impl std::error::Error for DatasetError {}
107
108impl DatasetManifest {
109    /// Validate everything but label shape: schema, cases, provenance,
110    /// unique ids and leakage across splits.
111    pub fn check_structure(&self) -> Result<(), DatasetError> {
112        let adapter = self.adapter.clone();
113        self.check(&adapter, |_| Ok(()))
114    }
115
116    /// Validate the manifest; `check_label` is the named adapter's label
117    /// shape check.
118    pub fn check(
119        &self,
120        adapter: &AdapterRef,
121        check_label: impl Fn(&str) -> Result<(), String>,
122    ) -> Result<(), DatasetError> {
123        if self.schema != DATASET {
124            return Err(DatasetError::Schema);
125        }
126        if &self.adapter != adapter {
127            return Err(DatasetError::Adapter);
128        }
129        if self.cases.is_empty() {
130            return Err(DatasetError::Empty);
131        }
132        if let DatasetProvenance::Real {
133            source,
134            license,
135            labeling,
136            limitations,
137        } = &self.provenance
138            && [source, license, labeling, limitations]
139                .iter()
140                .any(|s| s.trim().is_empty())
141        {
142            return Err(DatasetError::Provenance);
143        }
144        let mut ids = BTreeSet::new();
145        let mut split_of: BTreeMap<String, Split> = BTreeMap::new();
146        for c in &self.cases {
147            if !ids.insert(&c.id) {
148                return Err(DatasetError::DuplicateCase { case: c.id.clone() });
149            }
150            for identity in [
151                format!("source:{}", c.source),
152                format!("snapshot:{}", c.snapshot),
153            ] {
154                match split_of.get(&identity) {
155                    Some(s) if *s != c.split => {
156                        return Err(DatasetError::Leakage {
157                            case: c.id.clone(),
158                            identity,
159                        });
160                    }
161                    _ => {
162                        split_of.insert(identity, c.split);
163                    }
164                }
165            }
166            if let Label::Value(v) = &c.label {
167                check_label(v).map_err(|detail| DatasetError::Label {
168                    case: c.id.clone(),
169                    detail,
170                })?;
171            }
172        }
173        Ok(())
174    }
175
176    /// The cases of one split, in manifest order.
177    pub fn split(&self, split: Split) -> impl Iterator<Item = &DatasetCase> {
178        self.cases.iter().filter(move |c| c.split == split)
179    }
180}
181
182// ---------------------------------------------------------------------------
183// Dataset admission (spec 023, 3.1 and 3.2).
184// ---------------------------------------------------------------------------
185
186/// Schema of an owner-reviewed dataset admission record (spec 023, 3.1).
187pub const DATASET_ADMISSION: &str = "rustev.dataset-admission/1";
188
189/// The record that admits one dataset manifest as qualification input
190/// (spec 023, 3.1). The owner writes and reviews it before any case is
191/// evaluated; Rustev checks that it is complete and agrees with the
192/// manifest, never that its statements are true. Its digest is the
193/// admission identity a qualification and every claim name. It holds no
194/// binary64 number: an agreement statistic is a Decimal string. The record
195/// and the data stay with the host, outside this repository (spec 023, 6.4).
196#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
197#[serde(deny_unknown_fields)]
198pub struct AdmissionRecord {
199    pub schema: String,
200    /// The admitted manifest's identity.
201    pub dataset: DatasetId,
202    pub source: AdmissionSource,
203    pub license: LicenseTerms,
204    pub privacy: PrivacyBasis,
205    pub labeling: LabelingMethod,
206    /// The models whose quality this dataset may qualify. Label
207    /// independence is judged against them (3.1.4).
208    pub systems_under_test: Vec<ModelRef>,
209    pub task_mapping: TaskMapping,
210    pub splits: SplitAssignment,
211    pub population: Population,
212    pub exposure: Exposure,
213    pub review: OwnerReview,
214}
215
216eval_document!(AdmissionRecord, DATASET_ADMISSION, REPLAY_V1, ContentDigest);
217
218/// Where the data came from (3.1.1).
219#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
220#[serde(rename_all = "snake_case", deny_unknown_fields)]
221pub enum AdmissionSource {
222    /// A public dataset: publisher, exact version or retrieval date, and the
223    /// digest of the content admitted.
224    Public {
225        publisher: String,
226        version: String,
227        content_digest: ContentDigest,
228    },
229    /// A commissioned set: who labeled it and under what instruction.
230    Commissioned {
231        labeled_by: String,
232        instruction: String,
233        content_digest: ContentDigest,
234    },
235}
236
237/// The license and what it permits (3.1.2).
238#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
239#[serde(deny_unknown_fields)]
240pub struct LicenseTerms {
241    /// The identity of the license text.
242    pub license: String,
243    pub may_commit: bool,
244    pub may_retain_outside_git: bool,
245    pub may_send_to_provider: bool,
246    pub may_redistribute_in_report: bool,
247}
248
249/// The privacy basis (3.1.3).
250#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
251#[serde(deny_unknown_fields)]
252pub struct PrivacyBasis {
253    pub personal_data: bool,
254    /// Derived from real user data: refused without the owner's separate
255    /// privacy decision (spec 012, 3.1.3).
256    pub derived_from_real_user_data: bool,
257    /// A reference to that decision.
258    pub privacy_decision: Option<String>,
259}
260
261/// How labels were produced (3.1.4).
262#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
263#[serde(deny_unknown_fields)]
264pub struct LabelingMethod {
265    pub guidelines: String,
266    pub annotators_per_case: u64,
267    pub agreement: AgreementStatistic,
268    pub adjudication: String,
269    /// Everyone and everything that produced a label.
270    pub producers: Vec<LabelProducer>,
271}
272
273/// An inter-annotator agreement statistic with its denominator.
274#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
275#[serde(deny_unknown_fields)]
276pub struct AgreementStatistic {
277    pub statistic: String,
278    pub value: Decimal,
279    pub denominator: u64,
280}
281
282/// A label producer. A model under test, or a model of the same family, is
283/// never an independent labeler.
284#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
285#[serde(rename_all = "snake_case", deny_unknown_fields)]
286pub enum LabelProducer {
287    Human { description: String },
288    Model(ModelRef),
289}
290
291/// A model, by family and exact identity.
292#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
293#[serde(deny_unknown_fields)]
294pub struct ModelRef {
295    pub family: String,
296    pub identity: String,
297}
298
299impl ModelRef {
300    /// The same model, or one of the same family.
301    pub fn related(&self, other: &ModelRef) -> bool {
302        self.family == other.family || self.identity == other.identity
303    }
304}
305
306/// How the dataset's labels map to the task adapter's (3.1.5).
307#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
308#[serde(deny_unknown_fields)]
309pub struct TaskMapping {
310    pub adapter: AdapterRef,
311    /// The adapter's rules digest (spec 014): the correctness every
312    /// qualified report is computed under.
313    pub rules: ContentDigest,
314    /// Source label to adapter label.
315    pub labels: Vec<LabelMapping>,
316    /// Cases whose source label maps to no adapter label: kept in the
317    /// manifest with a missing label, counted, never dropped.
318    pub unmapped: u64,
319    /// Cases the source left unlabeled.
320    pub unlabeled: u64,
321}
322
323#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
324#[serde(deny_unknown_fields)]
325pub struct LabelMapping {
326    pub from: String,
327    pub to: String,
328}
329
330/// How cases were assigned to splits (3.1.6). Disjointness by source and
331/// snapshot is the manifest's own check (spec 004, 3.5.1); the counts here
332/// must equal the manifest's.
333#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
334#[serde(deny_unknown_fields)]
335pub struct SplitAssignment {
336    pub method: String,
337    pub seed: String,
338    pub training: u64,
339    pub model_selection: u64,
340    pub calibration: u64,
341    pub final_test: u64,
342    /// The stated limit on near-duplicates the manifest cannot identify.
343    pub near_duplicates: String,
344}
345
346/// Who and what the data represents (3.1.7). A claim may name this
347/// population and no other (3.5).
348#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
349#[serde(deny_unknown_fields)]
350pub struct Population {
351    /// A short, exact name a claim must repeat.
352    pub id: String,
353    pub description: String,
354    pub limitations: String,
355    /// Declared slices: manifest subgroups, with their whole-manifest size.
356    pub slices: Vec<SliceSize>,
357}
358
359#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
360#[serde(deny_unknown_fields)]
361pub struct SliceSize {
362    pub name: String,
363    pub cases: u64,
364}
365
366/// Contamination and exposure (3.2).
367#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
368#[serde(deny_unknown_fields)]
369pub struct Exposure {
370    /// Whether a backend's training data may include this dataset: a stated
371    /// limit Rustev cannot verify, required for a public dataset.
372    pub training_data: String,
373    /// Final-test cases whose content reached a provider before admission:
374    /// reported separately and excluded from the headline figures.
375    pub exposed_final_test: Vec<String>,
376    /// For a ranking task, spec 008 4.2's caveat as it applies here.
377    pub ranking_caveat: Option<String>,
378}
379
380/// The owner's review of this record.
381#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
382#[serde(deny_unknown_fields)]
383pub struct OwnerReview {
384    pub reviewer: String,
385    pub date: String,
386}
387
388/// Why a dataset was not admitted.
389#[derive(Debug, Clone, PartialEq, Eq)]
390pub enum AdmissionError {
391    Schema,
392    /// The manifest itself is refused (spec 004, 3.5.1).
393    Dataset(DatasetError),
394    /// Synthetic or generated data is never independent evidence of
395    /// semantic quality (R-04).
396    Synthetic,
397    /// The record admits another manifest.
398    WrongDataset,
399    /// A required statement is empty.
400    Incomplete(&'static str),
401    /// Derived from real user data without the owner's privacy decision.
402    PrivacyDecisionMissing,
403    /// A label producer is a system under test or of the same family.
404    LabelsNotIndependent {
405        producer: ModelRef,
406    },
407    /// The mapping names another adapter than the manifest.
408    Adapter,
409    /// A source label is mapped twice.
410    DuplicateMapping {
411        from: String,
412    },
413    /// Unmapped plus unlabeled cases differ from the manifest's missing
414    /// labels: some case was dropped or invented.
415    MissingLabels {
416        declared: u64,
417        manifest: u64,
418    },
419    /// A split count differs from the manifest's.
420    SplitCount {
421        split: &'static str,
422    },
423    /// A slice is declared twice, is not a manifest subgroup, or its size
424    /// differs from the manifest's.
425    Slice {
426        name: String,
427    },
428    /// An exposed case is not a final-test case of the manifest, or is
429    /// listed twice.
430    Exposure {
431        case: String,
432    },
433}
434
435impl fmt::Display for AdmissionError {
436    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
437        write!(f, "{self:?}")
438    }
439}
440
441impl std::error::Error for AdmissionError {}
442
443/// A manifest with a checked admission record. Only [`admit`] makes one, so
444/// holding it is the statement that the checks of 3.1 ran.
445#[derive(Debug, Clone)]
446pub struct Admitted {
447    record: AdmissionRecord,
448    id: ContentDigest,
449}
450
451impl Admitted {
452    pub fn record(&self) -> &AdmissionRecord {
453        &self.record
454    }
455    /// The admission record's digest.
456    pub fn id(&self) -> &ContentDigest {
457        &self.id
458    }
459    pub fn dataset(&self) -> &DatasetId {
460        &self.record.dataset
461    }
462    /// A model label producer that is `model` or of its family, if any:
463    /// `model`'s outputs would then not meet independent labels.
464    pub fn related_labeler(&self, model: &ModelRef) -> Option<&ModelRef> {
465        self.record.labeling.producers.iter().find_map(|p| match p {
466            LabelProducer::Model(m) if m.related(model) => Some(m),
467            _ => None,
468        })
469    }
470    /// Whether the admission judged independence against this model.
471    pub fn declares_system(&self, model: &ModelRef) -> bool {
472        self.record.systems_under_test.contains(model)
473    }
474}
475
476/// Admit a manifest under an owner-reviewed record (spec 023, 3.1, 3.2).
477/// The record must be complete and agree with the manifest; labels must be
478/// independent of every system under test; data derived from real user data
479/// needs the owner's privacy decision. A synthetic manifest is never
480/// admitted.
481pub fn admit(
482    manifest: &DatasetManifest,
483    record: &AdmissionRecord,
484) -> Result<Admitted, AdmissionError> {
485    use AdmissionError as E;
486    if record.schema != DATASET_ADMISSION {
487        return Err(E::Schema);
488    }
489    manifest.check_structure().map_err(E::Dataset)?;
490    if matches!(manifest.provenance, DatasetProvenance::Synthetic { .. }) {
491        return Err(E::Synthetic);
492    }
493    if manifest.id().ok().as_ref() != Some(&record.dataset) {
494        return Err(E::WrongDataset);
495    }
496    let blank = |s: &str| s.trim().is_empty();
497    let need = |ok: bool, what: &'static str| {
498        if ok { Ok(()) } else { Err(E::Incomplete(what)) }
499    };
500    match &record.source {
501        AdmissionSource::Public {
502            publisher, version, ..
503        } => {
504            need(!blank(publisher) && !blank(version), "source")?;
505            need(
506                !blank(&record.exposure.training_data),
507                "training-data exposure",
508            )?;
509        }
510        AdmissionSource::Commissioned {
511            labeled_by,
512            instruction,
513            ..
514        } => need(!blank(labeled_by) && !blank(instruction), "source")?,
515    }
516    need(!blank(&record.license.license), "license")?;
517    if record.privacy.derived_from_real_user_data
518        && record.privacy.privacy_decision.as_deref().is_none_or(blank)
519    {
520        return Err(E::PrivacyDecisionMissing);
521    }
522    let l = &record.labeling;
523    need(
524        !blank(&l.guidelines)
525            && !blank(&l.adjudication)
526            && l.annotators_per_case > 0
527            && !l.producers.is_empty(),
528        "labeling method",
529    )?;
530    need(
531        !blank(&l.agreement.statistic)
532            && l.agreement.denominator > 0
533            && !l.agreement.value.is_negative(),
534        "agreement statistic",
535    )?;
536    for p in &l.producers {
537        let ok = match p {
538            LabelProducer::Human { description } => !blank(description),
539            LabelProducer::Model(m) => !blank(&m.family) && !blank(&m.identity),
540        };
541        need(ok, "label producer")?;
542    }
543    need(!record.systems_under_test.is_empty(), "systems under test")?;
544    for s in &record.systems_under_test {
545        need(
546            !blank(&s.family) && !blank(&s.identity),
547            "systems under test",
548        )?;
549        for p in &l.producers {
550            if let LabelProducer::Model(m) = p
551                && m.related(s)
552            {
553                return Err(E::LabelsNotIndependent {
554                    producer: m.clone(),
555                });
556            }
557        }
558    }
559    let m = &record.task_mapping;
560    if m.adapter != manifest.adapter {
561        return Err(E::Adapter);
562    }
563    let mut froms = BTreeSet::new();
564    for x in &m.labels {
565        need(!blank(&x.from) && !blank(&x.to), "label mapping")?;
566        if !froms.insert(&x.from) {
567            return Err(E::DuplicateMapping {
568                from: x.from.clone(),
569            });
570        }
571    }
572    let missing = manifest
573        .cases
574        .iter()
575        .filter(|c| c.label == Label::Missing)
576        .count() as u64;
577    let declared = m.unmapped.saturating_add(m.unlabeled);
578    if declared != missing {
579        return Err(E::MissingLabels {
580            declared,
581            manifest: missing,
582        });
583    }
584    let s = &record.splits;
585    need(
586        !blank(&s.method) && !blank(&s.seed) && !blank(&s.near_duplicates),
587        "split assignment",
588    )?;
589    for (split, n, name) in [
590        (Split::Training, s.training, "training"),
591        (Split::ModelSelection, s.model_selection, "model_selection"),
592        (Split::Calibration, s.calibration, "calibration"),
593        (Split::FinalTest, s.final_test, "final_test"),
594    ] {
595        if manifest.split(split).count() as u64 != n {
596            return Err(E::SplitCount { split: name });
597        }
598    }
599    let p = &record.population;
600    need(
601        !blank(&p.id) && !blank(&p.description) && !blank(&p.limitations),
602        "population",
603    )?;
604    let mut names = BTreeSet::new();
605    for slice in &p.slices {
606        let size = manifest
607            .cases
608            .iter()
609            .filter(|c| c.subgroups.contains(&slice.name))
610            .count() as u64;
611        if !names.insert(&slice.name) || size == 0 || size != slice.cases {
612            return Err(E::Slice {
613                name: slice.name.clone(),
614            });
615        }
616    }
617    let mut exposed = BTreeSet::new();
618    for case in &record.exposure.exposed_final_test {
619        let ok = manifest.split(Split::FinalTest).any(|c| &c.id == case);
620        if !ok || !exposed.insert(case) {
621            return Err(E::Exposure { case: case.clone() });
622        }
623    }
624    if record.exposure.ranking_caveat.as_deref().is_some_and(blank) {
625        return Err(E::Incomplete("ranking caveat"));
626    }
627    need(
628        !blank(&record.review.reviewer) && !blank(&record.review.date),
629        "owner review",
630    )?;
631    let id = record.id().map_err(|_| E::Incomplete("canonical form"))?;
632    Ok(Admitted {
633        record: record.clone(),
634        id,
635    })
636}
rustev_eval/dataset.rs

rustev_eval/
dataset.rs
