rustev_contract/
eval_report.rs

1//! The versioned evaluation report envelope, `rustev.eval-report/1` (R-06).
2//!
3//! Only the envelope lives here. Metrics, datasets, evaluation execution and
4//! calibration fitting live outside the contract crate. An absent measurement
5//! is `unknown`, never a pass; a report names its dataset, split and
6//! provenance, and synthetic data is labeled as such (R-04).
7
8use serde::{Deserialize, Serialize};
9
10use crate::ids::{ArtifactId, CalibrationId, DatasetId, EvaluatorConfigId, PlanId};
11use crate::limits::RECORD_V1;
12use crate::schema;
13
14#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
15#[serde(deny_unknown_fields)]
16pub struct EvalReport {
17    pub schema: String,
18    pub plan: PlanId,
19    pub artifacts: Vec<ArtifactId>,
20    pub calibrations: Vec<CalibrationId>,
21    pub dataset: DatasetRef,
22    pub evaluator_config: EvaluatorConfigId,
23    pub status: ReportStatus,
24    pub metrics: Vec<Metric>,
25}
26
27crate::document::document!(EvalReport, schema::EVAL_REPORT, RECORD_V1);
28
29#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
30#[serde(deny_unknown_fields)]
31pub struct DatasetRef {
32    pub id: DatasetId,
33    pub split: Split,
34    pub provenance: DatasetProvenance,
35}
36
37/// A split used to fit a head or a calibration map is not a holdout afterwards.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
39#[serde(rename_all = "snake_case")]
40pub enum Split {
41    Training,
42    ModelSelection,
43    Calibration,
44    FinalTest,
45}
46
47#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
48#[serde(rename_all = "snake_case", deny_unknown_fields)]
49pub enum DatasetProvenance {
50    /// Generated; evidence of mechanics, never of semantic quality (R-04).
51    Synthetic { generator: String },
52    Real {
53        source: String,
54        license: String,
55        labeling: String,
56        limitations: String,
57    },
58}
59
60#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
61#[serde(rename_all = "snake_case", deny_unknown_fields)]
62pub enum ReportStatus {
63    Complete,
64    Incomplete { reason: String },
65    Incomparable { reason: String },
66}
67
68#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
69#[serde(deny_unknown_fields)]
70pub struct Metric {
71    pub name: String,
72    pub value: MetricValue,
73}
74
75#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
76#[serde(rename_all = "snake_case", deny_unknown_fields)]
77pub enum MetricValue {
78    Measured { value: f64, n: u64 },
79    Unknown { reason: String },
80}
rustev_contract/eval_report.rs

rustev_contract/
eval_report.rs
