use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use crate::common::{Attachment, Ext, Producer, Redaction, Relation};
use crate::facet::{Env, Generation, Grading, Harness, Model, Task, Trial};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct Card {
#[schemars(extend("const" = "evalhub.card/1.0"))]
pub schema: String,
pub title: String,
pub producer: Producer,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model: Option<Model>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub task: Option<Task>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub harness: Option<Harness>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub generation: Option<Generation>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trial: Option<Trial>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub grading: Option<Grading>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub env: Option<Env>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub results: Vec<ResultEntry>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub counts: Option<Counts>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub relations: Vec<Relation>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub attachments: Vec<Attachment>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub redaction: Option<Redaction>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provenance: Option<Provenance>,
#[serde(default)]
pub ext: Ext,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct ResultEntry {
pub metric: String,
pub value: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub n: Option<u64>,
pub aggregation: Aggregation,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub uncertainty: Option<Uncertainty>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub by: Option<serde_json::Map<String, serde_json::Value>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub samples_ref: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum Aggregation {
Mean,
PassAtK,
Median,
Sum,
Custom,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct Uncertainty {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stderr: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ci: Option<ConfidenceInterval>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct ConfidenceInterval {
pub level: f64,
pub low: f64,
pub high: f64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct Counts {
pub attempted: u64,
#[serde(default)]
pub completed: u64,
#[serde(default)]
pub failed: u64,
#[serde(default)]
pub skipped: u64,
#[serde(default)]
pub errored: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct Provenance {
pub source_type: SourceType,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub evaluator_relationship: Option<EvaluatorRelationship>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum SourceType {
EvaluationRun,
Paper,
ModelCard,
Leaderboard,
Other,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum EvaluatorRelationship {
FirstParty,
ThirdParty,
}