use evalhub_schema::RecordKind;
pub const CORE_VERSION: &str = "1";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct MetricDef {
pub id: &'static str,
pub lower_is_better: bool,
pub description: &'static str,
}
pub const CORE_METRICS: &[MetricDef] = &[
MetricDef {
id: "core/pass_rate",
lower_is_better: false,
description: "Fraction of items graded as passing.",
},
MetricDef {
id: "core/accuracy",
lower_is_better: false,
description: "Fraction of items answered correctly.",
},
MetricDef {
id: "core/f1",
lower_is_better: false,
description: "Harmonic mean of precision and recall.",
},
MetricDef {
id: "core/exact_match",
lower_is_better: false,
description: "Fraction of items whose output matched the reference exactly.",
},
MetricDef {
id: "core/pass_at_k",
lower_is_better: false,
description: "Probability that at least one of k samples passes.",
},
MetricDef {
id: "core/mean_score",
lower_is_better: false,
description: "Mean of a per-item score on the producer's scale.",
},
MetricDef {
id: "core/latency_ms",
lower_is_better: true,
description: "Wall-clock time per item, in milliseconds.",
},
MetricDef {
id: "core/cost_usd",
lower_is_better: true,
description: "Cost per item, in US dollars.",
},
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RelationTypeDef {
pub id: &'static str,
pub from: RecordKind,
pub to: RecordKind,
pub inverse: &'static str,
}
pub const CORE_RELATION_TYPES: &[RelationTypeDef] = &[
RelationTypeDef {
id: "core/uses_eval",
from: RecordKind::Card,
to: RecordKind::Eval,
inverse: "used_by",
},
RelationTypeDef {
id: "core/retry_of",
from: RecordKind::Card,
to: RecordKind::Card,
inverse: "retried_by",
},
RelationTypeDef {
id: "core/rerun_of",
from: RecordKind::Card,
to: RecordKind::Card,
inverse: "rerun_by",
},
RelationTypeDef {
id: "core/judged_by",
from: RecordKind::Card,
to: RecordKind::Card,
inverse: "judges",
},
RelationTypeDef {
id: "core/baseline_of",
from: RecordKind::Card,
to: RecordKind::Card,
inverse: "compared_to",
},
RelationTypeDef {
id: "core/derived_from",
from: RecordKind::Eval,
to: RecordKind::Eval,
inverse: "derives",
},
RelationTypeDef {
id: "core/subset_of",
from: RecordKind::Eval,
to: RecordKind::Eval,
inverse: "superset_of",
},
];
pub fn core_metric(id: &str) -> Option<&'static MetricDef> {
CORE_METRICS.iter().find(|m| m.id == id)
}
pub fn core_relation_type(id: &str) -> Option<&'static RelationTypeDef> {
CORE_RELATION_TYPES.iter().find(|r| r.id == id)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
use crate::validate::is_valid_id;
#[test]
fn ids_are_well_formed_and_in_core() {
for m in CORE_METRICS {
assert!(is_valid_id(m.id), "{}", m.id);
assert!(m.id.starts_with("core/"), "{}", m.id);
}
for r in CORE_RELATION_TYPES {
assert!(is_valid_id(r.id), "{}", r.id);
assert!(r.id.starts_with("core/"), "{}", r.id);
assert!(!r.inverse.is_empty());
}
assert_eq!(CORE_METRICS.len(), 8);
assert_eq!(CORE_RELATION_TYPES.len(), 7);
}
#[test]
fn lookups() {
assert!(core_metric("core/latency_ms").is_some_and(|m| m.lower_is_better));
assert!(core_metric("core/pass_rate").is_some_and(|m| !m.lower_is_better));
assert!(core_metric("alice/x").is_none());
let r = core_relation_type("core/uses_eval").unwrap();
assert_eq!(
(r.from, r.to, r.inverse),
(RecordKind::Card, RecordKind::Eval, "used_by")
);
assert!(core_relation_type("core/nope").is_none());
}
}