use std::collections::{BTreeMap, HashSet};
use std::str::FromStr;
use std::sync::OnceLock;
use serde_json::Value;
use sha2::{Digest, Sha256};
use evalhub_schema::RecordKind;
use crate::canonical::{CanonicalError, canonicalize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Facet {
Model,
Task,
Harness,
Generation,
Trial,
Grading,
Env,
}
impl Facet {
pub const ALL: [Facet; 7] = [
Facet::Model,
Facet::Task,
Facet::Harness,
Facet::Generation,
Facet::Trial,
Facet::Grading,
Facet::Env,
];
pub fn for_kind(kind: RecordKind) -> &'static [Facet] {
const EVAL: [Facet; 6] = [
Facet::Model,
Facet::Task,
Facet::Harness,
Facet::Generation,
Facet::Trial,
Facet::Env,
];
match kind {
RecordKind::Card => &Facet::ALL,
RecordKind::Eval => &EVAL,
}
}
pub const fn as_str(self) -> &'static str {
match self {
Facet::Model => "model",
Facet::Task => "task",
Facet::Harness => "harness",
Facet::Generation => "generation",
Facet::Trial => "trial",
Facet::Grading => "grading",
Facet::Env => "env",
}
}
const fn def_name(self) -> &'static str {
match self {
Facet::Model => "Model",
Facet::Task => "Task",
Facet::Harness => "Harness",
Facet::Generation => "Generation",
Facet::Trial => "Trial",
Facet::Grading => "Grading",
Facet::Env => "Env",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[error("not a facet: {0:?}")]
pub struct ParseFacetError(String);
impl FromStr for Facet {
type Err = ParseFacetError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Facet::ALL
.into_iter()
.find(|f| f.as_str() == s)
.ok_or_else(|| ParseFacetError(s.to_string()))
}
}
impl std::fmt::Display for Facet {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct Fingerprints(BTreeMap<Facet, [u8; 32]>);
impl Fingerprints {
pub fn get(&self, facet: Facet) -> Option<&[u8; 32]> {
self.0.get(&facet)
}
pub fn iter(&self) -> impl Iterator<Item = (Facet, &[u8; 32])> {
self.0.iter().map(|(f, h)| (*f, h))
}
pub fn to_hex_map(&self) -> BTreeMap<&'static str, String> {
self.0
.iter()
.map(|(f, h)| (f.as_str(), hex::encode(h)))
.collect()
}
pub fn len(&self) -> usize {
self.0.len()
}
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
}
fn exclusions() -> &'static BTreeMap<Facet, HashSet<String>> {
static TABLE: OnceLock<BTreeMap<Facet, HashSet<String>>> = OnceLock::new();
TABLE.get_or_init(|| {
let schema = RecordKind::Card.schema().to_value();
let mut table = BTreeMap::new();
for facet in Facet::ALL {
table.insert(facet, excluded_keys(&schema, facet));
}
table
})
}
fn excluded_keys(schema: &Value, facet: Facet) -> HashSet<String> {
let mut keys: HashSet<String> = HashSet::from(["ext".to_string()]);
let props = schema
.pointer(&format!("/$defs/{}/properties", facet.def_name()))
.and_then(Value::as_object);
if let Some(props) = props {
for (name, prop) in props {
if prop.get("x-fingerprint") == Some(&Value::Bool(false)) {
keys.insert(name.clone());
}
}
}
keys
}
pub fn schema_shape_is_understood() -> bool {
let schema = RecordKind::Card.schema().to_value();
Facet::ALL.into_iter().all(|f| {
schema
.pointer(&format!("/$defs/{}/properties", f.def_name()))
.is_some_and(Value::is_object)
})
}
pub fn fingerprints(kind: RecordKind, value: &Value) -> Result<Fingerprints, CanonicalError> {
let table = exclusions();
let mut out = BTreeMap::new();
for &facet in Facet::for_kind(kind) {
let mut object = match value.get(facet.as_str()) {
Some(Value::Object(m)) => m.clone(),
_ => serde_json::Map::new(),
};
if let Some(excluded) = table.get(&facet) {
object.retain(|k, _| !excluded.contains(k));
}
let bytes = canonicalize(&Value::Object(object))?;
let digest: [u8; 32] = Sha256::digest(&bytes).into();
out.insert(facet, digest);
}
Ok(Fingerprints(out))
}
pub fn for_run(run: &Value) -> Result<Fingerprints, CanonicalError> {
fingerprints(RecordKind::Eval, run)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
#[test]
fn schema_shape() {
assert!(schema_shape_is_understood());
}
#[test]
fn exclusion_table_matches_the_schema_annotations() {
let t = exclusions();
let names = |f: Facet| {
let mut v: Vec<&str> = t[&f].iter().map(String::as_str).collect();
v.sort_unstable();
v
};
assert_eq!(names(Facet::Model), ["context_window", "ext"]);
assert_eq!(names(Facet::Env), ["ext", "hardware", "os"]);
for f in [
Facet::Task,
Facet::Harness,
Facet::Generation,
Facet::Trial,
Facet::Grading,
] {
assert_eq!(names(f), ["ext"], "{f}");
}
}
#[test]
fn facet_names_round_trip() {
for f in Facet::ALL {
assert_eq!(f.as_str().parse::<Facet>().unwrap(), f);
}
assert!("nope".parse::<Facet>().is_err());
assert_eq!(Facet::for_kind(RecordKind::Eval).len(), 6);
assert!(!Facet::for_kind(RecordKind::Eval).contains(&Facet::Grading));
}
}