entrenar/hf_pipeline/leaderboard/
types.rs

1//! HuggingFace leaderboard types
2//!
3//! Defines leaderboard kinds, entries, and result containers for
4//! interacting with HuggingFace open evaluation leaderboards.
5
6use std::collections::HashMap;
7
8use crate::eval::evaluator::Metric;
9
10/// Known HuggingFace evaluation leaderboards
11#[derive(Clone, Debug, PartialEq, Eq)]
12pub enum LeaderboardKind {
13    /// HF Audio Open ASR Leaderboard
14    OpenASR,
15    /// Open LLM Leaderboard v2
16    OpenLLMv2,
17    /// MTEB (Massive Text Embedding Benchmark)
18    MTEB,
19    /// BigCodeBench
20    BigCodeBench,
21    /// Custom leaderboard by dataset repository ID
22    Custom(String),
23}
24
25impl LeaderboardKind {
26    /// Get the HuggingFace dataset repository ID for this leaderboard
27    #[must_use]
28    pub fn dataset_repo_id(&self) -> &str {
29        match self {
30            Self::OpenASR => "hf-audio/open_asr_leaderboard",
31            Self::OpenLLMv2 => "open-llm-leaderboard/results",
32            Self::MTEB => "mteb/leaderboard",
33            Self::BigCodeBench => "bigcode/bigcodebench-results",
34            Self::Custom(id) => id,
35        }
36    }
37
38    /// Get the primary ranking metric for this leaderboard
39    #[must_use]
40    pub fn primary_metric(&self) -> Metric {
41        match self {
42            Self::OpenASR => Metric::WER,
43            Self::OpenLLMv2 => Metric::MMLUAccuracy,
44            Self::MTEB => Metric::NDCGAtK(10),
45            Self::BigCodeBench => Metric::PassAtK(1),
46            Self::Custom(_) => Metric::Accuracy,
47        }
48    }
49
50    /// Get all tracked metrics for this leaderboard
51    #[must_use]
52    pub fn tracked_metrics(&self) -> Vec<Metric> {
53        match self {
54            Self::OpenASR => vec![Metric::WER, Metric::RTFx],
55            Self::OpenLLMv2 => vec![Metric::MMLUAccuracy, Metric::Accuracy],
56            Self::MTEB => vec![Metric::NDCGAtK(10), Metric::Accuracy],
57            Self::BigCodeBench => vec![Metric::PassAtK(1), Metric::PassAtK(10)],
58            Self::Custom(_) => vec![Metric::Accuracy],
59        }
60    }
61}
62
63impl std::fmt::Display for LeaderboardKind {
64    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
65        match self {
66            Self::OpenASR => write!(f, "Open ASR Leaderboard"),
67            Self::OpenLLMv2 => write!(f, "Open LLM Leaderboard v2"),
68            Self::MTEB => write!(f, "MTEB Leaderboard"),
69            Self::BigCodeBench => write!(f, "BigCodeBench"),
70            Self::Custom(id) => write!(f, "Custom ({id})"),
71        }
72    }
73}
74
75/// A single entry (row) from a HuggingFace leaderboard
76#[derive(Clone, Debug)]
77pub struct LeaderboardEntry {
78    /// Model repository ID (e.g., "openai/whisper-large-v3")
79    pub model_id: String,
80    /// Raw string-keyed scores from HuggingFace
81    pub scores: HashMap<String, f64>,
82    /// Additional metadata (license, parameter count, etc.)
83    pub metadata: HashMap<String, String>,
84}
85
86impl LeaderboardEntry {
87    /// Create a new leaderboard entry
88    #[must_use]
89    pub fn new(model_id: impl Into<String>) -> Self {
90        Self { model_id: model_id.into(), scores: HashMap::new(), metadata: HashMap::new() }
91    }
92
93    /// Get a score by column name
94    #[must_use]
95    pub fn get_score(&self, column: &str) -> Option<f64> {
96        self.scores.get(column).copied()
97    }
98}
99
100/// Container for leaderboard data fetched from HuggingFace
101#[derive(Clone, Debug)]
102pub struct HfLeaderboard {
103    /// Leaderboard kind
104    pub kind: LeaderboardKind,
105    /// Entries (rows)
106    pub entries: Vec<LeaderboardEntry>,
107    /// Total number of entries available (may be more than fetched)
108    pub total_count: usize,
109}
110
111impl HfLeaderboard {
112    /// Create a new leaderboard container
113    #[must_use]
114    pub fn new(kind: LeaderboardKind) -> Self {
115        Self { kind, entries: Vec::new(), total_count: 0 }
116    }
117
118    /// Find an entry by model ID
119    #[must_use]
120    pub fn find_model(&self, model_id: &str) -> Option<&LeaderboardEntry> {
121        self.entries.iter().find(|e| e.model_id == model_id)
122    }
123}
entrenar/hf_pipeline/leaderboard/types.rs

entrenar/hf_pipeline/leaderboard/
types.rs
