entrenar 0.7.13

Training & Optimization library with autograd, LoRA, quantization, and model merging
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

//! Leaderboard-to-EvalResult parser
//!
//! Converts HuggingFace leaderboard data into the native evaluation types,
//! enabling comparison of your models against published leaderboard entries.

use super::types::{HfLeaderboard, LeaderboardKind};
use crate::eval::evaluator::{EvalResult, Leaderboard, Metric};
use crate::eval::RougeVariant;

/// Convert an `HfLeaderboard` into a native `Leaderboard` for comparison.
///
/// Maps leaderboard columns to `Metric` variants using kind-specific mappings.
pub fn to_leaderboard(hf: &HfLeaderboard) -> Leaderboard {
    let primary = hf.kind.primary_metric();
    let mut leaderboard = Leaderboard::new(primary);

    for entry in &hf.entries {
        let mut result = EvalResult::new(&entry.model_id);

        for (column, &value) in &entry.scores {
            if let Some(metric) = column_to_metric(&hf.kind, column) {
                result.add_score(metric, value);
            }
        }

        leaderboard.add(result);
    }

    leaderboard
}

/// Map a leaderboard column name to a `Metric` variant.
///
/// Each leaderboard kind has its own column naming conventions.
#[must_use]
pub fn column_to_metric(kind: &LeaderboardKind, column: &str) -> Option<Metric> {
    let col_lower = column.to_lowercase();

    match kind {
        LeaderboardKind::OpenASR => match col_lower.as_str() {
            "wer" | "average_wer" | "word_error_rate" => Some(Metric::WER),
            "rtfx" | "rtf" | "real_time_factor" => Some(Metric::RTFx),
            _ => None,
        },
        LeaderboardKind::OpenLLMv2 => match col_lower.as_str() {
            "mmlu" | "mmlu_pro" | "mmlu_accuracy" => Some(Metric::MMLUAccuracy),
            "accuracy" | "average" | "avg" => Some(Metric::Accuracy),
            _ => None,
        },
        LeaderboardKind::MTEB => match col_lower.as_str() {
            "ndcg@10" | "ndcg_at_10" => Some(Metric::NDCGAtK(10)),
            "accuracy" => Some(Metric::Accuracy),
            _ => None,
        },
        LeaderboardKind::BigCodeBench => match col_lower.as_str() {
            "pass@1" | "pass_at_1" => Some(Metric::PassAtK(1)),
            "pass@10" | "pass_at_10" => Some(Metric::PassAtK(10)),
            _ => None,
        },
        LeaderboardKind::Custom(_) => generic_column_to_metric(&col_lower),
    }
}

/// Best-effort column name → Metric mapping for custom leaderboards
fn generic_column_to_metric(column: &str) -> Option<Metric> {
    match column {
        "accuracy" | "acc" => Some(Metric::Accuracy),
        "wer" | "word_error_rate" => Some(Metric::WER),
        "bleu" => Some(Metric::BLEU),
        "rouge1" | "rouge_1" => Some(Metric::ROUGE(RougeVariant::Rouge1)),
        "rouge2" | "rouge_2" => Some(Metric::ROUGE(RougeVariant::Rouge2)),
        "rougel" | "rouge_l" => Some(Metric::ROUGE(RougeVariant::RougeL)),
        "perplexity" | "ppl" => Some(Metric::Perplexity),
        "mmlu" => Some(Metric::MMLUAccuracy),
        "pass@1" | "pass_at_1" => Some(Metric::PassAtK(1)),
        "ndcg@10" | "ndcg_at_10" => Some(Metric::NDCGAtK(10)),
        _ => None,
    }
}

/// Compare your model's `EvalResult` against a HuggingFace leaderboard.
///
/// Inserts your result into the leaderboard for ranking, returning
/// a sorted `Leaderboard` with your model included.
pub fn compare_with_leaderboard(my_result: &EvalResult, hf: &HfLeaderboard) -> Leaderboard {
    let mut leaderboard = to_leaderboard(hf);
    leaderboard.add(my_result.clone());
    leaderboard
}