entrenar/hf_pipeline/publish/
submission.rs

1//! Leaderboard submission formatting
2//!
3//! Formats `EvalResult` as JSONL for submission to HuggingFace
4//! leaderboard-specific result repositories.
5
6use crate::eval::evaluator::{EvalResult, Metric};
7
8/// Format an `EvalResult` as a JSONL line for leaderboard submission
9pub fn format_submission_jsonl(result: &EvalResult) -> String {
10    let mut obj = serde_json::Map::new();
11
12    obj.insert("model".to_string(), serde_json::Value::String(result.model_name.clone()));
13
14    for (metric, &value) in &result.scores {
15        let key = metric_to_submission_key(metric);
16        obj.insert(key, serde_json::json!(value));
17    }
18
19    if result.inference_time_ms > 0.0 {
20        obj.insert("inference_time_ms".to_string(), serde_json::json!(result.inference_time_ms));
21    }
22
23    serde_json::Value::Object(obj).to_string()
24}
25
26/// Format multiple `EvalResult`s as JSONL
27pub fn format_submissions_jsonl(results: &[EvalResult]) -> String {
28    results.iter().map(format_submission_jsonl).collect::<Vec<_>>().join("\n")
29}
30
31/// Convert a `Metric` to a leaderboard-compatible key name
32fn metric_to_submission_key(metric: &Metric) -> String {
33    match metric {
34        Metric::WER => "wer".to_string(),
35        Metric::RTFx => "rtfx".to_string(),
36        Metric::BLEU => "bleu".to_string(),
37        Metric::ROUGE(v) => format!("{v}").to_lowercase().replace('-', "_"),
38        Metric::Perplexity => "perplexity".to_string(),
39        Metric::MMLUAccuracy => "mmlu_accuracy".to_string(),
40        Metric::PassAtK(k) => format!("pass@{k}"),
41        Metric::NDCGAtK(k) => format!("ndcg@{k}"),
42        other => other.name().to_lowercase(),
43    }
44}
entrenar/hf_pipeline/publish/submission.rs

entrenar/hf_pipeline/publish/
submission.rs
