pondus 0.2.1

Opinionated AI model benchmark aggregator
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

use crate::cache::Cache;
use crate::config::Config;
use crate::models::{MetricValue, ModelScore, SourceResult, SourceStatus};
use crate::sources::Source;
use anyhow::{Context, Result};
use chrono::Utc;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;

const AIDER_URL: &str = "https://raw.githubusercontent.com/Aider-AI/aider/main/aider/website/_data/polyglot_leaderboard.yml";

#[derive(Debug, Serialize, Deserialize)]
struct AiderEntry {
    model: String,
    #[serde(default)]
    pass_rate_1: Option<f64>,
    #[serde(default)]
    total_cost: Option<f64>,
    #[serde(default)]
    percent_cases_well_formed: Option<f64>,
}

pub struct Aider;

impl Source for Aider {
    fn name(&self) -> &str {
        "aider"
    }

    fn fetch(&self, _config: &Config, cache: &Cache) -> Result<SourceResult> {
        // Check cache first
        if let Some((fetched_at, cached_data)) = cache.get("aider") {
            return Ok(SourceResult {
                source: self.name().into(),
                fetched_at: Some(fetched_at),
                status: SourceStatus::Cached,
                scores: parse_scores(&cached_data),
            });
        }

        // Fetch YAML from GitHub
        let client = reqwest::blocking::Client::builder()
            .timeout(Duration::from_secs(30))
            .build()
            .context("Failed to build HTTP client")?;
        let response = client
            .get(AIDER_URL)
            .send()
            .context("Failed to fetch Aider leaderboard")?;

        if !response.status().is_success() {
            return Ok(SourceResult {
                source: self.name().into(),
                fetched_at: None,
                status: SourceStatus::Error(format!("HTTP {}", response.status())),
                scores: vec![],
            });
        }

        let yaml_text = response.text().context("Failed to read Aider response")?;

        // Parse YAML into entries
        let entries: Vec<AiderEntry> =
            serde_yaml::from_str(&yaml_text).context("Failed to parse Aider YAML")?;

        // Convert to JSON Value for caching
        let data = serde_json::to_value(&entries)?;
        cache.set("aider", &data)?;

        Ok(SourceResult {
            source: self.name().into(),
            fetched_at: Some(Utc::now()),
            status: SourceStatus::Ok,
            scores: parse_scores(&data),
        })
    }
}

fn parse_scores(data: &serde_json::Value) -> Vec<ModelScore> {
    let mut scores = Vec::new();

    let Some(entries) = data.as_array() else {
        return scores;
    };

    for entry in entries {
        let Some(model_name) = entry.get("model").and_then(|v| v.as_str()) else {
            continue;
        };

        let mut metrics = HashMap::new();

        if let Some(rate) = entry.get("pass_rate_1").and_then(|v| v.as_f64()) {
            metrics.insert("pass_rate_1".into(), MetricValue::Float(rate));
        }

        if let Some(cost) = entry.get("total_cost").and_then(|v| v.as_f64()) {
            metrics.insert("cost".into(), MetricValue::Float(cost));
        }

        if let Some(wf) = entry
            .get("percent_cases_well_formed")
            .and_then(|v| v.as_f64())
        {
            metrics.insert("percent_cases_well_formed".into(), MetricValue::Float(wf));
        }

        scores.push(ModelScore {
            model: model_name.to_lowercase().replace([' ', '_'], "-"),
            source_model_name: model_name.to_string(),
            metrics,
            rank: None,
        });
    }

    // Sort by pass_rate_1 descending
    scores.sort_by(|a, b| {
        let a_rate = get_float(&a.metrics, "pass_rate_1");
        let b_rate = get_float(&b.metrics, "pass_rate_1");
        b_rate
            .partial_cmp(&a_rate)
            .unwrap_or(std::cmp::Ordering::Equal)
    });

    for (i, score) in scores.iter_mut().enumerate() {
        score.rank = Some((i + 1) as u32);
    }

    scores
}

fn get_float(metrics: &HashMap<String, MetricValue>, key: &str) -> f64 {
    match metrics.get(key) {
        Some(MetricValue::Float(f)) => *f,
        _ => 0.0,
    }
}