eval-containers 0.1.0

A build system for AI agent evaluations
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

use clap::{Args, Subcommand};
use std::process::Command;

#[derive(Args)]
pub struct ListArgs {
    #[command(subcommand)]
    pub target: ListTarget,
}

#[derive(Subcommand)]
pub enum ListTarget {
    /// List available benchmarks
    Benchmarks,
    /// List available agents
    Agents,
    /// List available models
    Models,
    /// List eval images (benchmark + agent combinations)
    Evals {
        #[arg(long)]
        benchmark: Option<String>,
        #[arg(long)]
        agent: Option<String>,
    },
}

pub fn execute(registry: &str, args: ListArgs) -> Result<(), String> {
    match args.target {
        ListTarget::Benchmarks => {
            // Docker: docker images --format '{{.Repository}}:{{.Tag}}' {registry}/benchmarks/*
            let images = get_images(registry, "benchmarks")?;
            if images.is_empty() {
                eprintln!("no benchmark images found");
                return Ok(());
            }
            println!(
                "{:<30} {:<50} {:<6} {:<10} INTERNET",
                "IMAGE", "DESCRIPTION", "TASKS", "TYPE"
            );
            println!("{}", "-".repeat(110));
            for image in &images {
                let desc = get_label(image, "eval.benchmark.description");
                let tasks = get_label(image, "eval.benchmark.tasks");
                let env = get_label(image, "eval.benchmark.env");
                let internet = get_label(image, "eval.benchmark.internet");
                println!("{image:<30} {desc:<50} {tasks:<6} {env:<10} {internet}");
            }
            Ok(())
        }
        ListTarget::Agents => {
            let images = get_images(registry, "agents")?;
            if images.is_empty() {
                eprintln!("no agent images found");
                return Ok(());
            }
            println!("{:<30} {:<40} RUNTIME", "IMAGE", "DESCRIPTION");
            println!("{}", "-".repeat(80));
            for image in &images {
                let desc = get_label(image, "eval.agent.description");
                let runtime = get_label(image, "eval.agent.runtime");
                println!("{image:<30} {desc:<40} {runtime}");
            }
            Ok(())
        }
        ListTarget::Models => {
            let images = get_images(registry, "models")?;
            if images.is_empty() {
                eprintln!("no model images found");
                return Ok(());
            }
            println!("{:<30} PROVIDER", "IMAGE");
            println!("{}", "-".repeat(45));
            for image in &images {
                let provider = get_label(image, "eval.model.provider");
                println!("{image:<30} {provider}");
            }
            Ok(())
        }
        ListTarget::Evals { benchmark, agent } => {
            let filter = match (benchmark, agent) {
                (Some(b), Some(a)) => format!("{b}--{a}"),
                (Some(b), None) => format!("{b}--"),
                (None, Some(a)) => format!("--{a}"),
                (None, None) => String::new(),
            };
            let reference = format!("{registry}/evals/*{filter}*");
            let fmt = "{{.Repository}}:{{.Tag}}";
            eprintln!("$ docker images --format '{fmt}' {reference}");
            let output = Command::new("docker")
                .args(["images", "--format", fmt, &reference])
                .output()
                .map_err(|e| format!("failed to run docker: {e}"))?;
            print!("{}", String::from_utf8_lossy(&output.stdout));
            Ok(())
        }
    }
}

fn get_images(registry: &str, category: &str) -> Result<Vec<String>, String> {
    let reference = format!("{registry}/{category}/*");
    let fmt = "{{.Repository}}:{{.Tag}}";
    eprintln!("$ docker images --format '{fmt}' {reference}");
    let output = Command::new("docker")
        .args(["images", "--format", fmt, &reference])
        .output()
        .map_err(|e| format!("failed to run docker: {e}"))?;

    let stdout = String::from_utf8_lossy(&output.stdout);
    Ok(stdout
        .lines()
        .map(|s| s.to_string())
        .filter(|s| !s.is_empty())
        .collect())
}

/// Read a label from a Docker image
/// Docker: docker inspect --format '{{index .Config.Labels "label"}}' image
fn get_label(image: &str, label: &str) -> String {
    let format_str = format!("{{{{index .Config.Labels \"{label}\"}}}}");
    Command::new("docker")
        .args(["inspect", "--format", &format_str, image])
        .output()
        .ok()
        .and_then(|o| {
            let s = String::from_utf8_lossy(&o.stdout).trim().to_string();
            if s.is_empty() || s == "<no value>" {
                None
            } else {
                Some(s)
            }
        })
        .unwrap_or_else(|| "-".to_string())
}