eval-containers 0.1.0

A build system for AI agent evaluations
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

[package]
name = "eval-containers"
version = "0.1.0"
edition = "2024"
rust-version = "1.85"
description = "A build system for AI agent evaluations"
authors = ["The Exgentic organization and its contributors"]
license = "Apache-2.0"
repository = "https://github.com/Exgentic/eval-containers"
homepage = "https://github.com/Exgentic/eval-containers"
readme = "README.md"
keywords = ["evaluation", "benchmark", "llm", "agents", "docker"]
categories = ["command-line-utilities", "development-tools::build-utils"]
# Publish only the Rust crate, not the 100-benchmark / 20-agent monorepo
# around it. Without this, `cargo publish` bundles every benchmarks/,
# agents/, models/, compose/ tree into the tarball and blows crates.io's
# 10 MB limit. Patterns are gitignore-style: the LEADING SLASH anchors each
# to the package root — without it a bare `README.md` matches every
# benchmarks/*/README.md at any depth. The binary embeds nothing at compile
# time (no include_str!/include_bytes!), so src/ + manifests are enough.
include = ["/src/**/*.rs", "/Cargo.toml", "/Cargo.lock", "/README.md", "/LICENSE"]

[dependencies]
clap = { version = "4", features = ["derive", "env"] }
# `dotenvy::dotenv()` walks up from cwd to find a `.env` file and loads
# its contents into the process env without overwriting existing keys —
# the same well-trodden pattern as Node's `dotenv`. Called from main()
# so every `eval-containers` invocation auto-picks up the repo's `.env`
# (HF_TOKEN, OPENAI_API_KEY, …). Tests inherit this behavior when they
# shell out to the CLI; nothing test-specific to maintain.
dotenvy = "0.15"
serde = { version = "1", features = ["derive"] }
serde_json = "1"

[dev-dependencies]
testcontainers = { version = "0.27", features = ["docker-compose", "http_wait"] }
tokio = { version = "1", features = ["full"] }
tempfile = "3"
# HTTP client for the gateway test suite. The tests POST real
# /openai/v1/chat/completions, /anthropic/v1/messages, and
# /genai/v1beta/.../generateContent against gateway containers started
# by testcontainers — the only way to verify the protocol-translation
# matrix from the outside. rustls-tls avoids dragging openssl into the
# test build; testcontainers' internal reqwest also uses rustls so no
# duplicated TLS stack.
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }

# Integration tests are organized into category subfolders under
# `tests/<category>/`. Each RULES.md in that subfolder describes what
# testing means for that category. See tests/RULES.md for the overall
# strategy. Test names are preserved (e.g. `cargo test --test build`
# still works) so CI scripts and docs don't need to change.

[[test]]
name = "check"
path = "tests/sanity/check.rs"

[[test]]
name = "oracle"
path = "tests/oracle/test.rs"

[[test]]
name = "cli_conformance"
path = "tests/cli/conformance.rs"

[[test]]
name = "compose"
path = "tests/sanity/compose.rs"

[[test]]
name = "dockerfile_inspection"
path = "tests/sanity/dockerfile_inspection.rs"

[[test]]
name = "task_inspection"
path = "tests/sanity/task_inspection.rs"

[[test]]
name = "build"
path = "tests/build/test.rs"

[[test]]
name = "replay"
path = "tests/replay/test.rs"

[[test]]
name = "upstream"
path = "tests/upstream/test.rs"

[[test]]
name = "fleet"
path = "tests/fleet/test.rs"

[[test]]
name = "live"
path = "tests/live/test.rs"

[[test]]
name = "gateways"
path = "tests/gateways/test.rs"

[[test]]
name = "agents"
path = "tests/agents/test.rs"

# The profile that 'dist' will build with
[profile.dist]
inherits = "release"
lto = "thin"