vtcode-eval 0.173.1

Agent evaluation framework for VT Code: pass@k / pass^k metrics, capability and regression evals, and environment-based outcome verification.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

pub mod environment;
pub mod executor;
pub mod metric;
pub mod report;
pub mod suite;
pub mod task;
pub mod test_quality;
pub mod trace_analyzer;

pub use environment::{CommandProbe, EnvironmentProbe, FileExistsProbe, GitCleanProbe};
pub use executor::{EvalExecutor, EvalRunOptions, run_suite, run_suite_with_options};
pub use metric::{
    EvalMetric, aggregate_metrics, compute_metric, compute_metric_with_k, pass_all_k, pass_at_k, pass_at_k_with_k,
    pass_power_k,
};
pub use report::{CostEfficiency, EvalReport, SuiteReport, TaskReport, build_task_report, priced_cost};
pub use suite::EvalSuite;
pub use task::{EvalCategory, EvalRunResult, EvalTask, RunOutcome};
pub use test_quality::{TestQualitySummary, analyze_test_source};
pub use trace_analyzer::{
    HarnessTraceSummary, LatencyStatistics, TokenUsage, analyze_jsonl, analyze_jsonl_file, analyze_jsonl_reader,
};