forge_engine/lab/
suite.rs

1use std::path::{Path, PathBuf};
2
3use serde::{Deserialize, Serialize};
4
5use crate::error::{ForgeError, ForgeResult};
6
7/// A task loaded from a fixture directory.
8#[derive(Debug, Clone, Serialize, Deserialize)]
9pub struct EvalTask {
10    pub task_id: String,
11    pub prompt: String,
12    pub constraints: TaskConstraints,
13    pub weights: TaskWeights,
14    pub expected: TaskExpected,
15    pub cea: TaskCea,
16}
17
18#[derive(Debug, Clone, Serialize, Deserialize)]
19pub struct TaskConstraints {
20    #[serde(default)]
21    pub allow_test_modifications: bool,
22    pub max_files_changed: Option<usize>,
23}
24
25#[derive(Debug, Clone, Serialize, Deserialize)]
26pub struct TaskWeights {
27    #[serde(default = "default_correctness_weight")]
28    pub correctness: f64,
29    #[serde(default = "default_novelty_weight")]
30    pub novelty: f64,
31    #[serde(default = "default_stability_weight")]
32    pub stability: f64,
33}
34
35fn default_correctness_weight() -> f64 {
36    0.7
37}
38fn default_novelty_weight() -> f64 {
39    0.2
40}
41fn default_stability_weight() -> f64 {
42    0.1
43}
44
45#[derive(Debug, Clone, Serialize, Deserialize)]
46pub struct TaskExpected {
47    #[serde(default = "default_true")]
48    pub require_fmt: bool,
49    #[serde(default = "default_true")]
50    pub require_clippy: bool,
51    #[serde(default = "default_true")]
52    pub require_tests: bool,
53}
54
55fn default_true() -> bool {
56    true
57}
58
59#[derive(Debug, Clone, Serialize, Deserialize)]
60pub struct TaskCea {
61    #[serde(default = "default_true")]
62    pub instrument: bool,
63    pub risk_threshold_override: Option<f64>,
64}
65
66/// A loaded fixture suite.
67#[derive(Debug)]
68pub struct EvalSuite {
69    pub name: String,
70    pub tasks: Vec<LoadedTask>,
71}
72
73/// A task with its fixture path.
74#[derive(Debug)]
75pub struct LoadedTask {
76    pub task: EvalTask,
77    pub fixture_path: PathBuf,
78}
79
80/// Load an evaluation suite from a fixtures directory.
81pub fn load_suite(suite_dir: &Path) -> ForgeResult<EvalSuite> {
82    let suite_name = suite_dir
83        .file_name()
84        .and_then(|n| n.to_str())
85        .unwrap_or("unknown")
86        .to_string();
87
88    let mut tasks = Vec::new();
89
90    if !suite_dir.exists() {
91        return Err(ForgeError::Fixture(format!(
92            "suite directory does not exist: {}",
93            suite_dir.display()
94        )));
95    }
96
97    // Each subdirectory is a task
98    let mut entries: Vec<_> = std::fs::read_dir(suite_dir)?
99        .filter_map(|e| e.ok())
100        .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false))
101        .collect();
102    entries.sort_by_key(|e| e.file_name());
103
104    for entry in entries {
105        let task_dir = entry.path();
106        let task_json = task_dir.join("task.json");
107        let repo_dir = task_dir.join("repo");
108
109        if !task_json.exists() {
110            tracing::warn!("skipping fixture {}: no task.json", task_dir.display());
111            continue;
112        }
113
114        if !repo_dir.exists() {
115            tracing::warn!(
116                "skipping fixture {}: no repo/ directory",
117                task_dir.display()
118            );
119            continue;
120        }
121
122        // Validate: repo must have Cargo.toml
123        if !repo_dir.join("Cargo.toml").exists() {
124            tracing::warn!(
125                "skipping fixture {}: repo/ has no Cargo.toml",
126                task_dir.display()
127            );
128            continue;
129        }
130
131        let task_content = std::fs::read_to_string(&task_json).map_err(|e| {
132            ForgeError::Fixture(format!("cannot read {}: {e}", task_json.display()))
133        })?;
134
135        let task: EvalTask = serde_json::from_str(&task_content).map_err(|e| {
136            ForgeError::Fixture(format!("cannot parse {}: {e}", task_json.display()))
137        })?;
138
139        tasks.push(LoadedTask {
140            task,
141            fixture_path: repo_dir,
142        });
143    }
144
145    Ok(EvalSuite {
146        name: suite_name,
147        tasks,
148    })
149}
forge_engine/lab/suite.rs

forge_engine/lab/
suite.rs
