lean_ctx/core/eval_ab/
suite.rs

1//! Eval suite + fixtures (#233): the deterministic task definitions an A/B run scores.
2//!
3//! A *suite* is an NDJSON file (one [`Task`] per line, `#`-comments + blank lines allowed).
4//! Each task carries everything the harness needs to (a) assemble context from a workspace,
5//! (b) prompt the pinned model, and (c) score the answer objectively. Two domains are
6//! supported today: free-form [`Domain::Qa`] (scored with EM / F1 / containment) and
7//! [`Domain::Code`] (scored by running a unit-test command against the model output).
8
9use std::path::{Path, PathBuf};
10
11use anyhow::{bail, Context, Result};
12use serde::{Deserialize, Serialize};
13
14/// What kind of task this is — selects the scorer and how the model output is interpreted.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "snake_case")]
17pub enum Domain {
18    /// Retrieval-augmented question answering, scored with EM / F1 / containment.
19    Qa,
20    /// Code task, scored by running a unit-test command against the model's output.
21    Code,
22}
23
24impl Domain {
25    /// Stable lowercase label used in digests and reports.
26    pub fn label(self) -> &'static str {
27        match self {
28            Domain::Qa => "qa",
29            Domain::Code => "code",
30        }
31    }
32}
33
34/// One scored unit of work. Fixtures are stored as NDJSON (one task per line) so suites are
35/// diff-friendly and stream without loading the whole file into a single JSON value.
36#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
37pub struct Task {
38    /// Stable, unique identifier (used in reports + the determinism digest).
39    pub id: String,
40    /// Selects the scorer and the meaning of the remaining fields.
41    pub domain: Domain,
42    /// The instruction shown to the model (the "user turn").
43    pub prompt: String,
44    /// Repo / corpus directory the context is assembled from. Relative paths resolve against
45    /// the suite file's parent directory; absolute paths are used as-is.
46    pub workspace: String,
47    /// Query used to retrieve context in the lean-ctx condition. Defaults to `prompt`.
48    #[serde(default, skip_serializing_if = "Option::is_none")]
49    pub retrieval_query: Option<String>,
50
51    // --- Domain::Qa --------------------------------------------------------
52    /// Accepted gold answers. Any match counts (EM/F1 take the best over this set).
53    #[serde(default, skip_serializing_if = "Vec::is_empty")]
54    pub answers: Vec<String>,
55
56    // --- Domain::Code ------------------------------------------------------
57    /// File inside a sandbox copy of `workspace` that the model output replaces.
58    #[serde(default, skip_serializing_if = "Option::is_none")]
59    pub target_file: Option<String>,
60    /// Shell command run inside the sandbox; exit code 0 = pass.
61    #[serde(default, skip_serializing_if = "Option::is_none")]
62    pub test_cmd: Option<String>,
63}
64
65impl Task {
66    /// The retrieval query for the lean-ctx condition (falls back to the prompt).
67    pub fn query(&self) -> &str {
68        self.retrieval_query.as_deref().unwrap_or(&self.prompt)
69    }
70
71    /// Absolute workspace directory, resolved against `suite_dir` for relative paths.
72    pub fn workspace_path(&self, suite_dir: &Path) -> PathBuf {
73        let p = Path::new(&self.workspace);
74        if p.is_absolute() {
75            p.to_path_buf()
76        } else {
77            suite_dir.join(p)
78        }
79    }
80
81    /// Validates the per-domain invariants. Returns a human-readable reason on failure.
82    fn validate(&self) -> std::result::Result<(), String> {
83        if self.id.trim().is_empty() {
84            return Err("task id is empty".into());
85        }
86        if self.prompt.trim().is_empty() {
87            return Err(format!("task {}: prompt is empty", self.id));
88        }
89        if self.workspace.trim().is_empty() {
90            return Err(format!("task {}: workspace is empty", self.id));
91        }
92        match self.domain {
93            Domain::Qa => {
94                if self.answers.iter().all(|a| a.trim().is_empty()) {
95                    return Err(format!(
96                        "task {}: qa task has no non-empty answers",
97                        self.id
98                    ));
99                }
100            }
101            Domain::Code => {
102                if self.target_file.as_deref().unwrap_or("").trim().is_empty() {
103                    return Err(format!("task {}: code task needs target_file", self.id));
104                }
105                if self.test_cmd.as_deref().unwrap_or("").trim().is_empty() {
106                    return Err(format!("task {}: code task needs test_cmd", self.id));
107                }
108            }
109        }
110        Ok(())
111    }
112}
113
114/// A loaded, validated suite: the tasks plus the directory used to resolve relative workspaces.
115#[derive(Debug, Clone)]
116pub struct EvalSuite {
117    /// Directory of the suite file (the resolution root for relative workspaces).
118    pub dir: PathBuf,
119    /// Validated tasks in file order.
120    pub tasks: Vec<Task>,
121}
122
123impl EvalSuite {
124    /// Parses + validates an NDJSON suite file.
125    pub fn load(path: &Path) -> Result<Self> {
126        let raw = std::fs::read_to_string(path)
127            .with_context(|| format!("reading suite {}", path.display()))?;
128        let dir = path
129            .parent()
130            .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
131        Self::parse(&raw, dir)
132    }
133
134    /// Pure parser (testable without touching disk for the suite body itself).
135    pub fn parse(raw: &str, dir: PathBuf) -> Result<Self> {
136        let mut tasks = Vec::new();
137        for (lineno, line) in raw.lines().enumerate() {
138            let trimmed = line.trim();
139            if trimmed.is_empty() || trimmed.starts_with('#') {
140                continue;
141            }
142            let task: Task = serde_json::from_str(trimmed)
143                .with_context(|| format!("parsing task on line {}", lineno + 1))?;
144            if let Err(reason) = task.validate() {
145                bail!("invalid task on line {}: {reason}", lineno + 1);
146            }
147            tasks.push(task);
148        }
149        if tasks.is_empty() {
150            bail!("suite contains no tasks");
151        }
152        // Unique ids keep the determinism digest unambiguous.
153        let mut seen = std::collections::HashSet::new();
154        for t in &tasks {
155            if !seen.insert(t.id.as_str()) {
156                bail!("duplicate task id: {}", t.id);
157            }
158        }
159        Ok(Self { dir, tasks })
160    }
161}
162
163#[cfg(test)]
164mod tests {
165    use super::*;
166
167    fn qa_line() -> &'static str {
168        r#"{"id":"q1","domain":"qa","prompt":"What stores does consolidation write to?","workspace":"corpus","answers":["bm25, graph, knowledge, session"]}"#
169    }
170
171    fn code_line() -> &'static str {
172        r#"{"id":"c1","domain":"code","prompt":"Implement add","workspace":"code","target_file":"solution.sh","test_cmd":"sh test.sh"}"#
173    }
174
175    #[test]
176    fn parses_qa_and_code_skipping_comments_and_blanks() {
177        let raw = format!("# header\n\n{}\n{}\n", qa_line(), code_line());
178        let suite = EvalSuite::parse(&raw, PathBuf::from("/suites")).unwrap();
179        assert_eq!(suite.tasks.len(), 2);
180        assert_eq!(suite.tasks[0].domain, Domain::Qa);
181        assert_eq!(suite.tasks[1].domain, Domain::Code);
182        assert_eq!(suite.tasks[0].query(), suite.tasks[0].prompt);
183    }
184
185    #[test]
186    fn relative_workspace_resolves_against_suite_dir() {
187        let suite = EvalSuite::parse(qa_line(), PathBuf::from("/suites")).unwrap();
188        assert_eq!(
189            suite.tasks[0].workspace_path(&suite.dir),
190            PathBuf::from("/suites/corpus")
191        );
192    }
193
194    #[test]
195    fn rejects_qa_without_answers() {
196        let bad = r#"{"id":"q","domain":"qa","prompt":"p","workspace":"w"}"#;
197        assert!(EvalSuite::parse(bad, PathBuf::from(".")).is_err());
198    }
199
200    #[test]
201    fn rejects_code_without_test_cmd() {
202        let bad = r#"{"id":"c","domain":"code","prompt":"p","workspace":"w","target_file":"f"}"#;
203        assert!(EvalSuite::parse(bad, PathBuf::from(".")).is_err());
204    }
205
206    #[test]
207    fn rejects_duplicate_ids() {
208        let raw = format!("{}\n{}", qa_line(), qa_line());
209        assert!(EvalSuite::parse(&raw, PathBuf::from(".")).is_err());
210    }
211
212    #[test]
213    fn rejects_empty_suite() {
214        assert!(EvalSuite::parse("# only comments\n\n", PathBuf::from(".")).is_err());
215    }
216}
lean_ctx/core/eval_ab/suite.rs

lean_ctx/core/eval_ab/
suite.rs
