speccoder_runtime/
held_out.rs

1//! Held-out acceptance tests: written by a second author call
2//! after the visible ones, kept in the state directory — never in the
3//! worktree, never in an implementer's input — red-checked on the
4//! checkpoint, and run over the final tree after an attempt that passed
5//! visibly. Only the scenario IDs of a failure reach a retry.
6
7use std::collections::BTreeSet;
8use std::path::{Path, PathBuf};
9
10use serde::{Deserialize, Serialize};
11use speccoder_core::verify::{CheckResult, RunnerSummary};
12
13use crate::git::Git;
14use crate::integrity::{JudgeSnapshot, sha256_hex};
15use crate::tools::policy::Policy;
16use crate::{sandbox, worktree};
17
18/// Held-out files per task, composition tests included.
19pub const LIMIT: usize = 4;
20
21/// One kept held-out test file.
22#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
23pub struct HeldOutFile {
24    /// Workspace-relative path it is written to for a run.
25    pub path: String,
26    /// SHA-256 of its content.
27    pub sha256: String,
28    /// The scenarios it covers.
29    pub scenarios: Vec<String>,
30    /// The command that runs only it.
31    pub argv: Vec<String>,
32    /// It exercises several scenarios together.
33    pub composition: bool,
34}
35
36/// A file the author wrote that was not kept, and why.
37#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
38pub struct Discarded {
39    /// Its path as given.
40    pub path: String,
41    /// `unknown_scenario`, `not_covered_by_visible`, `path`, `command`,
42    /// `not_red`, `collision` or `limit`.
43    pub reason: String,
44}
45
46/// What judged a task, stored beside the files' contents.
47#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
48pub struct Manifest {
49    /// Format version.
50    pub version: u32,
51    /// The feature (or change).
52    pub feature: String,
53    /// The task.
54    pub task: String,
55    /// The author run.
56    pub author_run: String,
57    /// The tree the red runs used.
58    pub checkpoint: String,
59    /// Scenarios this task's acceptance tests cover.
60    pub visible_scenarios: Vec<String>,
61    /// Kept files.
62    pub files: Vec<HeldOutFile>,
63    /// Files not kept.
64    pub discarded: Vec<Discarded>,
65}
66
67impl Manifest {
68    /// SHA-256 of the canonical JSON: what the ticking commit names.
69    #[must_use]
70    pub fn sha256(&self) -> String {
71        sha256_hex(&serde_json::to_vec(self).unwrap_or_default())
72    }
73}
74
75/// Where a task's held-out set lives: `<project state>/held-out/<feature>/<task>`.
76#[must_use]
77pub fn dir(project_state: &Path, feature: &str, task: &str) -> PathBuf {
78    project_state.join("held-out").join(feature).join(task)
79}
80
81/// Stores the manifest and the kept files' contents; returns the
82/// manifest's hash. A previous set for the task is replaced.
83pub fn store(
84    dir: &Path,
85    manifest: &Manifest,
86    contents: &[(String, String)],
87) -> Result<String, String> {
88    let _ = std::fs::remove_dir_all(dir);
89    for (path, content) in contents {
90        let file = dir.join("files").join(path);
91        if let Some(parent) = file.parent() {
92            std::fs::create_dir_all(parent).map_err(|e| format!("{}: {e}", parent.display()))?;
93        }
94        std::fs::write(&file, content).map_err(|e| format!("{}: {e}", file.display()))?;
95    }
96    std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
97    let text = serde_json::to_string_pretty(manifest).map_err(|e| e.to_string())?;
98    std::fs::write(dir.join("manifest.json"), text).map_err(|e| e.to_string())?;
99    Ok(manifest.sha256())
100}
101
102/// A stored set that no longer matches what was recorded.
103#[derive(Debug, Clone, PartialEq, Eq)]
104pub struct Tampered(pub String);
105
106/// The stored set, checked: the manifest must hash to `expected` and every
107/// file to its recorded hash.
108pub fn load_verified(
109    dir: &Path,
110    expected: &str,
111) -> Result<(Manifest, Vec<(String, String)>), Tampered> {
112    let text = std::fs::read_to_string(dir.join("manifest.json"))
113        .map_err(|e| Tampered(format!("the held-out manifest is unreadable: {e}")))?;
114    let manifest: Manifest = serde_json::from_str(&text)
115        .map_err(|e| Tampered(format!("the held-out manifest is not valid: {e}")))?;
116    if manifest.sha256() != expected {
117        return Err(Tampered(
118            "the held-out manifest changed since it was written".to_owned(),
119        ));
120    }
121    let mut contents = Vec::new();
122    for f in &manifest.files {
123        let content = std::fs::read_to_string(dir.join("files").join(&f.path))
124            .map_err(|_| Tampered(format!("held-out file {} is missing", f.path)))?;
125        if sha256_hex(content.as_bytes()) != f.sha256 {
126            return Err(Tampered(format!("held-out file {} changed", f.path)));
127        }
128        contents.push((f.path.clone(), content));
129    }
130    Ok((manifest, contents))
131}
132
133/// One held-out file's run.
134#[derive(Debug, Clone)]
135pub struct Ran {
136    /// Its path.
137    pub path: String,
138    /// The run, or `None` when its path already existed (not overwritten).
139    pub result: Option<(CheckResult, Option<RunnerSummary>)>,
140}
141
142/// Writes each held-out file over the worktree, runs its command, and puts
143/// the tree back exactly as it was: no file of the set, nothing a test wrote,
144/// no changed judge input survives — a later attempt can read none of it.
145pub fn overlay_run(
146    wt_root: &Path,
147    project: &Path,
148    tmp: &Path,
149    files: &[(HeldOutFile, String)],
150    runner: &sandbox::Runner,
151    policy: &Policy,
152) -> Result<Vec<Ran>, String> {
153    let before = worktree::checkpoint(wt_root, tmp).map_err(|e| e.0)?;
154    let judged = JudgeSnapshot::take(wt_root, policy);
155    let mut out = Vec::new();
156    for (f, content) in files {
157        let full = project.join(&f.path);
158        if full.exists() {
159            out.push(Ran {
160                path: f.path.clone(),
161                result: None,
162            });
163            continue;
164        }
165        if let Some(parent) = full.parent() {
166            let _ = std::fs::create_dir_all(parent);
167        }
168        std::fs::write(&full, content).map_err(|e| format!("{}: {e}", f.path))?;
169        let ran = crate::pipeline::acceptance_run(project, &f.argv, runner);
170        let _ = std::fs::remove_file(&full);
171        out.push(Ran {
172            path: f.path.clone(),
173            result: Some(ran),
174        });
175    }
176    worktree::restore(wt_root, tmp, &before).map_err(|e| e.0)?;
177    // What a test wrote to its private scratch would outlive the run.
178    if let Some(scratch) = runner.scratch() {
179        let _ = std::fs::remove_dir_all(&scratch);
180    }
181    let after = worktree::checkpoint(wt_root, tmp).map_err(|e| e.0)?;
182    if after != before {
183        return Err("the held-out run left changes in the worktree".to_owned());
184    }
185    let drift = judged.drift(
186        &JudgeSnapshot::take(wt_root, policy),
187        policy,
188        &BTreeSet::new(),
189    );
190    if let Some(d) = drift.first() {
191        return Err(format!("the held-out run changed judge input {}", d.path));
192    }
193    Ok(out)
194}
195
196/// Literals a copied test would carry: quoted strings of 6+ characters and
197/// numbers with 4+ significant digits.
198#[must_use]
199pub fn literals(text: &str) -> BTreeSet<String> {
200    let mut out = BTreeSet::new();
201    for quote in ['"', '\''] {
202        let mut parts = text.split(quote);
203        parts.next();
204        while let (Some(inside), Some(_)) = (parts.next(), parts.next().or(Some(""))) {
205            if inside.chars().count() >= 6 && !inside.contains('\n') {
206                out.insert(inside.to_owned());
207            }
208        }
209    }
210    let mut number = String::new();
211    for c in text.chars().chain(std::iter::once(' ')) {
212        if c.is_ascii_digit() || (c == '.' && !number.is_empty()) {
213            number.push(c);
214        } else {
215            let n = number.trim_end_matches('.');
216            let significant = n
217                .trim_start_matches(['0', '.'])
218                .chars()
219                .filter(char::is_ascii_digit)
220                .count();
221            if significant >= 4 {
222                out.insert(n.to_owned());
223            }
224            number.clear();
225        }
226    }
227    out
228}
229
230/// Held-out literals in the attempt's added lines that it could only have
231/// copied: not in the spec, the visible tests or the tree it started from
232///. For a native attempt this never fires; for an external
233/// agent, which can read the state directory, it is the mechanical check.
234#[must_use]
235pub fn leaked(
236    held_out: &[(HeldOutFile, String)],
237    known: &str,
238    wt_root: &Path,
239    start_tree: &worktree::Checkpoint,
240    patch: &str,
241) -> Vec<String> {
242    let added: String = patch
243        .lines()
244        .filter(|l| l.starts_with('+') && !l.starts_with("+++"))
245        .collect::<Vec<_>>()
246        .join("\n");
247    let git = Git::at(wt_root);
248    held_out
249        .iter()
250        .flat_map(|(_, content)| literals(content))
251        .collect::<BTreeSet<_>>()
252        .into_iter()
253        .filter(|lit| added.contains(lit.as_str()) && !known.contains(lit.as_str()))
254        .filter(|lit| {
255            !git.succeeds(&["grep", "-q", "-F", "-e", lit, &start_tree.0])
256                .unwrap_or(true)
257        })
258        .collect()
259}
260
261/// Where a finished feature's held-out sets are published.
262pub const REF_PREFIX: &str = "refs/speccoder/held-out";
263
264/// Publishes a finished feature's held-out sets: one commit,
265/// parented on the feature branch's tip, with `<task>/manifest.json` and
266/// `<task>/files/…` of every task, under `refs/speccoder/held-out/<feature>`.
267/// Only once the feature is done — earlier, a later task could read them.
268/// Never pushed by speccoder itself. `Ok(None)` when no task has a set.
269pub fn publish(
270    repo: &Path,
271    project_state: &Path,
272    feature: &str,
273    tmp: &Path,
274) -> Result<Option<String>, String> {
275    let base = project_state.join("held-out").join(feature);
276    let mut files: Vec<(String, PathBuf)> = Vec::new();
277    let mut stack = vec![base.clone()];
278    while let Some(d) = stack.pop() {
279        let Ok(entries) = std::fs::read_dir(&d) else {
280            continue;
281        };
282        for e in entries.flatten() {
283            let p = e.path();
284            if p.is_dir() {
285                stack.push(p);
286            } else if let Ok(rel) = p.strip_prefix(&base) {
287                files.push((rel.to_string_lossy().replace('\\', "/"), p.clone()));
288            }
289        }
290    }
291    if files.is_empty() {
292        return Ok(None);
293    }
294    files.sort();
295    let git = Git::at(repo);
296    std::fs::create_dir_all(tmp).map_err(|e| e.to_string())?;
297    let index = tmp.join(format!("held-out-index-{}", std::process::id()));
298    let _ = std::fs::remove_file(&index);
299    for (rel, path) in &files {
300        let blob = git
301            .line(&["hash-object", "-w", "--", &path.to_string_lossy()])
302            .map_err(|e| e.0)?;
303        git.run_with_index(
304            &index,
305            &[
306                "update-index",
307                "--add",
308                "--cacheinfo",
309                &format!("100644,{blob},{rel}"),
310            ],
311            None,
312        )
313        .map_err(|e| e.0)?;
314    }
315    let tree = git
316        .run_with_index(&index, &["write-tree"], None)
317        .map_err(|e| e.0)?;
318    let _ = std::fs::remove_file(&index);
319    let tree = String::from_utf8_lossy(&tree).trim().to_owned();
320    let branch = format!("speccoder/{feature}");
321    let parent = git
322        .line(&["rev-parse", "--verify", &format!("{branch}^{{commit}}")])
323        .map_err(|e| e.0)?;
324    let message = format!(
325        "speccoder: held-out tests of {feature}\n\nThe tests that judged each task, published once the feature is done.\nEach <task>/manifest.json hashes to its task's Held-Out trailer.\n\nSpec-Feature: {feature}\n"
326    );
327    let mut args = crate::attempts::IDENTITY.to_vec();
328    args.extend(["commit-tree", &tree, "-p", &parent, "-m", &message]);
329    let commit = git.line(&args).map_err(|e| e.0)?;
330    git.run(&["update-ref", &format!("{REF_PREFIX}/{feature}"), &commit])
331        .map_err(|e| e.0)?;
332    Ok(Some(commit))
333}
334
335#[cfg(test)]
336mod tests {
337    use super::*;
338
339    #[test]
340    fn literals_are_long_strings_and_precise_numbers() {
341        let l = literals(
342            "assert_eq!(slug(\"Hello, World\"), \"hello-world\"); x(3.14159); y(42); z('ab')",
343        );
344        assert!(l.contains("Hello, World") && l.contains("hello-world"));
345        assert!(l.contains("3.14159"));
346        assert!(!l.contains("42") && !l.contains("ab"));
347    }
348
349    #[test]
350    fn a_changed_file_or_manifest_is_tampering() {
351        let d = tempfile::tempdir().unwrap();
352        let content = "fn t() {}\n".to_owned();
353        let m = Manifest {
354            version: 1,
355            feature: "f".to_owned(),
356            task: "T1".to_owned(),
357            author_run: "r".to_owned(),
358            checkpoint: "c".to_owned(),
359            visible_scenarios: vec!["US1-1".to_owned()],
360            files: vec![HeldOutFile {
361                path: "tests/h.rs".to_owned(),
362                sha256: sha256_hex(content.as_bytes()),
363                scenarios: vec!["US1-1".to_owned()],
364                argv: vec!["true".to_owned()],
365                composition: false,
366            }],
367            discarded: Vec::new(),
368        };
369        let sha = store(d.path(), &m, &[("tests/h.rs".to_owned(), content)]).unwrap();
370        assert_eq!(load_verified(d.path(), &sha).unwrap().1.len(), 1);
371        assert!(load_verified(d.path(), "other").is_err());
372        std::fs::write(d.path().join("files/tests/h.rs"), "fn t() { changed }").unwrap();
373        assert!(
374            load_verified(d.path(), &sha)
375                .unwrap_err()
376                .0
377                .contains("changed")
378        );
379        std::fs::remove_file(d.path().join("files/tests/h.rs")).unwrap();
380        assert!(
381            load_verified(d.path(), &sha)
382                .unwrap_err()
383                .0
384                .contains("missing")
385        );
386    }
387}
speccoder_runtime/held_out.rs

speccoder_runtime/
held_out.rs
