use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use speccoder_core::verify::{CheckResult, RunnerSummary};
use crate::git::Git;
use crate::integrity::{JudgeSnapshot, sha256_hex};
use crate::tools::policy::Policy;
use crate::{sandbox, worktree};
pub const LIMIT: usize = 4;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HeldOutFile {
pub path: String,
pub sha256: String,
pub scenarios: Vec<String>,
pub argv: Vec<String>,
pub composition: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub red_tests: Option<usize>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Discarded {
pub path: String,
pub reason: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Manifest {
pub version: u32,
pub feature: String,
pub task: String,
pub author_run: String,
pub checkpoint: String,
pub visible_scenarios: Vec<String>,
pub files: Vec<HeldOutFile>,
pub discarded: Vec<Discarded>,
}
impl Manifest {
#[must_use]
pub fn sha256(&self) -> String {
sha256_hex(&serde_json::to_vec(self).unwrap_or_default())
}
}
#[must_use]
pub fn dir(project_state: &Path, feature: &str, task: &str) -> PathBuf {
project_state.join("held-out").join(feature).join(task)
}
pub fn store(
dir: &Path,
manifest: &Manifest,
contents: &[(String, String)],
) -> Result<String, String> {
let _ = std::fs::remove_dir_all(dir);
for (path, content) in contents {
let file = dir.join("files").join(path);
if let Some(parent) = file.parent() {
std::fs::create_dir_all(parent).map_err(|e| format!("{}: {e}", parent.display()))?;
}
std::fs::write(&file, content).map_err(|e| format!("{}: {e}", file.display()))?;
}
std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
let text = serde_json::to_string_pretty(manifest).map_err(|e| e.to_string())?;
std::fs::write(dir.join("manifest.json"), text).map_err(|e| e.to_string())?;
Ok(manifest.sha256())
}
#[must_use]
pub fn sealed_dir(project_state: &Path, unit: &str, key: &str) -> PathBuf {
project_state
.join("held-out")
.join(unit)
.join(format!("req-{}", key.replace([':', '/'], "-")))
}
pub fn seal(
project_state: &Path,
unit: &str,
key: &str,
manifest: &Manifest,
contents: &[(String, String)],
) -> Result<speccoder_core::acceptance::SealedRef, String> {
let sha = store(&sealed_dir(project_state, unit, key), manifest, contents)?;
Ok(speccoder_core::acceptance::SealedRef {
key: key.to_owned(),
manifest_sha256: sha,
count: manifest.files.len(),
})
}
pub fn open_sealed(
project_state: &Path,
unit: &str,
sealed: &speccoder_core::acceptance::SealedRef,
) -> Result<(Manifest, Vec<(String, String)>), Tampered> {
load_verified(
&sealed_dir(project_state, unit, &sealed.key),
&sealed.manifest_sha256,
)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Tampered(pub String);
pub fn load_verified(
dir: &Path,
expected: &str,
) -> Result<(Manifest, Vec<(String, String)>), Tampered> {
let text = std::fs::read_to_string(dir.join("manifest.json"))
.map_err(|e| Tampered(format!("the held-out manifest is unreadable: {e}")))?;
let manifest: Manifest = serde_json::from_str(&text)
.map_err(|e| Tampered(format!("the held-out manifest is not valid: {e}")))?;
if manifest.sha256() != expected {
return Err(Tampered(
"the held-out manifest changed since it was written".to_owned(),
));
}
let mut contents = Vec::new();
for f in &manifest.files {
let content = std::fs::read_to_string(dir.join("files").join(&f.path))
.map_err(|_| Tampered(format!("held-out file {} is missing", f.path)))?;
if sha256_hex(content.as_bytes()) != f.sha256 {
return Err(Tampered(format!("held-out file {} changed", f.path)));
}
contents.push((f.path.clone(), content));
}
Ok((manifest, contents))
}
#[derive(Debug, Clone)]
pub struct Ran {
pub path: String,
pub result: Option<(CheckResult, Option<RunnerSummary>)>,
pub without: Option<(CheckResult, Option<RunnerSummary>)>,
}
const OVERLAY_MARKER: &str = "speccoder-held-out-overlay.json";
fn overlay_marker(wt_root: &Path) -> Option<PathBuf> {
crate::git::Git::at(wt_root)
.line(&["rev-parse", "--absolute-git-dir"])
.ok()
.filter(|d| !d.is_empty())
.map(|d| PathBuf::from(d).join(OVERLAY_MARKER))
}
#[must_use]
pub fn overlaid(wt_root: &Path) -> Option<Vec<String>> {
let Some(marker) = overlay_marker(wt_root) else {
return Some(Vec::new());
};
match std::fs::read_to_string(&marker) {
Ok(t) => serde_json::from_str(&t).ok(),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Some(Vec::new()),
Err(_) => None,
}
}
#[must_use]
pub fn overlay_key(rel: &str) -> String {
sha256_hex(rel.as_bytes())
}
struct OverlayMarker(Option<PathBuf>);
impl Drop for OverlayMarker {
fn drop(&mut self) {
if let Some(m) = &self.0 {
let _ = std::fs::remove_file(m);
}
}
}
pub fn overlay_run(
wt_root: &Path,
project: &Path,
tmp: &Path,
files: &[(HeldOutFile, String)],
runner: &sandbox::Runner,
policy: &Policy,
) -> Result<Vec<Ran>, String> {
overlay(wt_root, project, tmp, files, runner, policy, false)
}
type Outcome = (CheckResult, Option<RunnerSummary>);
pub fn overlay_red(
wt_root: &Path,
project: &Path,
tmp: &Path,
files: &[(HeldOutFile, String)],
runner: &sandbox::Runner,
policy: &Policy,
) -> Result<Vec<Ran>, String> {
overlay(wt_root, project, tmp, files, runner, policy, true)
}
#[must_use]
pub fn red_class(ran: &Ran) -> speccoder_core::verify::RedClass {
use speccoder_core::verify::{RedClass, classify_red, classify_red_added};
match (&ran.result, &ran.without) {
(None, _) => RedClass::Unclassified,
(Some((r, s)), Some((o, os))) => classify_red_added((r, s.as_ref()), (o, os.as_ref())),
(Some((r, s)), None) => classify_red(r, s.as_ref()),
}
}
fn overlay(
wt_root: &Path,
project: &Path,
tmp: &Path,
files: &[(HeldOutFile, String)],
runner: &sandbox::Runner,
policy: &Policy,
baseline: bool,
) -> Result<Vec<Ran>, String> {
let before = worktree::checkpoint(wt_root, tmp).map_err(|e| e.0)?;
let judged = JudgeSnapshot::take(wt_root, policy);
let prefix = {
let canonical = |p: &Path| std::fs::canonicalize(p).unwrap_or_else(|_| p.to_path_buf());
canonical(project)
.strip_prefix(canonical(wt_root))
.map(|r| r.to_string_lossy().replace('\\', "/"))
.unwrap_or_default()
};
let mut keys: Vec<String> = Vec::new();
for (f, _) in files {
let rel = if prefix.is_empty() {
f.path.clone()
} else {
format!("{prefix}/{}", f.path)
};
let mut at = rel.as_str();
loop {
let key = overlay_key(at);
if !keys.contains(&key) {
keys.push(key);
}
match at.rfind('/') {
Some(i) => at = &at[..i],
None => break,
}
}
}
let marker = OverlayMarker(overlay_marker(wt_root));
if let Some(m) = &marker.0 {
std::fs::write(m, serde_json::to_string(&keys).unwrap_or_default())
.map_err(|e| format!("the held-out overlay marker: {e}"))?;
}
let mut without: Vec<(Vec<String>, Outcome)> = Vec::new();
if baseline {
for (f, _) in files {
if !without.iter().any(|(argv, _)| *argv == f.argv) {
let ran = crate::pipeline::acceptance_run(project, &f.argv, runner);
without.push((f.argv.clone(), ran));
}
}
}
let without_of = |argv: &[String]| {
without
.iter()
.find(|(a, _)| a.as_slice() == argv)
.map(|(_, r)| r.clone())
};
let mut out = Vec::new();
for (f, content) in files {
let full = project.join(&f.path);
if full.exists() {
out.push(Ran {
path: f.path.clone(),
result: None,
without: None,
});
continue;
}
if let Some(parent) = full.parent() {
let _ = std::fs::create_dir_all(parent);
}
std::fs::write(&full, content).map_err(|e| format!("{}: {e}", f.path))?;
let ran = crate::pipeline::acceptance_run(project, &f.argv, runner);
let _ = std::fs::remove_file(&full);
out.push(Ran {
path: f.path.clone(),
result: Some(ran),
without: without_of(&f.argv),
});
}
worktree::restore(wt_root, tmp, &before).map_err(|e| e.0)?;
if let Some(scratch) = runner.scratch() {
let _ = std::fs::remove_dir_all(&scratch);
}
let after = worktree::checkpoint(wt_root, tmp).map_err(|e| e.0)?;
if after != before {
return Err("the held-out run left changes in the worktree".to_owned());
}
let drift = judged.drift(
&JudgeSnapshot::take(wt_root, policy),
policy,
&BTreeSet::new(),
);
if let Some(d) = drift.first() {
return Err(format!("the held-out run changed judge input {}", d.path));
}
Ok(out)
}
#[must_use]
pub fn literals(text: &str) -> BTreeSet<String> {
let mut out = BTreeSet::new();
for quote in ['"', '\''] {
let mut parts = text.split(quote);
parts.next();
while let (Some(inside), Some(_)) = (parts.next(), parts.next().or(Some(""))) {
if inside.chars().count() >= 6 && !inside.contains('\n') {
out.insert(inside.to_owned());
}
}
}
let mut number = String::new();
for c in text.chars().chain(std::iter::once(' ')) {
if c.is_ascii_digit() || (c == '.' && !number.is_empty()) {
number.push(c);
} else {
let n = number.trim_end_matches('.');
let significant = n
.trim_start_matches(['0', '.'])
.chars()
.filter(char::is_ascii_digit)
.count();
if significant >= 4 {
out.insert(n.to_owned());
}
number.clear();
}
}
out
}
#[must_use]
pub fn leaked(
held_out: &[(HeldOutFile, String)],
known: &str,
wt_root: &Path,
start_tree: &worktree::Checkpoint,
patch: &str,
) -> Vec<String> {
let added: String = patch
.lines()
.filter(|l| l.starts_with('+') && !l.starts_with("+++"))
.collect::<Vec<_>>()
.join("\n");
let git = Git::at(wt_root);
held_out
.iter()
.flat_map(|(_, content)| literals(content))
.collect::<BTreeSet<_>>()
.into_iter()
.filter(|lit| added.contains(lit.as_str()) && !known.contains(lit.as_str()))
.filter(|lit| {
!git.succeeds(&["grep", "-q", "-F", "-e", lit, &start_tree.0])
.unwrap_or(true)
})
.collect()
}
pub const REF_PREFIX: &str = "refs/speccoder/held-out";
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Selection {
pub sets: Vec<(String, PathBuf)>,
pub tasks: Vec<String>,
pub withheld: usize,
pub tampered: Vec<String>,
}
#[must_use]
pub fn named_digests(repo: &Path, commit: &str) -> Vec<String> {
let mut out: Vec<String> = crate::evidence::statements(repo, commit)
.iter()
.filter(|s| s["predicate"]["configuration"][0]["name"] == "speccoder:phase/held_out")
.filter_map(|s| {
s["predicate"]["configuration"][0]["digest"]["sha256"]
.as_str()
.map(str::to_owned)
})
.collect();
let trailer = Git::at(repo)
.line(&[
"log",
"-1",
"--format=%(trailers:key=Held-Out,valueonly)",
commit,
])
.unwrap_or_default();
out.extend(
trailer
.split_whitespace()
.filter_map(|w| w.strip_prefix("manifest=sha256:"))
.map(str::to_owned),
);
out.sort();
out.dedup();
out
}
#[must_use]
pub fn select_closed(
project_state: &Path,
unit: &str,
closed: &[(String, Vec<String>)],
) -> Selection {
let base = project_state.join("held-out").join(unit);
let mut stored: Vec<(String, PathBuf)> = std::fs::read_dir(&base)
.into_iter()
.flatten()
.flatten()
.filter(|e| e.path().join("manifest.json").is_file())
.map(|e| (e.file_name().to_string_lossy().into_owned(), e.path()))
.collect();
stored.sort();
let mut out = Selection::default();
for (key, dir) in stored {
let digest = std::fs::read_to_string(dir.join("manifest.json"))
.ok()
.and_then(|t| serde_json::from_str::<Manifest>(&t).ok())
.map(|m| m.sha256());
let named_by = digest.as_ref().and_then(|d| {
closed
.iter()
.find(|(_, digests)| digests.contains(d))
.map(|(task, _)| task.clone())
});
match (named_by, digest) {
(Some(task), Some(d)) => {
if load_verified(&dir, &d).is_ok() {
if !out.tasks.contains(&task) {
out.tasks.push(task);
}
out.sets.push((key, dir));
} else {
out.tampered.push(key);
}
}
_ => out.withheld += 1,
}
}
out
}
pub fn publication_store(project_state: &Path, repo: &Path) -> Result<PathBuf, String> {
let dir = project_state.join("held-out.git");
if !dir.join("HEAD").is_file() {
std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
Git::at(&dir)
.run(&["init", "-q", "--bare"])
.map_err(|e| e.0)?;
}
let common = Git::at(repo)
.line(&["rev-parse", "--path-format=absolute", "--git-common-dir"])
.map_err(|e| e.0)?;
let info = dir.join("objects/info");
std::fs::create_dir_all(&info).map_err(|e| format!("{}: {e}", info.display()))?;
std::fs::write(
info.join("alternates"),
format!("{}\n", Path::new(&common).join("objects").display()),
)
.map_err(|e| format!("{}: {e}", info.display()))?;
let project = Git::at(repo);
let legacy = project
.line(&[
"for-each-ref",
"--format=%(refname) %(objectname)",
REF_PREFIX,
])
.map_err(|e| e.0)?;
let private = Git::at(&dir);
for line in legacy.lines() {
let Some((name, commit)) = line.split_once(' ') else {
continue;
};
if !private
.succeeds(&["rev-parse", "--verify", "-q", name])
.unwrap_or(false)
{
private
.run(&["update-ref", name, commit])
.map_err(|e| e.0)?;
}
project.run(&["update-ref", "-d", name]).map_err(|e| e.0)?;
}
Ok(dir)
}
pub fn push(
repo: &Path,
store: &Path,
remote: &str,
unit: &str,
commit: &str,
) -> Result<(), String> {
let objects = store.join("objects");
Git::at(repo)
.run_with_env(
&[
"push",
"--quiet",
remote,
&format!("{commit}:{REF_PREFIX}/{unit}"),
],
&[("GIT_ALTERNATE_OBJECT_DIRECTORIES", objects.as_os_str())],
)
.map(|_| ())
.map_err(|e| e.0)
}
pub fn publish_closed(
store: &Path,
unit: &str,
tip: &str,
selection: &Selection,
tmp: &Path,
) -> Result<Option<String>, String> {
let mut files: Vec<(String, PathBuf)> = Vec::new();
for (key, dir) in &selection.sets {
let mut stack = vec![dir.clone()];
while let Some(d) = stack.pop() {
let Ok(entries) = std::fs::read_dir(&d) else {
continue;
};
for e in entries.flatten() {
let p = e.path();
if p.is_dir() {
stack.push(p);
} else if let Ok(rel) = p.strip_prefix(dir) {
let rel = rel.to_string_lossy().replace('\\', "/");
files.push((format!("{key}/{rel}"), p.clone()));
}
}
}
}
if files.is_empty() {
return Ok(None);
}
files.sort();
let git = Git::at(store);
std::fs::create_dir_all(tmp).map_err(|e| e.to_string())?;
let index = tmp.join(format!("held-out-index-{}", std::process::id()));
let _ = std::fs::remove_file(&index);
for (rel, path) in &files {
let blob = git
.line(&["hash-object", "-w", "--", &path.to_string_lossy()])
.map_err(|e| e.0)?;
git.run_with_index(
&index,
&[
"update-index",
"--add",
"--cacheinfo",
&format!("100644,{blob},{rel}"),
],
None,
)
.map_err(|e| e.0)?;
}
let tree = git
.run_with_index(&index, &["write-tree"], None)
.map_err(|e| e.0)?;
let _ = std::fs::remove_file(&index);
let tree = String::from_utf8_lossy(&tree).trim().to_owned();
let reference = format!("{REF_PREFIX}/{unit}");
let previous = git
.line(&[
"rev-parse",
"--verify",
"-q",
&format!("{reference}^{{commit}}"),
])
.ok()
.filter(|c| !c.is_empty());
if let Some(prev) = &previous
&& git
.line(&["rev-parse", &format!("{prev}^{{tree}}")])
.ok()
.as_deref()
== Some(&tree)
&& git
.succeeds(&["merge-base", "--is-ancestor", tip, prev])
.unwrap_or(false)
{
return Ok(Some(prev.clone()));
}
let message = format!(
"speccoder: held-out tests of {unit}\n\nThe tests that judged the tasks closed on {tip}: {}.\nEach <key>/manifest.json hashes to the digest its task's evidence names.\n\nSpec-Feature: {unit}\n",
selection.tasks.join(", ")
);
let mut args = crate::attempts::IDENTITY.to_vec();
args.extend(["commit-tree", &tree, "-p", tip]);
if let Some(prev) = &previous {
args.extend(["-p", prev]);
}
args.extend(["-m", &message]);
let commit = git.line(&args).map_err(|e| e.0)?;
git.run(&["update-ref", &reference, &commit])
.map_err(|e| e.0)?;
Ok(Some(commit))
}
pub fn publish(
repo: &Path,
project_state: &Path,
feature: &str,
tmp: &Path,
) -> Result<Option<String>, String> {
let branch = format!("speccoder/{feature}");
let tip = Git::at(repo)
.line(&["rev-parse", "--verify", &format!("{branch}^{{commit}}")])
.map_err(|e| e.0)?;
let closed = closed_on(repo, &tip, feature);
let selection = select_closed(project_state, feature, &closed);
if selection.sets.is_empty() {
return Ok(None);
}
publish_closed(
&publication_store(project_state, repo)?,
feature,
&tip,
&selection,
tmp,
)
}
#[must_use]
pub fn closed_on(repo: &Path, head: &str, unit: &str) -> Vec<(String, Vec<String>)> {
crate::attest::branch::closed(repo, head, unit)
.into_iter()
.map(|(task, commit)| {
let digests = named_digests(repo, &commit);
(task, digests)
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn literals_are_long_strings_and_precise_numbers() {
let l = literals(
"assert_eq!(slug(\"Hello, World\"), \"hello-world\"); x(3.14159); y(42); z('ab')",
);
assert!(l.contains("Hello, World") && l.contains("hello-world"));
assert!(l.contains("3.14159"));
assert!(!l.contains("42") && !l.contains("ab"));
}
#[test]
fn a_changed_file_or_manifest_is_tampering() {
let d = tempfile::tempdir().unwrap();
let content = "fn t() {}\n".to_owned();
let m = Manifest {
version: 1,
feature: "f".to_owned(),
task: "T1".to_owned(),
author_run: "r".to_owned(),
checkpoint: "c".to_owned(),
visible_scenarios: vec!["US1-1".to_owned()],
files: vec![HeldOutFile {
path: "tests/h.rs".to_owned(),
sha256: sha256_hex(content.as_bytes()),
scenarios: vec!["US1-1".to_owned()],
argv: vec!["true".to_owned()],
composition: false,
red_tests: None,
}],
discarded: Vec::new(),
};
let sha = store(d.path(), &m, &[("tests/h.rs".to_owned(), content)]).unwrap();
assert_eq!(load_verified(d.path(), &sha).unwrap().1.len(), 1);
assert!(load_verified(d.path(), "other").is_err());
std::fs::write(d.path().join("files/tests/h.rs"), "fn t() { changed }").unwrap();
assert!(
load_verified(d.path(), &sha)
.unwrap_err()
.0
.contains("changed")
);
std::fs::remove_file(d.path().join("files/tests/h.rs")).unwrap();
assert!(
load_verified(d.path(), &sha)
.unwrap_err()
.0
.contains("missing")
);
}
}