lean_ctx/core/eval_ab/testbench/
lockfile.rs

1//! Pinned-repo lockfile for the public off-vs-on testbench (#611).
2//!
3//! A lockfile names the external repositories the testbench runs against and pins
4//! each to an exact commit, so a public run is reproducible by anyone. Each entry is
5//! either a **remote** repo (`url` + `commit`, cloned + checked out by
6//! [`super::clone`]) or a **local** fixture (`path`, used by the committed
7//! deterministic CI subset which must run offline). Every entry points at an NDJSON
8//! [`super::super::suite`] file whose task `workspace`s resolve *inside* the repo.
9
10use std::collections::HashSet;
11use std::path::{Path, PathBuf};
12
13use anyhow::{Context, Result, bail};
14use serde::{Deserialize, Serialize};
15
16use crate::core::eval_ab::sha256_hex;
17
18/// Lockfile schema discriminator.
19pub const TESTBENCH_LOCK_KIND: &str = "lean-ctx.testbench-lock";
20
21/// One pinned repository under test.
22#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
23pub struct RepoEntry {
24    /// Stable, unique label used in reports + the cache directory name.
25    pub name: String,
26    /// Git remote to clone (mutually exclusive with `path`).
27    #[serde(default, skip_serializing_if = "Option::is_none")]
28    pub url: Option<String>,
29    /// Exact commit to check out (required with `url`).
30    #[serde(default, skip_serializing_if = "Option::is_none")]
31    pub commit: Option<String>,
32    /// Local fixture directory (relative to the lockfile), used instead of cloning.
33    #[serde(default, skip_serializing_if = "Option::is_none")]
34    pub path: Option<String>,
35    /// NDJSON suite (relative to the lockfile) whose task workspaces resolve inside the repo.
36    pub suite: String,
37}
38
39impl RepoEntry {
40    /// True for a committed local fixture (no network), false for a remote clone.
41    pub fn is_local(&self) -> bool {
42        self.path.is_some()
43    }
44
45    fn validate(&self) -> std::result::Result<(), String> {
46        if self.name.trim().is_empty() {
47            return Err("repo entry has an empty name".into());
48        }
49        if self.suite.trim().is_empty() {
50            return Err(format!("repo {}: suite is empty", self.name));
51        }
52        match (&self.url, &self.commit, &self.path) {
53            (Some(u), Some(c), None) => {
54                if u.trim().is_empty() || c.trim().is_empty() {
55                    return Err(format!(
56                        "repo {}: url and commit must be non-empty",
57                        self.name
58                    ));
59                }
60                Ok(())
61            }
62            (None, None, Some(p)) => {
63                if p.trim().is_empty() {
64                    return Err(format!("repo {}: path is empty", self.name));
65                }
66                Ok(())
67            }
68            _ => Err(format!(
69                "repo {}: set EITHER url+commit (remote) OR path (local fixture)",
70                self.name
71            )),
72        }
73    }
74}
75
76/// A parsed, validated lockfile plus the directory it was loaded from (the resolution
77/// root for relative `path` / `suite` entries).
78#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
79pub struct TestbenchLock {
80    pub kind: String,
81    pub repos: Vec<RepoEntry>,
82    #[serde(skip)]
83    dir: PathBuf,
84}
85
86impl TestbenchLock {
87    /// Loads + validates a lockfile, recording its parent dir for path resolution.
88    pub fn load(path: &Path) -> Result<Self> {
89        let raw = std::fs::read_to_string(path)
90            .with_context(|| format!("reading testbench lock {}", path.display()))?;
91        let dir = path
92            .parent()
93            .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
94        Self::parse(&raw, dir)
95    }
96
97    /// Pure parser (testable without a file on disk).
98    pub fn parse(raw: &str, dir: PathBuf) -> Result<Self> {
99        let mut lock: TestbenchLock =
100            serde_json::from_str(raw).context("parsing testbench lock JSON")?;
101        if lock.kind != TESTBENCH_LOCK_KIND {
102            bail!("not a {TESTBENCH_LOCK_KIND} file (kind = {:?})", lock.kind);
103        }
104        if lock.repos.is_empty() {
105            bail!("testbench lock contains no repos");
106        }
107        let mut seen = HashSet::new();
108        for repo in &lock.repos {
109            if let Err(reason) = repo.validate() {
110                bail!("invalid lock entry: {reason}");
111            }
112            if !seen.insert(repo.name.as_str()) {
113                bail!("duplicate repo name: {}", repo.name);
114            }
115        }
116        lock.dir = dir;
117        Ok(lock)
118    }
119
120    /// Resolution root for relative `path` / `suite` entries.
121    pub fn dir(&self) -> &Path {
122        &self.dir
123    }
124
125    /// Machine-independent digest of the pinned set (names, sources, commits, suites),
126    /// embedded in the report so a third party can confirm *what* was run.
127    pub fn digest(&self) -> String {
128        // Serialize only the repos (not the local `dir`, which varies per machine).
129        let bytes = serde_json::to_vec(&self.repos).unwrap_or_default();
130        sha256_hex(&bytes)
131    }
132}
133
134#[cfg(test)]
135mod tests {
136    use super::*;
137
138    fn local_lock() -> &'static str {
139        r#"{"kind":"lean-ctx.testbench-lock","repos":[
140          {"name":"qa","path":"repos/qa","suite":"qa.ndjson"},
141          {"name":"code","path":"repos/code","suite":"code.ndjson"}
142        ]}"#
143    }
144
145    #[test]
146    fn parses_local_fixture_lock() {
147        let lock = TestbenchLock::parse(local_lock(), PathBuf::from("/lock")).unwrap();
148        assert_eq!(lock.repos.len(), 2);
149        assert!(lock.repos[0].is_local());
150        assert_eq!(lock.dir(), Path::new("/lock"));
151    }
152
153    #[test]
154    fn parses_remote_entry() {
155        let raw = r#"{"kind":"lean-ctx.testbench-lock","repos":[
156          {"name":"r","url":"https://example.com/r.git","commit":"abc123","suite":"r.ndjson"}
157        ]}"#;
158        let lock = TestbenchLock::parse(raw, PathBuf::from(".")).unwrap();
159        assert!(!lock.repos[0].is_local());
160    }
161
162    #[test]
163    fn rejects_mixed_source() {
164        let raw = r#"{"kind":"lean-ctx.testbench-lock","repos":[
165          {"name":"r","url":"u","commit":"c","path":"p","suite":"s"}
166        ]}"#;
167        assert!(TestbenchLock::parse(raw, PathBuf::from(".")).is_err());
168    }
169
170    #[test]
171    fn rejects_remote_without_commit() {
172        let raw = r#"{"kind":"lean-ctx.testbench-lock","repos":[
173          {"name":"r","url":"u","suite":"s"}
174        ]}"#;
175        assert!(TestbenchLock::parse(raw, PathBuf::from(".")).is_err());
176    }
177
178    #[test]
179    fn rejects_duplicate_names() {
180        let raw = r#"{"kind":"lean-ctx.testbench-lock","repos":[
181          {"name":"r","path":"a","suite":"s"},
182          {"name":"r","path":"b","suite":"s"}
183        ]}"#;
184        assert!(TestbenchLock::parse(raw, PathBuf::from(".")).is_err());
185    }
186
187    #[test]
188    fn rejects_foreign_kind() {
189        let raw = r#"{"kind":"nope","repos":[{"name":"r","path":"p","suite":"s"}]}"#;
190        assert!(TestbenchLock::parse(raw, PathBuf::from(".")).is_err());
191    }
192
193    #[test]
194    fn digest_is_stable_and_ignores_dir() {
195        let a = TestbenchLock::parse(local_lock(), PathBuf::from("/one")).unwrap();
196        let b = TestbenchLock::parse(local_lock(), PathBuf::from("/two")).unwrap();
197        assert_eq!(a.digest(), b.digest());
198    }
199}
lean_ctx/core/eval_ab/testbench/lockfile.rs

lean_ctx/core/eval_ab/testbench/
lockfile.rs
