lean_ctx/core/eval_ab/
artifact.rs

1//! Signed, reproducible artifact (#238).
2//!
3//! Wraps an [`AbReport`] into a portable, tamper-evident attestation. Two guarantees:
4//!
5//! 1. **Reproducibility** — `determinism_digest` is a SHA-256 over the *evidence only* (task ids,
6//!    context + answer digests, scores, model fingerprint, stats config, verdict). Timestamps and
7//!    the build version are excluded, so the same inputs yield the same digest on any machine.
8//! 2. **Integrity + origin** — an Ed25519 signature over the canonical bytes (signature fields
9//!    cleared) proves the artifact was produced by a specific key and not altered since. This
10//!    mirrors the established `savings_ledger::signed_batch` pattern exactly.
11
12use std::path::{Path, PathBuf};
13
14use ed25519_dalek::{Signer, SigningKey};
15use serde::{Deserialize, Serialize};
16
17use super::report::{AbReport, Verdict};
18use super::sha256_hex;
19
20const SCHEMA_VERSION: u32 = 1;
21const KIND: &str = "lean-ctx.eval-ab-artifact";
22
23/// A signed A/B quality attestation.
24#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
25pub struct SignedAbReportV1 {
26    pub schema_version: u32,
27    pub kind: String,
28    pub created_at: String,
29    pub lean_ctx_version: String,
30    pub agent_id: String,
31    /// Machine-independent digest over the run evidence (see module docs).
32    pub determinism_digest: String,
33    /// Copied out of the report so a verifier sees the conclusion without parsing stats.
34    pub verdict: Verdict,
35    /// The full report (records + stats + provenance).
36    pub report: AbReport,
37    /// Ed25519 public key (hex). `None` until signed.
38    pub signer_public_key: Option<String>,
39    /// Ed25519 signature over the canonical bytes (hex). `None` until signed.
40    pub signature: Option<String>,
41}
42
43/// Outcome of verifying a [`SignedAbReportV1`].
44#[derive(Debug, Clone, PartialEq)]
45pub struct AbVerifyResult {
46    /// Signature present + valid over the canonical payload.
47    pub signature_valid: bool,
48    /// The recomputed digest matches the embedded one (run is internally consistent).
49    pub digest_matches: bool,
50    pub signer_public_key: Option<String>,
51    pub error: Option<String>,
52}
53
54impl AbVerifyResult {
55    /// Both checks passed.
56    pub fn ok(&self) -> bool {
57        self.signature_valid && self.digest_matches
58    }
59}
60
61impl SignedAbReportV1 {
62    /// Builds an unsigned artifact from a finished report.
63    pub fn from_report(report: AbReport, agent_id: &str) -> Self {
64        let determinism_digest = determinism_digest(&report);
65        Self {
66            schema_version: SCHEMA_VERSION,
67            kind: KIND.to_string(),
68            created_at: chrono::Utc::now().to_rfc3339(),
69            lean_ctx_version: env!("CARGO_PKG_VERSION").to_string(),
70            agent_id: agent_id.to_string(),
71            determinism_digest,
72            verdict: report.verdict,
73            report,
74            signer_public_key: None,
75            signature: None,
76        }
77    }
78
79    /// Deterministic bytes that get signed/verified: the whole struct with the signature fields
80    /// cleared (identical on sign + verify regardless of JSON float formatting).
81    pub fn canonical_bytes(&self) -> Result<Vec<u8>, String> {
82        let mut clone = self.clone();
83        clone.signature = None;
84        clone.signer_public_key = None;
85        serde_json::to_vec(&clone).map_err(|e| format!("serialize for signing: {e}"))
86    }
87
88    /// Signs with the persistent machine identity (`agent_identity` keystore).
89    pub fn sign(&mut self, agent_id: &str) -> Result<(), String> {
90        let key = crate::core::agent_identity::get_or_create_keypair(agent_id)?;
91        self.sign_with_key(&key)
92    }
93
94    /// Signs with an explicit key (used by `sign` and hermetic tests).
95    pub fn sign_with_key(&mut self, key: &SigningKey) -> Result<(), String> {
96        self.signature = None;
97        self.signer_public_key = None;
98        let canonical = self.canonical_bytes()?;
99        let sig = key.sign(&canonical);
100        self.signer_public_key = Some(crate::core::agent_identity::hex_encode(
101            &key.verifying_key().to_bytes(),
102        ));
103        self.signature = Some(crate::core::agent_identity::hex_encode(&sig.to_bytes()));
104        Ok(())
105    }
106
107    /// Verifies the embedded signature *and* recomputes the determinism digest — offline.
108    pub fn verify(&self) -> AbVerifyResult {
109        let digest_matches = determinism_digest(&self.report) == self.determinism_digest;
110        let fail = |msg: &str| AbVerifyResult {
111            signature_valid: false,
112            digest_matches,
113            signer_public_key: self.signer_public_key.clone(),
114            error: Some(msg.to_string()),
115        };
116
117        let (Some(sig_hex), Some(pk_hex)) = (&self.signature, &self.signer_public_key) else {
118            return fail("artifact is not signed");
119        };
120        let (Ok(sig_bytes), Ok(pk_bytes)) = (
121            crate::core::agent_identity::hex_decode(sig_hex),
122            crate::core::agent_identity::hex_decode(pk_hex),
123        ) else {
124            return fail("malformed signature or public key hex");
125        };
126        let canonical = match self.canonical_bytes() {
127            Ok(c) => c,
128            Err(e) => return fail(&e),
129        };
130        if crate::core::agent_identity::verify_signature(&pk_bytes, &canonical, &sig_bytes) {
131            AbVerifyResult {
132                signature_valid: true,
133                digest_matches,
134                signer_public_key: Some(pk_hex.clone()),
135                error: None,
136            }
137        } else {
138            fail("signature does not match payload (tampered or wrong key)")
139        }
140    }
141}
142
143/// Canonical evidence the determinism digest commits to — timestamps + version excluded.
144#[derive(Serialize)]
145struct Evidence {
146    suite: String,
147    budget_tokens: usize,
148    model_fingerprint: String,
149    bootstrap_iters: usize,
150    bootstrap_seed: u64,
151    noninferiority_margin: f64,
152    verdict: Verdict,
153    tasks: Vec<EvidenceRow>,
154}
155
156#[derive(Serialize)]
157struct EvidenceRow {
158    task_id: String,
159    domain: String,
160    baseline_value: f64,
161    lean_ctx_value: f64,
162    baseline_passed: bool,
163    lean_ctx_passed: bool,
164    baseline_context_digest: String,
165    lean_ctx_context_digest: String,
166    baseline_answer_digest: String,
167    lean_ctx_answer_digest: String,
168}
169
170/// Computes the machine-independent run digest (records sorted by id for a canonical order).
171pub fn determinism_digest(report: &AbReport) -> String {
172    let mut rows: Vec<EvidenceRow> = report
173        .records
174        .iter()
175        .map(|r| EvidenceRow {
176            task_id: r.task_id.clone(),
177            domain: r.domain.clone(),
178            baseline_value: r.baseline_value,
179            lean_ctx_value: r.lean_ctx_value,
180            baseline_passed: r.baseline_passed,
181            lean_ctx_passed: r.lean_ctx_passed,
182            baseline_context_digest: r.baseline_context_digest.clone(),
183            lean_ctx_context_digest: r.lean_ctx_context_digest.clone(),
184            baseline_answer_digest: r.baseline_answer_digest.clone(),
185            lean_ctx_answer_digest: r.lean_ctx_answer_digest.clone(),
186        })
187        .collect();
188    rows.sort_by(|a, b| a.task_id.cmp(&b.task_id));
189
190    let evidence = Evidence {
191        suite: report.suite.clone(),
192        budget_tokens: report.budget_tokens,
193        model_fingerprint: report.model.digest(),
194        bootstrap_iters: report.stats.bootstrap_iters,
195        bootstrap_seed: report.stats.bootstrap_seed,
196        noninferiority_margin: report.stats.noninferiority_margin,
197        verdict: report.verdict,
198        tasks: rows,
199    };
200    let bytes = serde_json::to_vec(&evidence).unwrap_or_default();
201    sha256_hex(&bytes)
202}
203
204/// Default artifact location: `<data_dir>/eval/ab-report-v1_<utc-stamp>.json`.
205pub fn default_artifact_path() -> Result<PathBuf, String> {
206    let dir = crate::core::data_dir::lean_ctx_data_dir()?.join("eval");
207    std::fs::create_dir_all(&dir).map_err(|e| format!("mkdir eval: {e}"))?;
208    let stamp = chrono::Utc::now().format("%Y%m%dT%H%M%SZ");
209    Ok(dir.join(format!("ab-report-v1_{stamp}.json")))
210}
211
212/// Pretty-prints the artifact to `out` (creating parent dirs). Returns the written path.
213pub fn write_artifact(artifact: &SignedAbReportV1, out: &Path) -> Result<PathBuf, String> {
214    if let Some(parent) = out.parent() {
215        std::fs::create_dir_all(parent).map_err(|e| format!("mkdir: {e}"))?;
216    }
217    let json = serde_json::to_string_pretty(artifact).map_err(|e| format!("serialize: {e}"))?;
218    std::fs::write(out, json).map_err(|e| format!("write {}: {e}", out.display()))?;
219    Ok(out.to_path_buf())
220}
221
222/// Loads + parses an artifact, rejecting unrelated JSON by `kind`.
223pub fn load_artifact(path: &Path) -> Result<SignedAbReportV1, String> {
224    let raw = std::fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))?;
225    let artifact: SignedAbReportV1 =
226        serde_json::from_str(&raw).map_err(|e| format!("parse {}: {e}", path.display()))?;
227    if artifact.kind != KIND {
228        return Err(format!(
229            "not a {KIND} artifact (kind = {:?})",
230            artifact.kind
231        ));
232    }
233    Ok(artifact)
234}
235
236#[cfg(test)]
237mod tests {
238    use super::*;
239    use crate::core::eval_ab::model::{ModelFingerprint, ModelParams};
240    use crate::core::eval_ab::report::{AbReport, PairRecord, ReportConfig};
241
242    fn report() -> AbReport {
243        let records = vec![PairRecord {
244            task_id: "t1".into(),
245            domain: "qa".into(),
246            baseline_value: 0.2,
247            lean_ctx_value: 0.9,
248            baseline_passed: false,
249            lean_ctx_passed: true,
250            baseline_tokens: 100,
251            lean_ctx_tokens: 90,
252            baseline_context_digest: "ca".into(),
253            lean_ctx_context_digest: "cb".into(),
254            baseline_answer_digest: "aa".into(),
255            lean_ctx_answer_digest: "ab".into(),
256        }];
257        let fp = ModelFingerprint {
258            provider: "recorded".into(),
259            endpoint: "rec".into(),
260            params: ModelParams::default(),
261        };
262        AbReport::build("suite", 4000, fp, records, ReportConfig::default())
263    }
264
265    fn key() -> SigningKey {
266        let mut seed = [0u8; 32];
267        getrandom::fill(&mut seed).unwrap();
268        SigningKey::from_bytes(&seed)
269    }
270
271    #[test]
272    fn digest_ignores_timestamp_and_version() {
273        let a = SignedAbReportV1::from_report(report(), "local");
274        let mut r2 = report();
275        r2.created_at = "2000-01-01T00:00:00Z".into();
276        r2.lean_ctx_version = "0.0.0".into();
277        let b = SignedAbReportV1::from_report(r2, "local");
278        assert_eq!(a.determinism_digest, b.determinism_digest);
279    }
280
281    #[test]
282    fn sign_then_verify_roundtrips() {
283        let mut a = SignedAbReportV1::from_report(report(), "local");
284        a.sign_with_key(&key()).unwrap();
285        let res = a.verify();
286        assert!(res.ok(), "{res:?}");
287    }
288
289    #[test]
290    fn tampering_with_scores_breaks_digest_and_signature() {
291        let mut a = SignedAbReportV1::from_report(report(), "local");
292        a.sign_with_key(&key()).unwrap();
293        a.report.records[0].lean_ctx_value = 0.0;
294        let res = a.verify();
295        assert!(!res.digest_matches, "edited score must break the digest");
296        assert!(
297            !res.signature_valid,
298            "edited payload must break the signature"
299        );
300    }
301
302    #[test]
303    fn write_load_roundtrip_preserves_signature() {
304        let mut a = SignedAbReportV1::from_report(report(), "local");
305        a.sign_with_key(&key()).unwrap();
306        let nanos = std::time::SystemTime::now()
307            .duration_since(std::time::UNIX_EPOCH)
308            .map_or(0, |d| d.as_nanos());
309        let path = std::env::temp_dir().join(format!("lc-ab-{}-{nanos}.json", std::process::id()));
310        write_artifact(&a, &path).unwrap();
311        let loaded = load_artifact(&path).unwrap();
312        assert_eq!(loaded, a);
313        assert!(loaded.verify().ok());
314        let _ = std::fs::remove_file(&path);
315    }
316
317    #[test]
318    fn load_rejects_foreign_json() {
319        let nanos = std::time::SystemTime::now()
320            .duration_since(std::time::UNIX_EPOCH)
321            .map_or(0, |d| d.as_nanos());
322        let path =
323            std::env::temp_dir().join(format!("lc-foreign-ab-{}-{nanos}.json", std::process::id()));
324        std::fs::write(&path, r#"{"kind":"nope"}"#).unwrap();
325        assert!(load_artifact(&path).is_err());
326        let _ = std::fs::remove_file(&path);
327    }
328}
lean_ctx/core/eval_ab/artifact.rs

lean_ctx/core/eval_ab/
artifact.rs
