dkp_core/validate/
gate7.rs

1use crate::{
2    pack::loader::Pack,
3    validate::gates::{CheckResult, GateResult, GateStatus},
4};
5
6/// Gate 7: Evaluation — eval_set.jsonl present and evidence/eval_results/eval_summary.json
7/// reports gate7_pass=true (SPEC.md §12.4, §16).
8pub fn run(pack: &Pack) -> GateResult {
9    let has_cases = pack.has_eval_set()
10        && pack
11            .load_eval_set()
12            .map(|cases| !cases.is_empty())
13            .unwrap_or(false);
14
15    if !has_cases {
16        return GateResult {
17            gate: 7,
18            status: GateStatus::Skipped,
19            checks: vec![CheckResult::skip("eval_set.jsonl (optional, or empty)")],
20            message: Some("eval_set.jsonl not present or empty; gate 7 skipped".to_string()),
21        };
22    }
23
24    let mut checks = Vec::new();
25
26    // Count eval cases
27    match pack.load_eval_set() {
28        Ok(cases) => checks.push(CheckResult::pass(format!(
29            "eval_set.jsonl: {} cases loaded",
30            cases.len()
31        ))),
32        Err(e) => checks.push(CheckResult::fail("eval_set.jsonl parses", e.to_string())),
33    }
34
35    // Check evidence/eval_results/eval_summary.json (SPEC.md §12.4, Appendix B §B.13)
36    let summary_path = pack
37        .evidence_dir()
38        .join("eval_results")
39        .join("eval_summary.json");
40    if summary_path.exists() {
41        match std::fs::read_to_string(&summary_path)
42            .ok()
43            .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
44        {
45            Some(v) => {
46                let gate7_pass = v["gate7_pass"].as_bool().unwrap_or(false);
47                let mean_delta = v["mean_delta"].as_f64().unwrap_or(0.0);
48                if gate7_pass {
49                    checks.push(CheckResult::pass(format!(
50                        "eval_summary.json: gate7_pass=true (mean_delta={mean_delta:.3})"
51                    )));
52                } else {
53                    let min_delta = pack.manifest.min_eval_delta.unwrap_or(0.0);
54                    checks.push(CheckResult::fail(
55                        "eval_summary.json",
56                        format!("gate7_pass=false (mean_delta={mean_delta:.3}, required >= {min_delta:.3})"),
57                    ));
58                }
59            }
60            None => checks.push(CheckResult::fail(
61                "eval_summary.json parses",
62                "could not parse evidence/eval_results/eval_summary.json",
63            )),
64        }
65    } else {
66        checks.push(CheckResult::fail(
67            "evidence/eval_results/eval_summary.json",
68            "not found; run `dkp eval` to generate before publishing",
69        ));
70    }
71
72    let failed = checks.iter().any(|c| c.status == GateStatus::Fail);
73    GateResult {
74        gate: 7,
75        status: if failed {
76            GateStatus::Fail
77        } else {
78            GateStatus::Pass
79        },
80        checks,
81        message: None,
82    }
83}
84
85#[cfg(test)]
86mod tests {
87    use super::*;
88    use tempfile::TempDir;
89
90    fn minimal_manifest_json() -> &'static str {
91        r#"{
92            "spec": "1.0.0",
93            "name": "test-pack",
94            "version": "1.0.0",
95            "domain": "testing",
96            "audience": "internal",
97            "intended_use": "unit tests",
98            "known_limitations": "none",
99            "update_date": "2026-01-01"
100        }"#
101    }
102
103    fn eval_case_line() -> &'static str {
104        r#"{"query": "q", "expected_dimensions": [], "critical_must_include": [], "scoring_rubric": "r", "version_meta": {"prompt_hash": "h", "model_version": "m", "dataset_version": "d"}}"#
105    }
106
107    fn open_pack(tmp: &TempDir) -> Pack {
108        std::fs::write(tmp.path().join("manifest.json"), minimal_manifest_json()).unwrap();
109        Pack::open(tmp.path()).unwrap()
110    }
111
112    #[test]
113    fn eval_set_absent_is_skipped() {
114        let tmp = TempDir::new().unwrap();
115        let pack = open_pack(&tmp);
116        let result = run(&pack);
117        assert_eq!(result.status, GateStatus::Skipped);
118        assert_eq!(result.gate, 7);
119    }
120
121    #[test]
122    fn eval_set_present_but_empty_is_skipped() {
123        let tmp = TempDir::new().unwrap();
124        let pack = open_pack(&tmp);
125        std::fs::create_dir_all(pack.machine_dir()).unwrap();
126        std::fs::write(pack.machine_file("eval_set.jsonl"), "").unwrap();
127
128        let result = run(&pack);
129        assert_eq!(result.status, GateStatus::Skipped);
130    }
131
132    #[test]
133    fn eval_set_present_but_no_summary_fails() {
134        let tmp = TempDir::new().unwrap();
135        let pack = open_pack(&tmp);
136        std::fs::create_dir_all(pack.machine_dir()).unwrap();
137        std::fs::write(
138            pack.machine_file("eval_set.jsonl"),
139            format!("{}\n", eval_case_line()),
140        )
141        .unwrap();
142
143        let result = run(&pack);
144        assert_eq!(result.status, GateStatus::Fail);
145        assert!(result.checks.iter().any(|c| {
146            c.description
147                .contains("evidence/eval_results/eval_summary.json")
148                && c.status == GateStatus::Fail
149        }));
150    }
151
152    #[test]
153    fn eval_summary_gate7_pass_true_passes() {
154        let tmp = TempDir::new().unwrap();
155        let pack = open_pack(&tmp);
156        std::fs::create_dir_all(pack.machine_dir()).unwrap();
157        std::fs::write(
158            pack.machine_file("eval_set.jsonl"),
159            format!("{}\n", eval_case_line()),
160        )
161        .unwrap();
162        let eval_results_dir = pack.evidence_dir().join("eval_results");
163        std::fs::create_dir_all(&eval_results_dir).unwrap();
164        std::fs::write(
165            eval_results_dir.join("eval_summary.json"),
166            r#"{"gate7_pass": true, "mean_delta": 0.42}"#,
167        )
168        .unwrap();
169
170        let result = run(&pack);
171        assert_eq!(result.status, GateStatus::Pass);
172    }
173
174    #[test]
175    fn eval_summary_gate7_pass_false_fails() {
176        let tmp = TempDir::new().unwrap();
177        let pack = open_pack(&tmp);
178        std::fs::create_dir_all(pack.machine_dir()).unwrap();
179        std::fs::write(
180            pack.machine_file("eval_set.jsonl"),
181            format!("{}\n", eval_case_line()),
182        )
183        .unwrap();
184        let eval_results_dir = pack.evidence_dir().join("eval_results");
185        std::fs::create_dir_all(&eval_results_dir).unwrap();
186        std::fs::write(
187            eval_results_dir.join("eval_summary.json"),
188            r#"{"gate7_pass": false, "mean_delta": 0.01}"#,
189        )
190        .unwrap();
191
192        let result = run(&pack);
193        assert_eq!(result.status, GateStatus::Fail);
194    }
195
196    #[test]
197    fn eval_summary_unparseable_fails() {
198        let tmp = TempDir::new().unwrap();
199        let pack = open_pack(&tmp);
200        std::fs::create_dir_all(pack.machine_dir()).unwrap();
201        std::fs::write(
202            pack.machine_file("eval_set.jsonl"),
203            format!("{}\n", eval_case_line()),
204        )
205        .unwrap();
206        let eval_results_dir = pack.evidence_dir().join("eval_results");
207        std::fs::create_dir_all(&eval_results_dir).unwrap();
208        std::fs::write(eval_results_dir.join("eval_summary.json"), "not json").unwrap();
209
210        let result = run(&pack);
211        assert_eq!(result.status, GateStatus::Fail);
212    }
213}
dkp_core/validate/gate7.rs

dkp_core/validate/
gate7.rs
