dkp_core/validate/
gate7.rs

1use crate::{
2    pack::loader::Pack,
3    validate::gates::{CheckResult, GateResult, GateStatus},
4};
5
6/// Gate 7: Evaluation — eval_set.jsonl present and evidence/eval_results/eval_summary.json
7/// reports gate7_pass=true (SPEC.md §12.4, §16).
8pub fn run(pack: &Pack) -> GateResult {
9    let has_cases = pack.has_eval_set()
10        && pack
11            .load_eval_set()
12            .map(|cases| !cases.is_empty())
13            .unwrap_or(false);
14
15    if !has_cases {
16        return GateResult {
17            gate: 7,
18            status: GateStatus::Skipped,
19            checks: vec![CheckResult::skip("eval_set.jsonl (optional, or empty)")],
20            message: Some("eval_set.jsonl not present or empty; gate 7 skipped".to_string()),
21        };
22    }
23
24    let mut checks = Vec::new();
25
26    // Count eval cases
27    match pack.load_eval_set() {
28        Ok(cases) => checks.push(CheckResult::pass(format!(
29            "eval_set.jsonl: {} cases loaded",
30            cases.len()
31        ))),
32        Err(e) => checks.push(CheckResult::fail("eval_set.jsonl parses", e.to_string())),
33    }
34
35    // Check evidence/eval_results/eval_summary.json (SPEC.md §12.4, Appendix B §B.13)
36    let summary_path = pack
37        .evidence_dir()
38        .join("eval_results")
39        .join("eval_summary.json");
40    if summary_path.exists() {
41        match std::fs::read_to_string(&summary_path)
42            .ok()
43            .and_then(|s| serde_json::from_str::<serde_json::Value>(&s).ok())
44        {
45            Some(v) => {
46                let gate7_pass = v["gate7_pass"].as_bool().unwrap_or(false);
47                let mean_delta = v["mean_delta"].as_f64().unwrap_or(0.0);
48                if gate7_pass {
49                    checks.push(CheckResult::pass(format!(
50                        "eval_summary.json: gate7_pass=true (mean_delta={mean_delta:.3})"
51                    )));
52                } else {
53                    let min_delta = pack.manifest.min_eval_delta.unwrap_or(0.0);
54                    checks.push(CheckResult::fail(
55                        "eval_summary.json",
56                        format!("gate7_pass=false (mean_delta={mean_delta:.3}, required >= {min_delta:.3})"),
57                    ));
58                }
59            }
60            None => checks.push(CheckResult::fail(
61                "eval_summary.json parses",
62                "could not parse evidence/eval_results/eval_summary.json",
63            )),
64        }
65    } else {
66        checks.push(CheckResult::fail(
67            "evidence/eval_results/eval_summary.json",
68            "not found; run `dkp eval` to generate before publishing",
69        ));
70    }
71
72    let failed = checks.iter().any(|c| c.status == GateStatus::Fail);
73    GateResult {
74        gate: 7,
75        status: if failed {
76            GateStatus::Fail
77        } else {
78            GateStatus::Pass
79        },
80        checks,
81        message: None,
82    }
83}
84
85#[cfg(test)]
86mod tests {
87    use super::*;
88    use tempfile::TempDir;
89
90    fn minimal_manifest_json() -> &'static str {
91        r#"{
92            "spec": "1.0.0",
93            "name": "test-pack",
94            "version": "1.0.0",
95            "domain": "testing",
96            "audience": "internal",
97            "intended_use": "unit tests",
98            "known_limitations": "none",
99            "update_date": "2026-01-01"
100        }"#
101    }
102
103    fn eval_case_line() -> &'static str {
104        r#"{"query": "q", "expected_dimensions": [], "critical_must_include": [], "scoring_rubric": "r", "version_meta": {"prompt_hash": "h", "model_version": "m", "dataset_version": "d"}}"#
105    }
106
107    fn open_pack(tmp: &TempDir) -> Pack {
108        std::fs::write(tmp.path().join("manifest.json"), minimal_manifest_json()).unwrap();
109        Pack::open(tmp.path()).unwrap()
110    }
111
112    #[test]
113    fn eval_set_absent_is_skipped() {
114        let tmp = TempDir::new().unwrap();
115        let pack = open_pack(&tmp);
116        let result = run(&pack);
117        assert_eq!(result.status, GateStatus::Skipped);
118        assert_eq!(result.gate, 7);
119    }
120
121    #[test]
122    fn eval_set_present_but_empty_is_skipped() {
123        let tmp = TempDir::new().unwrap();
124        let pack = open_pack(&tmp);
125        std::fs::create_dir_all(pack.machine_dir()).unwrap();
126        std::fs::write(pack.machine_file("eval_set.jsonl"), "").unwrap();
127
128        let result = run(&pack);
129        assert_eq!(result.status, GateStatus::Skipped);
130    }
131
132    #[test]
133    fn eval_set_present_but_no_summary_fails() {
134        let tmp = TempDir::new().unwrap();
135        let pack = open_pack(&tmp);
136        std::fs::create_dir_all(pack.machine_dir()).unwrap();
137        std::fs::write(
138            pack.machine_file("eval_set.jsonl"),
139            format!("{}\n", eval_case_line()),
140        )
141        .unwrap();
142
143        let result = run(&pack);
144        assert_eq!(result.status, GateStatus::Fail);
145        assert!(result.checks.iter().any(|c| c
146            .description
147            .contains("evidence/eval_results/eval_summary.json")
148            && c.status == GateStatus::Fail));
149    }
150
151    #[test]
152    fn eval_summary_gate7_pass_true_passes() {
153        let tmp = TempDir::new().unwrap();
154        let pack = open_pack(&tmp);
155        std::fs::create_dir_all(pack.machine_dir()).unwrap();
156        std::fs::write(
157            pack.machine_file("eval_set.jsonl"),
158            format!("{}\n", eval_case_line()),
159        )
160        .unwrap();
161        let eval_results_dir = pack.evidence_dir().join("eval_results");
162        std::fs::create_dir_all(&eval_results_dir).unwrap();
163        std::fs::write(
164            eval_results_dir.join("eval_summary.json"),
165            r#"{"gate7_pass": true, "mean_delta": 0.42}"#,
166        )
167        .unwrap();
168
169        let result = run(&pack);
170        assert_eq!(result.status, GateStatus::Pass);
171    }
172
173    #[test]
174    fn eval_summary_gate7_pass_false_fails() {
175        let tmp = TempDir::new().unwrap();
176        let pack = open_pack(&tmp);
177        std::fs::create_dir_all(pack.machine_dir()).unwrap();
178        std::fs::write(
179            pack.machine_file("eval_set.jsonl"),
180            format!("{}\n", eval_case_line()),
181        )
182        .unwrap();
183        let eval_results_dir = pack.evidence_dir().join("eval_results");
184        std::fs::create_dir_all(&eval_results_dir).unwrap();
185        std::fs::write(
186            eval_results_dir.join("eval_summary.json"),
187            r#"{"gate7_pass": false, "mean_delta": 0.01}"#,
188        )
189        .unwrap();
190
191        let result = run(&pack);
192        assert_eq!(result.status, GateStatus::Fail);
193    }
194
195    #[test]
196    fn eval_summary_unparseable_fails() {
197        let tmp = TempDir::new().unwrap();
198        let pack = open_pack(&tmp);
199        std::fs::create_dir_all(pack.machine_dir()).unwrap();
200        std::fs::write(
201            pack.machine_file("eval_set.jsonl"),
202            format!("{}\n", eval_case_line()),
203        )
204        .unwrap();
205        let eval_results_dir = pack.evidence_dir().join("eval_results");
206        std::fs::create_dir_all(&eval_results_dir).unwrap();
207        std::fs::write(eval_results_dir.join("eval_summary.json"), "not json").unwrap();
208
209        let result = run(&pack);
210        assert_eq!(result.status, GateStatus::Fail);
211    }
212}
dkp_core/validate/gate7.rs

dkp_core/validate/
gate7.rs
