1use std::collections::{BTreeMap, BTreeSet};
9use std::fs;
10use std::path::{Path, PathBuf};
11
12use serde::{Deserialize, Serialize};
13use serde_json::Value as JsonValue;
14use sha2::{Digest, Sha256};
15use walkdir::WalkDir;
16
17use super::estimate_chunk_tokens;
18use crate::value::VmError;
19
20pub const SKILL_GATE_SCHEMA_VERSION: u32 = 1;
21pub const SKILL_GATE_MANIFEST_TYPE: &str = "harn.skill_gate.manifest.v1";
22pub const SKILL_GATE_REPORT_TYPE: &str = "harn.skill_gate.report.v1";
23pub const SKILL_GATE_RECEIPT_TYPE: &str = "harn.skill_gate.receipt.v1";
24
25const EPSILON: f64 = 0.000_000_1;
26
27#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
28#[serde(default)]
29pub struct SkillGateManifest {
30 #[serde(rename = "_type")]
31 pub type_name: String,
32 pub version: u32,
33 pub id: String,
34 pub name: Option<String>,
35 pub description: Option<String>,
36 #[serde(default, alias = "base-dir")]
37 pub base_dir: Option<String>,
38 #[serde(default, alias = "target-model")]
39 pub target_model: SkillGateTargetModel,
40 pub policy: SkillGatePolicy,
41 pub grader: SkillGateGrader,
42 pub tasks: Vec<SkillGateTask>,
43 pub variants: Vec<SkillGateVariant>,
44 pub metadata: BTreeMap<String, JsonValue>,
45}
46
47#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
48#[serde(default)]
49pub struct SkillGateTargetModel {
50 pub id: String,
51 pub provider: Option<String>,
52 #[serde(default, alias = "knowledge-cutoff")]
53 pub knowledge_cutoff: Option<String>,
54 #[serde(default, alias = "context-budget-tokens")]
55 pub context_budget_tokens: Option<usize>,
56 pub metadata: BTreeMap<String, JsonValue>,
57}
58
59#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
60#[serde(default)]
61pub struct SkillGatePolicy {
62 #[serde(default, alias = "min-included-tasks")]
63 pub min_included_tasks: Option<usize>,
64 #[serde(default, alias = "min-score-lift")]
65 pub min_score_lift: Option<f64>,
66 #[serde(default, alias = "min-gap-recovery")]
67 pub min_gap_recovery: Option<f64>,
68 #[serde(default, alias = "min-cluster-gap-recovery")]
69 pub min_cluster_gap_recovery: Option<f64>,
70 #[serde(default, alias = "require-cluster-lift")]
71 pub require_cluster_lift: bool,
72 #[serde(default, alias = "max-regression-rate")]
73 pub max_regression_rate: Option<f64>,
74 #[serde(default, alias = "min-win-rate")]
75 pub min_win_rate: Option<f64>,
76 #[serde(default, alias = "max-context-delta-tokens")]
77 pub max_context_delta_tokens: Option<i64>,
78 #[serde(default, alias = "pass-score-threshold")]
79 pub pass_score_threshold: Option<f64>,
80 #[serde(default, alias = "require-no-tamper")]
81 pub require_no_tamper: Option<bool>,
82 pub metadata: BTreeMap<String, JsonValue>,
83}
84
85impl SkillGatePolicy {
86 fn min_included_tasks(&self) -> usize {
87 self.min_included_tasks.unwrap_or(1)
88 }
89
90 fn min_score_lift(&self) -> f64 {
91 self.min_score_lift.unwrap_or(0.0)
92 }
93
94 fn min_gap_recovery(&self) -> f64 {
95 self.min_gap_recovery.unwrap_or(0.0)
96 }
97
98 fn min_cluster_gap_recovery(&self) -> f64 {
99 self.min_cluster_gap_recovery
100 .unwrap_or_else(|| self.min_gap_recovery())
101 }
102
103 fn max_regression_rate(&self) -> f64 {
104 self.max_regression_rate.unwrap_or(0.0)
105 }
106
107 fn pass_score_threshold(&self) -> f64 {
108 self.pass_score_threshold.unwrap_or(0.5)
109 }
110
111 fn require_no_tamper(&self) -> bool {
112 self.require_no_tamper.unwrap_or(true)
113 }
114}
115
116#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
117#[serde(default)]
118pub struct SkillGateGrader {
119 pub id: String,
120 #[serde(default, alias = "immutable-paths", alias = "protected-paths")]
121 pub immutable_paths: Vec<SkillGateProtectedPath>,
122 pub metadata: BTreeMap<String, JsonValue>,
123}
124
125#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
126#[serde(default)]
127pub struct SkillGateProtectedPath {
128 pub path: String,
129 pub sha256: String,
130 pub label: Option<String>,
131}
132
133#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
134#[serde(default)]
135pub struct SkillGateTask {
136 pub id: String,
137 pub name: Option<String>,
138 pub cluster: String,
139 pub source: Option<String>,
140 pub heldout: SkillGateHeldout,
141 #[serde(default, alias = "baseline-score")]
142 pub baseline_score: f64,
143 #[serde(default, alias = "frontier-score")]
144 pub frontier_score: f64,
145 #[serde(default, alias = "baseline-passed")]
146 pub baseline_passed: Option<bool>,
147 pub metadata: BTreeMap<String, JsonValue>,
148}
149
150#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
151#[serde(default)]
152pub struct SkillGateHeldout {
153 pub kind: String,
154 #[serde(default, alias = "created-at")]
155 pub created_at: Option<String>,
156 pub private: bool,
157 pub suite: Option<String>,
158 pub metadata: BTreeMap<String, JsonValue>,
159}
160
161#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
162#[serde(default)]
163pub struct SkillGateVariant {
164 pub id: String,
165 pub name: Option<String>,
166 pub description: Option<String>,
167 pub baseline: SkillGateArtifact,
168 pub candidate: SkillGateArtifact,
169 #[serde(default, alias = "case-results")]
170 pub case_results: Vec<SkillGateCaseResult>,
171 pub metadata: BTreeMap<String, JsonValue>,
172}
173
174#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
175#[serde(default)]
176pub struct SkillGateArtifact {
177 pub kind: String,
178 pub paths: Vec<String>,
179 #[serde(default, alias = "context-tokens")]
180 pub context_tokens: Option<usize>,
181 pub metadata: BTreeMap<String, JsonValue>,
182}
183
184#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
185#[serde(default)]
186pub struct SkillGateCaseResult {
187 #[serde(default, alias = "task-id")]
188 pub task_id: String,
189 pub score: Option<f64>,
190 pub passed: Option<bool>,
191 pub notes: Option<String>,
192 pub metadata: BTreeMap<String, JsonValue>,
193}
194
195#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
196#[serde(default)]
197pub struct SkillGateReport {
198 #[serde(rename = "_type")]
199 pub type_name: String,
200 pub schema_version: u32,
201 pub manifest_id: String,
202 pub manifest_name: Option<String>,
203 pub target_model: SkillGateTargetModel,
204 pub pass: bool,
205 pub selected_variant_id: Option<String>,
206 pub included_task_count: usize,
207 pub excluded_task_count: usize,
208 pub task_safety: Vec<SkillGateTaskSafetyReport>,
209 pub tamper: SkillGateTamperReport,
210 pub variants: Vec<SkillGateVariantReport>,
211 pub pareto_frontier: Vec<String>,
212 pub receipt: SkillGateReceipt,
213 pub metadata: BTreeMap<String, JsonValue>,
214}
215
216#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
217#[serde(default)]
218pub struct SkillGateTaskSafetyReport {
219 pub task_id: String,
220 pub cluster: String,
221 pub included: bool,
222 pub heldout_kind: String,
223 pub created_at: Option<String>,
224 pub private: bool,
225 pub exclusion_reason: Option<String>,
226}
227
228#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
229#[serde(default)]
230pub struct SkillGateTamperReport {
231 pub pass: bool,
232 pub checks: Vec<SkillGateTamperCheck>,
233 pub failures: Vec<String>,
234}
235
236#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
237#[serde(default)]
238pub struct SkillGateTamperCheck {
239 pub path: String,
240 pub label: Option<String>,
241 pub expected_sha256: String,
242 pub actual_sha256: Option<String>,
243 pub status: String,
244 pub failure: Option<String>,
245}
246
247#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
248#[serde(default)]
249pub struct SkillGateVariantReport {
250 pub id: String,
251 pub name: Option<String>,
252 pub accepted: bool,
253 pub decision: String,
254 pub failures: Vec<String>,
255 pub warnings: Vec<String>,
256 pub metrics: SkillGateVariantMetrics,
257 pub context: SkillGateContextReport,
258 pub clusters: Vec<SkillGateClusterReport>,
259 pub cases: Vec<SkillGateCaseReport>,
260}
261
262#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
263#[serde(default)]
264pub struct SkillGateVariantMetrics {
265 pub included_task_count: usize,
266 pub scored_task_count: usize,
267 pub gap_task_count: usize,
268 pub mean_baseline_score: f64,
269 pub mean_candidate_score: f64,
270 pub mean_frontier_score: f64,
271 pub mean_score_lift: f64,
272 pub mean_gap_recovery: f64,
273 pub candidate_win_count: usize,
274 pub candidate_tie_count: usize,
275 pub candidate_loss_count: usize,
276 pub win_rate: f64,
277 pub regression_count: usize,
278 pub regression_denominator: usize,
279 pub regression_rate: f64,
280}
281
282#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
283#[serde(default)]
284pub struct SkillGateContextReport {
285 pub baseline_tokens: usize,
286 pub candidate_tokens: usize,
287 pub delta_tokens: i64,
288 pub max_delta_tokens: Option<i64>,
289 pub target_context_budget_tokens: Option<usize>,
290 pub within_delta_budget: bool,
291 pub within_target_budget: bool,
292 pub artifact_hashes: Vec<SkillGateArtifactHash>,
293}
294
295#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
296#[serde(default)]
297pub struct SkillGateArtifactHash {
298 pub role: String,
299 pub path: String,
300 pub sha256: String,
301 pub tokens: usize,
302 pub bytes: usize,
303}
304
305#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
306#[serde(default)]
307pub struct SkillGateClusterReport {
308 pub cluster: String,
309 pub task_count: usize,
310 pub gap_task_count: usize,
311 pub mean_baseline_score: f64,
312 pub mean_candidate_score: f64,
313 pub mean_frontier_score: f64,
314 pub mean_score_lift: f64,
315 pub mean_gap_recovery: f64,
316 pub pass: bool,
317}
318
319#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
320#[serde(default)]
321pub struct SkillGateCaseReport {
322 pub task_id: String,
323 pub cluster: String,
324 pub included: bool,
325 pub exclusion_reason: Option<String>,
326 pub baseline_score: f64,
327 pub candidate_score: Option<f64>,
328 pub frontier_score: f64,
329 pub score_lift: Option<f64>,
330 pub gap_recovery: Option<f64>,
331 pub baseline_passed: bool,
332 pub candidate_passed: Option<bool>,
333 pub regression: bool,
334 pub failures: Vec<String>,
335 pub notes: Option<String>,
336}
337
338#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
339#[serde(default)]
340pub struct SkillGateReceipt {
341 #[serde(rename = "_type")]
342 pub type_name: String,
343 pub schema_version: u32,
344 pub manifest_id: String,
345 pub target_model_id: String,
346 pub accepted: bool,
347 pub selected_variant_id: Option<String>,
348 pub decision: String,
349 pub metrics: Option<SkillGateVariantMetrics>,
350 pub context: Option<SkillGateContextReport>,
351 pub tamper: SkillGateTamperReport,
352 pub pareto_frontier: Vec<String>,
353 pub excluded_task_ids: Vec<String>,
354 pub variant_receipts: Vec<SkillGateVariantReceipt>,
355 pub metadata: BTreeMap<String, JsonValue>,
356}
357
358#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
359#[serde(default)]
360pub struct SkillGateVariantReceipt {
361 pub variant_id: String,
362 pub accepted: bool,
363 pub decision: String,
364 pub metrics: SkillGateVariantMetrics,
365 pub context_delta_tokens: i64,
366 pub failures: Vec<String>,
367}
368
369pub fn load_skill_gate_manifest(path: &Path) -> Result<SkillGateManifest, VmError> {
370 let content = fs::read_to_string(path).map_err(|error| {
371 VmError::Runtime(format!("failed to read skill gate manifest: {error}"))
372 })?;
373 let mut manifest: SkillGateManifest =
374 if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
375 toml::from_str(&content).map_err(|error| {
376 VmError::Runtime(format!("failed to parse skill gate TOML: {error}"))
377 })?
378 } else {
379 serde_json::from_str(&content).map_err(|error| {
380 VmError::Runtime(format!("failed to parse skill gate JSON: {error}"))
381 })?
382 };
383 if manifest.base_dir.is_none() {
384 manifest.base_dir = path.parent().map(|parent| parent.display().to_string());
385 }
386 normalize_skill_gate_manifest(&mut manifest)?;
387 Ok(manifest)
388}
389
390pub fn evaluate_skill_gate_manifest(
391 manifest: &SkillGateManifest,
392) -> Result<SkillGateReport, VmError> {
393 let mut manifest = manifest.clone();
394 normalize_skill_gate_manifest(&mut manifest)?;
395 let base_dir = manifest.base_dir.as_deref().map(Path::new);
396 let task_safety = manifest
397 .tasks
398 .iter()
399 .map(|task| task_safety_report(task, &manifest.target_model))
400 .collect::<Vec<_>>();
401 let included_task_count = task_safety.iter().filter(|task| task.included).count();
402 let excluded_task_count = task_safety.len().saturating_sub(included_task_count);
403 let safety_by_id = task_safety
404 .iter()
405 .map(|task| (task.task_id.as_str(), task))
406 .collect::<BTreeMap<_, _>>();
407 let tamper = verify_immutable_grader(&manifest.grader, base_dir);
408 let variants = manifest
409 .variants
410 .iter()
411 .map(|variant| evaluate_variant(variant, &manifest, &safety_by_id, &tamper, base_dir))
412 .collect::<Vec<_>>();
413 let pareto_frontier = pareto_frontier(&variants);
414 let selected_variant_id = select_variant(&variants, &pareto_frontier);
415 let pass = selected_variant_id.is_some();
416 let receipt = build_receipt(
417 &manifest,
418 pass,
419 selected_variant_id.clone(),
420 &task_safety,
421 tamper.clone(),
422 &variants,
423 pareto_frontier.clone(),
424 );
425 Ok(SkillGateReport {
426 type_name: SKILL_GATE_REPORT_TYPE.to_string(),
427 schema_version: SKILL_GATE_SCHEMA_VERSION,
428 manifest_id: manifest.id,
429 manifest_name: manifest.name,
430 target_model: manifest.target_model,
431 pass,
432 selected_variant_id,
433 included_task_count,
434 excluded_task_count,
435 task_safety,
436 tamper,
437 variants,
438 pareto_frontier,
439 receipt,
440 metadata: manifest.metadata,
441 })
442}
443
444fn normalize_skill_gate_manifest(manifest: &mut SkillGateManifest) -> Result<(), VmError> {
445 if manifest.type_name.is_empty() {
446 manifest.type_name = SKILL_GATE_MANIFEST_TYPE.to_string();
447 }
448 if manifest.type_name != SKILL_GATE_MANIFEST_TYPE {
449 return Err(VmError::Runtime(format!(
450 "skill gate manifest _type must be {SKILL_GATE_MANIFEST_TYPE}"
451 )));
452 }
453 if manifest.version == 0 {
454 manifest.version = SKILL_GATE_SCHEMA_VERSION;
455 }
456 if manifest.version != SKILL_GATE_SCHEMA_VERSION {
457 return Err(VmError::Runtime(format!(
458 "skill gate manifest version must be {SKILL_GATE_SCHEMA_VERSION}"
459 )));
460 }
461 if manifest.id.trim().is_empty() {
462 manifest.id = "skill-gate".to_string();
463 }
464 if manifest.target_model.id.trim().is_empty() {
465 return Err(VmError::Runtime(
466 "skill gate manifest target_model.id is required".to_string(),
467 ));
468 }
469 if manifest.tasks.is_empty() {
470 return Err(VmError::Runtime(
471 "skill gate manifest must declare at least one task".to_string(),
472 ));
473 }
474 if manifest.variants.is_empty() {
475 return Err(VmError::Runtime(
476 "skill gate manifest must declare at least one variant".to_string(),
477 ));
478 }
479 let mut task_ids = BTreeSet::new();
480 for (index, task) in manifest.tasks.iter_mut().enumerate() {
481 if task.id.trim().is_empty() {
482 task.id = format!("task_{}", index + 1);
483 }
484 if !task_ids.insert(task.id.clone()) {
485 return Err(VmError::Runtime(format!(
486 "skill gate manifest has duplicate task id '{}'",
487 task.id
488 )));
489 }
490 if task.cluster.trim().is_empty() {
491 task.cluster = "default".to_string();
492 }
493 validate_score("baseline_score", &task.id, task.baseline_score)?;
494 validate_score("frontier_score", &task.id, task.frontier_score)?;
495 }
496 let mut variant_ids = BTreeSet::new();
497 for (index, variant) in manifest.variants.iter_mut().enumerate() {
498 if variant.id.trim().is_empty() {
499 variant.id = format!("variant_{}", index + 1);
500 }
501 if !variant_ids.insert(variant.id.clone()) {
502 return Err(VmError::Runtime(format!(
503 "skill gate manifest has duplicate variant id '{}'",
504 variant.id
505 )));
506 }
507 let mut result_ids = BTreeSet::new();
508 for result in &variant.case_results {
509 if result.task_id.trim().is_empty() {
510 return Err(VmError::Runtime(format!(
511 "skill gate variant '{}' has a case result with no task_id",
512 variant.id
513 )));
514 }
515 if !task_ids.contains(&result.task_id) {
516 return Err(VmError::Runtime(format!(
517 "skill gate variant '{}' references unknown task '{}'",
518 variant.id, result.task_id
519 )));
520 }
521 if !result_ids.insert(result.task_id.clone()) {
522 return Err(VmError::Runtime(format!(
523 "skill gate variant '{}' has duplicate result for task '{}'",
524 variant.id, result.task_id
525 )));
526 }
527 if let Some(score) = result.score {
528 validate_score("candidate score", &result.task_id, score)?;
529 }
530 }
531 }
532 Ok(())
533}
534
535fn validate_score(label: &str, task_id: &str, score: f64) -> Result<(), VmError> {
536 if !(0.0..=1.0).contains(&score) {
537 return Err(VmError::Runtime(format!(
538 "skill gate task '{task_id}' {label} must be between 0 and 1"
539 )));
540 }
541 Ok(())
542}
543
544fn task_safety_report(
545 task: &SkillGateTask,
546 target_model: &SkillGateTargetModel,
547) -> SkillGateTaskSafetyReport {
548 let kind = normalize_kind(&task.heldout.kind);
549 let (included, exclusion_reason) = if task.heldout.private || kind == "private" {
550 (true, None)
551 } else if matches!(kind.as_str(), "public_static" | "static" | "pre_cutoff") {
552 (
553 false,
554 Some("static public or declared pre-cutoff task is contamination-prone".to_string()),
555 )
556 } else if matches!(
557 kind.as_str(),
558 "post_cutoff" | "rolling" | "livecodebench" | "swe_mera" | "swe_rebench"
559 ) {
560 match (
561 task.heldout.created_at.as_deref(),
562 target_model.knowledge_cutoff.as_deref(),
563 ) {
564 (Some(created_at), Some(cutoff)) if date_after(created_at, cutoff).unwrap_or(false) => {
565 (true, None)
566 }
567 (Some(_), Some(cutoff)) => (
568 false,
569 Some(format!(
570 "task does not post-date target model cutoff {cutoff}"
571 )),
572 ),
573 (Some(_), None) => (
574 false,
575 Some(
576 "target model knowledge_cutoff is required for non-private held-out tasks"
577 .to_string(),
578 ),
579 ),
580 (None, _) => (
581 false,
582 Some("non-private held-out task must declare created_at".to_string()),
583 ),
584 }
585 } else {
586 (
587 false,
588 Some(format!(
589 "held-out kind '{}' is not recognized as contamination-safe",
590 task.heldout.kind
591 )),
592 )
593 };
594
595 SkillGateTaskSafetyReport {
596 task_id: task.id.clone(),
597 cluster: task.cluster.clone(),
598 included,
599 heldout_kind: task.heldout.kind.clone(),
600 created_at: task.heldout.created_at.clone(),
601 private: task.heldout.private,
602 exclusion_reason,
603 }
604}
605
606fn normalize_kind(kind: &str) -> String {
607 kind.trim().to_ascii_lowercase().replace(['-', ' '], "_")
608}
609
610fn date_after(created_at: &str, cutoff: &str) -> Option<bool> {
611 Some(parse_date_prefix(created_at)? > parse_date_prefix(cutoff)?)
612}
613
614#[expect(
615 clippy::string_slice,
616 reason = "prefix is 10 bytes verified ASCII digits and dashes above"
617)]
618fn parse_date_prefix(value: &str) -> Option<(u32, u32, u32)> {
619 let trimmed = value.trim();
620 let prefix = trimmed.get(..10)?;
621 let bytes = prefix.as_bytes();
622 if bytes.get(4) != Some(&b'-') || bytes.get(7) != Some(&b'-') {
623 return None;
624 }
625 for index in [0, 1, 2, 3, 5, 6, 8, 9] {
626 if !bytes[index].is_ascii_digit() {
627 return None;
628 }
629 }
630 let year = prefix[0..4].parse::<u32>().ok()?;
631 let month = prefix[5..7].parse::<u32>().ok()?;
632 let day = prefix[8..10].parse::<u32>().ok()?;
633 let max_day = match month {
634 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
635 4 | 6 | 9 | 11 => 30,
636 2 if is_leap_year(year) => 29,
637 2 => 28,
638 _ => return None,
639 };
640 if day == 0 || day > max_day {
641 None
642 } else {
643 Some((year, month, day))
644 }
645}
646
647fn is_leap_year(year: u32) -> bool {
648 (year.is_multiple_of(4) && !year.is_multiple_of(100)) || year.is_multiple_of(400)
649}
650
651fn verify_immutable_grader(
652 grader: &SkillGateGrader,
653 base_dir: Option<&Path>,
654) -> SkillGateTamperReport {
655 let mut checks = Vec::new();
656 let mut failures = Vec::new();
657 for protected in &grader.immutable_paths {
658 let resolved = resolve_manifest_path(base_dir, &protected.path);
659 let mut check = SkillGateTamperCheck {
660 path: protected.path.clone(),
661 label: protected.label.clone(),
662 expected_sha256: protected.sha256.clone(),
663 ..Default::default()
664 };
665 match sha256_path(&resolved) {
666 Ok(hash) => {
667 check.actual_sha256 = Some(hash.sha256.clone());
668 if hash.sha256.eq_ignore_ascii_case(protected.sha256.trim()) {
669 check.status = "pass".to_string();
670 } else {
671 check.status = "fail".to_string();
672 check.failure = Some(format!(
673 "checksum mismatch for immutable grader path {}",
674 protected.path
675 ));
676 }
677 }
678 Err(error) => {
679 check.status = "fail".to_string();
680 check.failure = Some(error);
681 }
682 }
683 if let Some(failure) = &check.failure {
684 failures.push(failure.clone());
685 }
686 checks.push(check);
687 }
688 SkillGateTamperReport {
689 pass: failures.is_empty(),
690 checks,
691 failures,
692 }
693}
694
695fn evaluate_variant(
696 variant: &SkillGateVariant,
697 manifest: &SkillGateManifest,
698 safety_by_id: &BTreeMap<&str, &SkillGateTaskSafetyReport>,
699 tamper: &SkillGateTamperReport,
700 base_dir: Option<&Path>,
701) -> SkillGateVariantReport {
702 let mut failures = Vec::new();
703 let mut warnings = Vec::new();
704 let mut context_valid = true;
705 let context = match measure_context(variant, manifest, base_dir) {
706 Ok(context) => context,
707 Err(error) => {
708 context_valid = false;
709 failures.push(error);
710 SkillGateContextReport::default()
711 }
712 };
713 let results_by_task = variant
714 .case_results
715 .iter()
716 .map(|result| (result.task_id.as_str(), result))
717 .collect::<BTreeMap<_, _>>();
718 let cases = manifest
719 .tasks
720 .iter()
721 .map(|task| {
722 evaluate_case(
723 task,
724 results_by_task.get(task.id.as_str()).copied(),
725 safety_by_id.get(task.id.as_str()).copied(),
726 manifest.policy.pass_score_threshold(),
727 )
728 })
729 .collect::<Vec<_>>();
730 for case in &cases {
731 failures.extend(case.failures.iter().cloned());
732 }
733 let metrics = aggregate_variant_metrics(&cases);
734 let clusters = aggregate_cluster_reports(&cases, manifest.policy.min_cluster_gap_recovery());
735 if !tamper.pass && manifest.policy.require_no_tamper() {
736 failures.push("immutable grader check failed".to_string());
737 }
738 if metrics.included_task_count < manifest.policy.min_included_tasks() {
739 failures.push(format!(
740 "included held-out task count {} is below required {}",
741 metrics.included_task_count,
742 manifest.policy.min_included_tasks()
743 ));
744 }
745 if metrics.scored_task_count == 0 {
746 failures.push("no contamination-safe scored tasks were available".to_string());
747 }
748 if metrics.mean_score_lift + EPSILON < manifest.policy.min_score_lift() {
749 failures.push(format!(
750 "mean score lift {:.4} is below required {:.4}",
751 metrics.mean_score_lift,
752 manifest.policy.min_score_lift()
753 ));
754 }
755 if metrics.mean_gap_recovery + EPSILON < manifest.policy.min_gap_recovery() {
756 failures.push(format!(
757 "mean gap recovery {:.4} is below required {:.4}",
758 metrics.mean_gap_recovery,
759 manifest.policy.min_gap_recovery()
760 ));
761 }
762 if metrics.regression_rate > manifest.policy.max_regression_rate() + EPSILON {
763 failures.push(format!(
764 "regression rate {:.4} exceeds allowed {:.4}",
765 metrics.regression_rate,
766 manifest.policy.max_regression_rate()
767 ));
768 }
769 if let Some(min_win_rate) = manifest.policy.min_win_rate {
770 if metrics.win_rate + EPSILON < min_win_rate {
771 failures.push(format!(
772 "candidate win rate {:.4} is below required {:.4}",
773 metrics.win_rate, min_win_rate
774 ));
775 }
776 }
777 if context_valid && !context.within_delta_budget {
778 failures.push(format!(
779 "context delta {} tokens exceeds allowed {}",
780 context.delta_tokens,
781 context.max_delta_tokens.unwrap_or_default()
782 ));
783 }
784 if context_valid && !context.within_target_budget {
785 failures.push(format!(
786 "candidate context {} tokens exceeds target budget {}",
787 context.candidate_tokens,
788 context.target_context_budget_tokens.unwrap_or_default()
789 ));
790 }
791 if manifest.policy.require_cluster_lift {
792 for cluster in &clusters {
793 if !cluster.pass {
794 failures.push(format!(
795 "cluster '{}' gap recovery {:.4} is below required {:.4}",
796 cluster.cluster,
797 cluster.mean_gap_recovery,
798 manifest.policy.min_cluster_gap_recovery()
799 ));
800 }
801 }
802 }
803 if manifest.grader.immutable_paths.is_empty() {
804 let warning = "no immutable grader paths were declared".to_string();
805 if manifest.policy.require_no_tamper() {
806 failures.push(warning.clone());
807 }
808 warnings.push(warning);
809 }
810 let accepted = failures.is_empty();
811 SkillGateVariantReport {
812 id: variant.id.clone(),
813 name: variant.name.clone(),
814 accepted,
815 decision: if accepted {
816 "accepted".to_string()
817 } else {
818 "rejected".to_string()
819 },
820 failures,
821 warnings,
822 metrics,
823 context,
824 clusters,
825 cases,
826 }
827}
828
829fn evaluate_case(
830 task: &SkillGateTask,
831 result: Option<&SkillGateCaseResult>,
832 safety: Option<&SkillGateTaskSafetyReport>,
833 pass_score_threshold: f64,
834) -> SkillGateCaseReport {
835 let included = safety.is_none_or(|safety| safety.included);
836 let exclusion_reason = safety.and_then(|safety| safety.exclusion_reason.clone());
837 let baseline_passed = task
838 .baseline_passed
839 .unwrap_or(task.baseline_score >= pass_score_threshold);
840 let mut report = SkillGateCaseReport {
841 task_id: task.id.clone(),
842 cluster: task.cluster.clone(),
843 included,
844 exclusion_reason,
845 baseline_score: task.baseline_score,
846 candidate_score: result.and_then(|result| result.score),
847 frontier_score: task.frontier_score,
848 baseline_passed,
849 candidate_passed: result.map(|result| {
850 result
851 .passed
852 .unwrap_or_else(|| result.score.unwrap_or(0.0) >= pass_score_threshold)
853 }),
854 notes: result.and_then(|result| result.notes.clone()),
855 ..Default::default()
856 };
857 if !included {
858 return report;
859 }
860 let Some(candidate_score) = report.candidate_score else {
861 report
862 .failures
863 .push(format!("variant is missing result for task '{}'", task.id));
864 return report;
865 };
866 let score_lift = candidate_score - task.baseline_score;
867 report.score_lift = Some(score_lift);
868 if task.frontier_score > task.baseline_score + EPSILON {
869 report.gap_recovery = Some(score_lift / (task.frontier_score - task.baseline_score));
870 }
871 let candidate_passed = report.candidate_passed.unwrap_or(false);
872 report.regression = baseline_passed && !candidate_passed;
873 if report.regression {
874 report.failures.push(format!(
875 "task '{}' regressed from passing to failing",
876 task.id
877 ));
878 }
879 report
880}
881
882fn aggregate_variant_metrics(cases: &[SkillGateCaseReport]) -> SkillGateVariantMetrics {
883 let included_task_count = cases.iter().filter(|case| case.included).count();
884 let scored = cases
885 .iter()
886 .filter(|case| case.included && case.candidate_score.is_some())
887 .collect::<Vec<_>>();
888 let scored_task_count = scored.len();
889 let gap_cases = scored
890 .iter()
891 .filter(|case| case.gap_recovery.is_some())
892 .collect::<Vec<_>>();
893 let gap_task_count = gap_cases.len();
894 let mut metrics = SkillGateVariantMetrics {
895 included_task_count,
896 scored_task_count,
897 gap_task_count,
898 ..Default::default()
899 };
900 if scored_task_count > 0 {
901 metrics.mean_baseline_score =
902 scored.iter().map(|case| case.baseline_score).sum::<f64>() / scored_task_count as f64;
903 metrics.mean_candidate_score = scored
904 .iter()
905 .map(|case| case.candidate_score.unwrap_or_default())
906 .sum::<f64>()
907 / scored_task_count as f64;
908 metrics.mean_frontier_score =
909 scored.iter().map(|case| case.frontier_score).sum::<f64>() / scored_task_count as f64;
910 metrics.mean_score_lift = scored
911 .iter()
912 .map(|case| case.score_lift.unwrap_or_default())
913 .sum::<f64>()
914 / scored_task_count as f64;
915 metrics.candidate_win_count = scored
916 .iter()
917 .filter(|case| case.score_lift.unwrap_or_default() > EPSILON)
918 .count();
919 metrics.candidate_loss_count = scored
920 .iter()
921 .filter(|case| case.score_lift.unwrap_or_default() < -EPSILON)
922 .count();
923 metrics.candidate_tie_count = scored_task_count
924 .saturating_sub(metrics.candidate_win_count + metrics.candidate_loss_count);
925 metrics.win_rate = metrics.candidate_win_count as f64 / scored_task_count as f64;
926 }
927 if gap_task_count > 0 {
928 metrics.mean_gap_recovery = gap_cases
929 .iter()
930 .map(|case| case.gap_recovery.unwrap_or_default())
931 .sum::<f64>()
932 / gap_task_count as f64;
933 }
934 metrics.regression_denominator = cases
935 .iter()
936 .filter(|case| case.included && case.baseline_passed)
937 .count();
938 metrics.regression_count = cases
939 .iter()
940 .filter(|case| case.included && case.regression)
941 .count();
942 if metrics.regression_denominator > 0 {
943 metrics.regression_rate =
944 metrics.regression_count as f64 / metrics.regression_denominator as f64;
945 }
946 metrics
947}
948
949fn aggregate_cluster_reports(
950 cases: &[SkillGateCaseReport],
951 min_cluster_gap_recovery: f64,
952) -> Vec<SkillGateClusterReport> {
953 let mut grouped: BTreeMap<String, Vec<&SkillGateCaseReport>> = BTreeMap::new();
954 for case in cases
955 .iter()
956 .filter(|case| case.included && case.candidate_score.is_some())
957 {
958 grouped.entry(case.cluster.clone()).or_default().push(case);
959 }
960 grouped
961 .into_iter()
962 .map(|(cluster, cases)| {
963 let task_count = cases.len();
964 let gap_cases = cases
965 .iter()
966 .filter(|case| case.gap_recovery.is_some())
967 .copied()
968 .collect::<Vec<_>>();
969 let gap_task_count = gap_cases.len();
970 let mean_baseline_score =
971 cases.iter().map(|case| case.baseline_score).sum::<f64>() / task_count as f64;
972 let mean_candidate_score = cases
973 .iter()
974 .map(|case| case.candidate_score.unwrap_or_default())
975 .sum::<f64>()
976 / task_count as f64;
977 let mean_frontier_score =
978 cases.iter().map(|case| case.frontier_score).sum::<f64>() / task_count as f64;
979 let mean_score_lift = cases
980 .iter()
981 .map(|case| case.score_lift.unwrap_or_default())
982 .sum::<f64>()
983 / task_count as f64;
984 let mean_gap_recovery = if gap_task_count == 0 {
985 0.0
986 } else {
987 gap_cases
988 .iter()
989 .map(|case| case.gap_recovery.unwrap_or_default())
990 .sum::<f64>()
991 / gap_task_count as f64
992 };
993 SkillGateClusterReport {
994 cluster,
995 task_count,
996 gap_task_count,
997 mean_baseline_score,
998 mean_candidate_score,
999 mean_frontier_score,
1000 mean_score_lift,
1001 mean_gap_recovery,
1002 pass: gap_task_count == 0
1003 || mean_gap_recovery + EPSILON >= min_cluster_gap_recovery,
1004 }
1005 })
1006 .collect()
1007}
1008
1009fn measure_context(
1010 variant: &SkillGateVariant,
1011 manifest: &SkillGateManifest,
1012 base_dir: Option<&Path>,
1013) -> Result<SkillGateContextReport, String> {
1014 let baseline = measure_artifact("baseline", &variant.baseline, base_dir)?;
1015 let candidate = measure_artifact("candidate", &variant.candidate, base_dir)?;
1016 let baseline_tokens = baseline.context_tokens;
1017 let candidate_tokens = candidate.context_tokens;
1018 let delta_tokens = candidate_tokens as i64 - baseline_tokens as i64;
1019 let max_delta_tokens = manifest.policy.max_context_delta_tokens;
1020 let target_context_budget_tokens = manifest.target_model.context_budget_tokens;
1021 let within_delta_budget = max_delta_tokens.is_none_or(|max| delta_tokens <= max);
1022 let within_target_budget =
1023 target_context_budget_tokens.is_none_or(|max| candidate_tokens <= max);
1024 let mut artifact_hashes = baseline.hashes;
1025 artifact_hashes.extend(candidate.hashes);
1026 Ok(SkillGateContextReport {
1027 baseline_tokens,
1028 candidate_tokens,
1029 delta_tokens,
1030 max_delta_tokens,
1031 target_context_budget_tokens,
1032 within_delta_budget,
1033 within_target_budget,
1034 artifact_hashes,
1035 })
1036}
1037
1038#[derive(Debug, Default)]
1039struct ArtifactMeasurement {
1040 context_tokens: usize,
1041 hashes: Vec<SkillGateArtifactHash>,
1042}
1043
1044fn measure_artifact(
1045 role: &str,
1046 artifact: &SkillGateArtifact,
1047 base_dir: Option<&Path>,
1048) -> Result<ArtifactMeasurement, String> {
1049 let mut measurement = ArtifactMeasurement::default();
1050 for path in &artifact.paths {
1051 let resolved = resolve_manifest_path(base_dir, path);
1052 let hash = sha256_path(&resolved)?;
1053 measurement.context_tokens += hash.tokens;
1054 measurement.hashes.push(SkillGateArtifactHash {
1055 role: role.to_string(),
1056 path: path.clone(),
1057 sha256: hash.sha256,
1058 tokens: hash.tokens,
1059 bytes: hash.bytes,
1060 });
1061 }
1062 if let Some(tokens) = artifact.context_tokens {
1063 measurement.context_tokens = tokens;
1064 }
1065 Ok(measurement)
1066}
1067
1068#[derive(Debug)]
1069struct PathHash {
1070 sha256: String,
1071 tokens: usize,
1072 bytes: usize,
1073}
1074
1075fn sha256_path(path: &Path) -> Result<PathHash, String> {
1076 let metadata = fs::symlink_metadata(path)
1077 .map_err(|error| format!("failed to stat {}: {error}", path.display()))?;
1078 if metadata.file_type().is_symlink() {
1079 return Err(format!(
1080 "refusing to hash symlink protected path {}",
1081 path.display()
1082 ));
1083 }
1084 if metadata.is_file() {
1085 return sha256_file(path);
1086 }
1087 if metadata.is_dir() {
1088 return sha256_dir(path);
1089 }
1090 Err(format!(
1091 "protected path {} is neither a file nor a directory",
1092 path.display()
1093 ))
1094}
1095
1096fn sha256_file(path: &Path) -> Result<PathHash, String> {
1097 let bytes =
1098 fs::read(path).map_err(|error| format!("failed to read {}: {error}", path.display()))?;
1099 let sha256 = hex_digest(&bytes);
1100 let tokens = estimate_chunk_tokens(&String::from_utf8_lossy(&bytes));
1101 Ok(PathHash {
1102 sha256,
1103 tokens,
1104 bytes: bytes.len(),
1105 })
1106}
1107
1108fn sha256_dir(path: &Path) -> Result<PathHash, String> {
1109 let mut files = Vec::new();
1110 for entry in WalkDir::new(path).follow_links(false) {
1111 let entry = entry.map_err(|error| format!("failed to walk {}: {error}", path.display()))?;
1112 if entry.file_type().is_symlink() {
1113 return Err(format!(
1114 "refusing to hash symlink inside protected directory {}",
1115 entry.path().display()
1116 ));
1117 }
1118 if entry.file_type().is_file() {
1119 files.push(entry.path().to_path_buf());
1120 }
1121 }
1122 files.sort();
1123 let mut hasher = Sha256::new();
1124 let mut tokens = 0;
1125 let mut bytes_total = 0;
1126 for file in files {
1127 let rel = file
1128 .strip_prefix(path)
1129 .map_err(|error| format!("failed to relativize {}: {error}", file.display()))?;
1130 let rel = rel.to_string_lossy().replace('\\', "/");
1131 let bytes = fs::read(&file)
1132 .map_err(|error| format!("failed to read {}: {error}", file.display()))?;
1133 hasher.update(rel.as_bytes());
1134 hasher.update([0]);
1135 hasher.update(&bytes);
1136 hasher.update([0xff]);
1137 tokens += estimate_chunk_tokens(&String::from_utf8_lossy(&bytes));
1138 bytes_total += bytes.len();
1139 }
1140 Ok(PathHash {
1141 sha256: bytes_to_hex(hasher.finalize().as_ref()),
1142 tokens,
1143 bytes: bytes_total,
1144 })
1145}
1146
1147fn hex_digest(bytes: &[u8]) -> String {
1148 let mut hasher = Sha256::new();
1149 hasher.update(bytes);
1150 bytes_to_hex(hasher.finalize().as_ref())
1151}
1152
1153fn bytes_to_hex(bytes: &[u8]) -> String {
1154 bytes.iter().map(|byte| format!("{byte:02x}")).collect()
1155}
1156
1157fn resolve_manifest_path(base_dir: Option<&Path>, path: &str) -> PathBuf {
1158 let path_buf = PathBuf::from(path);
1159 if path_buf.is_absolute() {
1160 path_buf
1161 } else if let Some(base_dir) = base_dir {
1162 base_dir.join(path_buf)
1163 } else {
1164 path_buf
1165 }
1166}
1167
1168fn pareto_frontier(variants: &[SkillGateVariantReport]) -> Vec<String> {
1169 variants
1170 .iter()
1171 .filter(|variant| variant.metrics.scored_task_count > 0)
1172 .filter(|variant| {
1173 !variants
1174 .iter()
1175 .any(|other| other.id != variant.id && dominates(other, variant))
1176 })
1177 .map(|variant| variant.id.clone())
1178 .collect()
1179}
1180
1181fn dominates(left: &SkillGateVariantReport, right: &SkillGateVariantReport) -> bool {
1182 if left.metrics.scored_task_count == 0 {
1183 return false;
1184 }
1185 let at_least_as_good = left.metrics.mean_gap_recovery + EPSILON
1186 >= right.metrics.mean_gap_recovery
1187 && left.metrics.mean_score_lift + EPSILON >= right.metrics.mean_score_lift
1188 && left.metrics.regression_rate <= right.metrics.regression_rate + EPSILON
1189 && left.context.delta_tokens <= right.context.delta_tokens;
1190 let strictly_better = left.metrics.mean_gap_recovery
1191 > right.metrics.mean_gap_recovery + EPSILON
1192 || left.metrics.mean_score_lift > right.metrics.mean_score_lift + EPSILON
1193 || left.metrics.regression_rate + EPSILON < right.metrics.regression_rate
1194 || left.context.delta_tokens < right.context.delta_tokens;
1195 at_least_as_good && strictly_better
1196}
1197
1198fn select_variant(
1199 variants: &[SkillGateVariantReport],
1200 pareto_frontier: &[String],
1201) -> Option<String> {
1202 let frontier = pareto_frontier.iter().collect::<BTreeSet<_>>();
1203 let mut accepted = variants
1204 .iter()
1205 .filter(|variant| variant.accepted && frontier.contains(&variant.id))
1206 .collect::<Vec<_>>();
1207 if accepted.is_empty() {
1208 accepted = variants.iter().filter(|variant| variant.accepted).collect();
1209 }
1210 accepted.sort_by(|left, right| {
1211 right
1212 .metrics
1213 .mean_gap_recovery
1214 .partial_cmp(&left.metrics.mean_gap_recovery)
1215 .unwrap_or(std::cmp::Ordering::Equal)
1216 .then_with(|| {
1217 right
1218 .metrics
1219 .mean_score_lift
1220 .partial_cmp(&left.metrics.mean_score_lift)
1221 .unwrap_or(std::cmp::Ordering::Equal)
1222 })
1223 .then_with(|| left.context.delta_tokens.cmp(&right.context.delta_tokens))
1224 .then_with(|| left.id.cmp(&right.id))
1225 });
1226 accepted.first().map(|variant| variant.id.clone())
1227}
1228
1229fn build_receipt(
1230 manifest: &SkillGateManifest,
1231 accepted: bool,
1232 selected_variant_id: Option<String>,
1233 task_safety: &[SkillGateTaskSafetyReport],
1234 tamper: SkillGateTamperReport,
1235 variants: &[SkillGateVariantReport],
1236 pareto_frontier: Vec<String>,
1237) -> SkillGateReceipt {
1238 let selected = selected_variant_id
1239 .as_ref()
1240 .and_then(|id| variants.iter().find(|variant| &variant.id == id));
1241 SkillGateReceipt {
1242 type_name: SKILL_GATE_RECEIPT_TYPE.to_string(),
1243 schema_version: SKILL_GATE_SCHEMA_VERSION,
1244 manifest_id: manifest.id.clone(),
1245 target_model_id: manifest.target_model.id.clone(),
1246 accepted,
1247 selected_variant_id,
1248 decision: if accepted {
1249 "accepted".to_string()
1250 } else {
1251 "rejected".to_string()
1252 },
1253 metrics: selected.map(|variant| variant.metrics.clone()),
1254 context: selected.map(|variant| variant.context.clone()),
1255 tamper,
1256 pareto_frontier,
1257 excluded_task_ids: task_safety
1258 .iter()
1259 .filter(|task| !task.included)
1260 .map(|task| task.task_id.clone())
1261 .collect(),
1262 variant_receipts: variants
1263 .iter()
1264 .map(|variant| SkillGateVariantReceipt {
1265 variant_id: variant.id.clone(),
1266 accepted: variant.accepted,
1267 decision: variant.decision.clone(),
1268 metrics: variant.metrics.clone(),
1269 context_delta_tokens: variant.context.delta_tokens,
1270 failures: variant.failures.clone(),
1271 })
1272 .collect(),
1273 metadata: manifest.metadata.clone(),
1274 }
1275}
1276
1277#[cfg(test)]
1278mod tests {
1279 use super::*;
1280
1281 fn write(path: &Path, content: &str) {
1282 fs::create_dir_all(path.parent().unwrap()).unwrap();
1283 fs::write(path, content).unwrap();
1284 }
1285
1286 fn fixture_manifest(root: &Path, grader_hash: String) -> SkillGateManifest {
1287 SkillGateManifest {
1288 type_name: SKILL_GATE_MANIFEST_TYPE.to_string(),
1289 version: SKILL_GATE_SCHEMA_VERSION,
1290 id: "skill-gate-test".to_string(),
1291 base_dir: Some(root.display().to_string()),
1292 target_model: SkillGateTargetModel {
1293 id: "mock-cheap".to_string(),
1294 knowledge_cutoff: Some("2026-05-01".to_string()),
1295 context_budget_tokens: Some(220),
1296 ..Default::default()
1297 },
1298 policy: SkillGatePolicy {
1299 min_included_tasks: Some(2),
1300 min_score_lift: Some(0.10),
1301 min_gap_recovery: Some(0.25),
1302 max_regression_rate: Some(0.0),
1303 max_context_delta_tokens: Some(120),
1304 min_win_rate: Some(0.5),
1305 ..Default::default()
1306 },
1307 grader: SkillGateGrader {
1308 id: "immutable".to_string(),
1309 immutable_paths: vec![SkillGateProtectedPath {
1310 path: "grader/check.txt".to_string(),
1311 sha256: grader_hash,
1312 label: Some("grader".to_string()),
1313 }],
1314 ..Default::default()
1315 },
1316 tasks: vec![
1317 SkillGateTask {
1318 id: "post-cutoff-failure".to_string(),
1319 cluster: "api-drift".to_string(),
1320 heldout: SkillGateHeldout {
1321 kind: "post_cutoff".to_string(),
1322 created_at: Some("2026-05-20".to_string()),
1323 ..Default::default()
1324 },
1325 baseline_score: 0.20,
1326 frontier_score: 1.0,
1327 baseline_passed: Some(false),
1328 ..Default::default()
1329 },
1330 SkillGateTask {
1331 id: "private-regression-check".to_string(),
1332 cluster: "regression".to_string(),
1333 heldout: SkillGateHeldout {
1334 kind: "private".to_string(),
1335 private: true,
1336 ..Default::default()
1337 },
1338 baseline_score: 0.90,
1339 frontier_score: 1.0,
1340 baseline_passed: Some(true),
1341 ..Default::default()
1342 },
1343 SkillGateTask {
1344 id: "old-public-benchmark".to_string(),
1345 cluster: "contaminated".to_string(),
1346 heldout: SkillGateHeldout {
1347 kind: "public_static".to_string(),
1348 created_at: Some("2024-01-01".to_string()),
1349 ..Default::default()
1350 },
1351 baseline_score: 0.0,
1352 frontier_score: 1.0,
1353 baseline_passed: Some(false),
1354 ..Default::default()
1355 },
1356 ],
1357 variants: vec![
1358 SkillGateVariant {
1359 id: "known-good".to_string(),
1360 candidate: SkillGateArtifact {
1361 kind: "skill".to_string(),
1362 paths: vec!["skills/good/SKILL.md".to_string()],
1363 ..Default::default()
1364 },
1365 case_results: vec![
1366 SkillGateCaseResult {
1367 task_id: "post-cutoff-failure".to_string(),
1368 score: Some(0.80),
1369 passed: Some(true),
1370 ..Default::default()
1371 },
1372 SkillGateCaseResult {
1373 task_id: "private-regression-check".to_string(),
1374 score: Some(0.92),
1375 passed: Some(true),
1376 ..Default::default()
1377 },
1378 SkillGateCaseResult {
1379 task_id: "old-public-benchmark".to_string(),
1380 score: Some(1.0),
1381 passed: Some(true),
1382 ..Default::default()
1383 },
1384 ],
1385 ..Default::default()
1386 },
1387 SkillGateVariant {
1388 id: "bloated".to_string(),
1389 candidate: SkillGateArtifact {
1390 kind: "skill".to_string(),
1391 paths: vec!["skills/bloat/SKILL.md".to_string()],
1392 ..Default::default()
1393 },
1394 case_results: vec![
1395 SkillGateCaseResult {
1396 task_id: "post-cutoff-failure".to_string(),
1397 score: Some(0.85),
1398 passed: Some(true),
1399 ..Default::default()
1400 },
1401 SkillGateCaseResult {
1402 task_id: "private-regression-check".to_string(),
1403 score: Some(0.91),
1404 passed: Some(true),
1405 ..Default::default()
1406 },
1407 ],
1408 ..Default::default()
1409 },
1410 ],
1411 ..Default::default()
1412 }
1413 }
1414
1415 #[test]
1416 fn gate_accepts_compact_lift_rejects_bloat_and_excludes_contamination() {
1417 let temp = tempfile::tempdir().unwrap();
1418 write(
1419 temp.path().join("grader/check.txt").as_path(),
1420 "stable grader\n",
1421 );
1422 write(
1423 temp.path().join("skills/good/SKILL.md").as_path(),
1424 "Use the post-cutoff API name and keep the answer scoped.\n",
1425 );
1426 write(
1427 temp.path().join("skills/bloat/SKILL.md").as_path(),
1428 &"repeat this irrelevant guidance for token bloat.\n".repeat(80),
1429 );
1430 let grader_hash = sha256_file(&temp.path().join("grader/check.txt"))
1431 .unwrap()
1432 .sha256;
1433 let report = evaluate_skill_gate_manifest(&fixture_manifest(temp.path(), grader_hash))
1434 .expect("gate evaluates");
1435
1436 assert!(report.pass);
1437 assert_eq!(report.selected_variant_id.as_deref(), Some("known-good"));
1438 assert_eq!(report.included_task_count, 2);
1439 assert_eq!(report.excluded_task_count, 1);
1440 assert_eq!(
1441 report.receipt.excluded_task_ids,
1442 vec!["old-public-benchmark"]
1443 );
1444 let good = report
1445 .variants
1446 .iter()
1447 .find(|variant| variant.id == "known-good")
1448 .unwrap();
1449 assert!(good.accepted);
1450 assert!(good.metrics.mean_gap_recovery > 0.25);
1451 assert_eq!(good.metrics.regression_rate, 0.0);
1452 let bloat = report
1453 .variants
1454 .iter()
1455 .find(|variant| variant.id == "bloated")
1456 .unwrap();
1457 assert!(!bloat.accepted);
1458 assert!(bloat
1459 .failures
1460 .iter()
1461 .any(|failure| failure.contains("context delta")));
1462 }
1463
1464 #[test]
1465 fn gate_fails_when_immutable_grader_checksum_changes() {
1466 let temp = tempfile::tempdir().unwrap();
1467 write(
1468 temp.path().join("grader/check.txt").as_path(),
1469 "stable grader\n",
1470 );
1471 write(
1472 temp.path().join("skills/good/SKILL.md").as_path(),
1473 "Use the post-cutoff API name and keep the answer scoped.\n",
1474 );
1475 write(
1476 temp.path().join("skills/bloat/SKILL.md").as_path(),
1477 &"repeat this irrelevant guidance for token bloat.\n".repeat(80),
1478 );
1479 let mut manifest = fixture_manifest(temp.path(), "not-the-real-hash".to_string());
1480 manifest.variants.truncate(1);
1481 let report = evaluate_skill_gate_manifest(&manifest).expect("gate evaluates");
1482
1483 assert!(!report.pass);
1484 assert!(!report.tamper.pass);
1485 assert!(report
1486 .variants
1487 .first()
1488 .unwrap()
1489 .failures
1490 .iter()
1491 .any(|failure| failure.contains("immutable grader")));
1492 }
1493}