use anyhow::{Context, Result};
use serde::de::DeserializeOwned;
use serde_json::Value;
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Component, Path, PathBuf};
use super::types::{
BenchVerifyFailure, BenchVerifyOptions, BenchVerifyReport, BenchmarkLayer, MemoryRunArtifact,
PublicBenchmarkManifest, PublicBenchmarkReport, VerifiedArtifact, VerifiedBenchmarkArtifacts,
};
pub(super) mod coding;
mod policy;
mod public_bundle;
pub(super) mod security_snapshot;
const MEMORY_ARTIFACT_KEYS: [&str; 5] = [
"reader_input",
"retrieved_evidence",
"answer",
"score",
"diagnosis",
];
pub fn verify_benchmark_artifacts(options: BenchVerifyOptions) -> Result<BenchVerifyReport> {
let root = options.root;
let claim_registry_path = options.claim_registry_path;
let mut state = VerifyState::new(root.clone());
let mut context = security_snapshot::VerificationContext::new();
if !root.exists() {
state.fail(".".to_string(), "benchmark root does not exist");
return Ok(state.finish());
}
if !root.is_dir() {
state.fail(".".to_string(), "benchmark root is not a directory");
return Ok(state.finish());
}
policy::load_claim_registry(&claim_registry_path, &mut state);
policy::validate_required_schemas(&mut state);
let manifest_paths = collect_manifest_paths(&root)?;
if manifest_paths.is_empty() {
state.fail(
rel_display(&root, &root),
"benchmark root has no manifests under a manifests/ directory",
);
}
for manifest_path in manifest_paths {
state.manifests_checked += 1;
let Some(manifest_artifact) =
read_json_artifact::<PublicBenchmarkManifest>(&manifest_path, &mut state, "manifest")
else {
continue;
};
let manifest = &manifest_artifact.value;
state
.verified_artifacts
.manifests
.push(manifest_artifact.clone());
validate_manifest(&manifest_path, manifest, &mut state);
for report_path in &manifest.reports {
let Some(report_abs) = resolve_public_path(&mut state, report_path, report_path) else {
continue;
};
validate_report_path_layer(
&manifest_path,
manifest,
&report_abs,
&mut state,
&mut context,
);
}
}
Ok(state.finish())
}
fn validate_manifest(path: &Path, manifest: &PublicBenchmarkManifest, state: &mut VerifyState) {
let label = rel_display(&state.root, path);
if manifest.schema_version != 1 {
state.fail(label.clone(), "manifest schema_version must be 1");
}
require_non_blank(&manifest.benchmark_id, &label, "benchmark_id", state);
require_non_blank(&manifest.version, &label, "version", state);
if manifest.created_at_epoch <= 0 {
state.fail(label.clone(), "manifest created_at_epoch must be positive");
}
if manifest.conditions.is_empty() {
state.fail(label.clone(), "manifest conditions must not be empty");
}
validate_condition_list(
&manifest.conditions,
manifest.layer,
&label,
"manifest conditions",
state,
);
if manifest.reports.is_empty() {
state.fail(label.clone(), "manifest reports must not be empty");
}
if manifest.source_policy.private_user_memory_allowed {
state.fail(
label.clone(),
"public benchmark manifest must not allow private user memory",
);
}
if !manifest.source_policy.requires_temp_remem_data_dir {
state.fail(
label,
"public benchmark manifest must require temporary REMEM_DATA_DIR isolation",
);
}
if let Some(revision) = manifest.source_policy.external_dataset_revision.as_deref() {
scan_private_string(
revision,
path,
"source_policy.external_dataset_revision",
state,
);
}
}
fn validate_report_path_layer(
manifest_path: &Path,
manifest: &PublicBenchmarkManifest,
report_path: &Path,
state: &mut VerifyState,
context: &mut security_snapshot::VerificationContext,
) {
state.reports_checked += 1;
let Some(report_artifact) =
read_json_artifact::<PublicBenchmarkReport>(report_path, state, "report")
else {
return;
};
let report = &report_artifact.value;
state
.verified_artifacts
.reports
.push(report_artifact.clone());
let label = rel_display(&state.root, report_path);
if report.schema_version != 1 {
state.fail(label.clone(), "report schema_version must be 1");
}
require_non_blank(&report.benchmark_id, &label, "benchmark_id", state);
require_non_blank(
&report.benchmark_version,
&label,
"benchmark_version",
state,
);
if report.layer == BenchmarkLayer::CodingAgentOutcome {
match report.run_phase.as_deref() {
Some(run_phase) => require_non_blank(run_phase, &label, "run_phase", state),
None => state.fail(label.clone(), "coding report run_phase is missing"),
}
match report.matrix_namespace.as_deref() {
Some(namespace) => require_non_blank(namespace, &label, "matrix_namespace", state),
None => state.fail(label.clone(), "coding report matrix_namespace is missing"),
}
}
require_non_blank(&report.claim_level, &label, "claim_level", state);
if report.layer != manifest.layer {
state.fail(
label.clone(),
"report layer must match the manifest layer that references it",
);
}
if report.benchmark_id != manifest.benchmark_id {
state.fail(
label.clone(),
"report benchmark_id must match the manifest benchmark_id",
);
}
if report.benchmark_version != manifest.version {
state.fail(
label.clone(),
format!(
"report benchmark_version {:?} must match manifest version {:?}",
report.benchmark_version, manifest.version
),
);
}
if report.conditions.is_empty() {
state.fail(label.clone(), "report conditions must not be empty");
}
validate_condition_list(
&report.conditions,
report.layer,
&label,
"report conditions",
state,
);
if condition_set(&report.conditions) != condition_set(&manifest.conditions) {
state.fail(
label.clone(),
"report conditions must exactly match manifest conditions",
);
}
if report.schema_refs.is_empty() {
state.fail(label.clone(), "report schema_refs must not be empty");
}
if report.run_artifacts.is_empty() {
state.fail(label.clone(), "report run_artifacts must not be empty");
}
if !report.verifier.required || report.verifier.schema_version != 1 {
state.fail(
label.clone(),
"report verifier metadata must require schema_version 1",
);
}
if report.aggregate_metrics.is_null() {
state.fail(label.clone(), "report aggregate_metrics must be present");
}
scan_private_json(
&serde_json::to_value(report).unwrap_or(Value::Null),
report_path,
"$",
state,
);
for schema_ref in &report.schema_refs {
let Some(schema_path) = resolve_public_path(state, schema_ref, schema_ref) else {
continue;
};
if !schema_path.exists() {
state.fail(schema_ref.clone(), "report schema_ref does not exist");
}
}
let mut represented_conditions = BTreeSet::new();
for run_artifact in &report.run_artifacts {
let Some(run_path) = resolve_public_path(state, run_artifact, run_artifact) else {
continue;
};
let run_label = rel_display(&state.root, &run_path);
if !state.run_artifact_paths.insert(run_label.clone()) {
state.fail(run_label, "run artifact path must be referenced only once");
continue;
}
let represented_condition = match report.layer {
BenchmarkLayer::MemorySystemCapability => {
validate_memory_run_artifact(&run_path, report, manifest_path, state, context)
}
BenchmarkLayer::CodingAgentOutcome => {
coding::validate_coding_run_artifact(&run_path, report, manifest_path, state)
}
};
if let Some(condition) = represented_condition {
represented_conditions.insert(condition);
}
}
if represented_conditions != condition_set(&report.conditions) {
state.fail(
label,
"report conditions must exactly match conditions represented by run artifacts",
);
}
}
fn validate_memory_run_artifact(
run_path: &Path,
report: &PublicBenchmarkReport,
_manifest_path: &Path,
state: &mut VerifyState,
context: &mut security_snapshot::VerificationContext,
) -> Option<String> {
state.run_artifacts_checked += 1;
let run_artifact =
read_json_artifact::<MemoryRunArtifact>(run_path, state, "memory run artifact")?;
let run = run_artifact.value.clone();
state.verified_artifacts.memory_runs.push(run_artifact);
let label = rel_display(&state.root, run_path);
if run.schema_version != 1 {
state.fail(label.clone(), "memory run schema_version must be 1");
}
if run.layer != BenchmarkLayer::MemorySystemCapability || run.layer != report.layer {
state.fail(
label.clone(),
"memory run layer must be memory_system_capability",
);
}
require_non_blank(&run.benchmark_id, &label, "benchmark_id", state);
if run.benchmark_id != report.benchmark_id {
state.fail(
label.clone(),
format!(
"memory run benchmark_id {:?} must match report benchmark_id {:?}",
run.benchmark_id, report.benchmark_id
),
);
}
require_non_blank(&run.benchmark_version, &label, "benchmark_version", state);
if run.benchmark_version != report.benchmark_version {
state.fail(
label.clone(),
format!(
"memory run benchmark_version {:?} must match report benchmark_version {:?}",
run.benchmark_version, report.benchmark_version
),
);
}
require_non_blank(&run.suite, &label, "suite", state);
match report.suite.as_deref() {
Some(report_suite) => {
require_non_blank(report_suite, &label, "report suite", state);
if run.suite != report_suite {
state.fail(
label.clone(),
format!(
"memory run suite {:?} must match report suite {:?}",
run.suite, report_suite
),
);
}
}
None => state.fail(label.clone(), "memory report suite is missing"),
}
require_non_blank(&run.condition, &label, "condition", state);
if !report.conditions.contains(&run.condition) {
state.fail(
label.clone(),
"memory run condition must be declared by its report",
);
}
require_non_blank(&run.task_id, &label, "task_id", state);
if run.reference_time_epoch <= 0 {
state.fail(
label.clone(),
"memory run reference_time_epoch must be positive",
);
}
validate_environment(&run.environment, &label, state);
if run.reader_model.is_null() {
state.fail(label.clone(), "memory run reader_model must be present");
}
if run.answer.is_null() {
state.fail(label.clone(), "memory run answer must be present");
}
if run.metrics.is_null() {
state.fail(label.clone(), "memory run metrics must be present");
}
if !run.diagnosis.write_side_gap
&& !run.diagnosis.retrieval_side_gap
&& !run.diagnosis.reader_gap
&& !run.diagnosis.policy_abstention
&& run
.diagnosis
.notes
.iter()
.any(|note| note.trim().is_empty())
{
state.fail(
label.clone(),
"memory run diagnosis notes must not be blank",
);
}
let abstained = run
.answer
.get("abstained")
.and_then(Value::as_bool)
.unwrap_or(false);
if run.retrieval.gold_supporting_event_ids.is_empty() {
state.fail(
label.clone(),
"memory run is missing gold supporting evidence IDs",
);
}
if !abstained && run.condition != "no_memory" && run.retrieval.retrieved_memory_ids.is_empty() {
state.fail(
label.clone(),
"memory run condition requires retrieved memory IDs",
);
}
let diagnosis_explains_abstention = run.diagnosis.policy_abstention
|| run.diagnosis.write_side_gap
|| run.diagnosis.retrieval_side_gap
|| run.diagnosis.reader_gap
|| run.condition == "no_memory";
if abstained && !diagnosis_explains_abstention {
state.fail(
label.clone(),
"abstained memory run must mark a diagnosis reason",
);
}
if !abstained {
if run.condition != "no_memory" && run.evidence.cited_memory_ids.is_empty() {
state.fail(label.clone(), "memory run is missing cited memory IDs");
}
if run.evidence.cited_event_ids.is_empty() {
state.fail(label.clone(), "memory run is missing cited event IDs");
}
}
for id in &run.retrieval.retrieved_supporting_evidence_ids {
require_non_blank(id, &label, "retrieved_supporting_evidence_ids", state);
}
for id in &run.retrieval.missing_supporting_evidence_ids {
require_non_blank(id, &label, "missing_supporting_evidence_ids", state);
}
validate_artifact_map(&run.artifacts, MEMORY_ARTIFACT_KEYS, &label, state);
if run.benchmark_id == "adversarial-policy" && run.benchmark_version == "v2" {
require_artifact_key(&run.artifacts, "remem_db_snapshot", &label, state);
validate_v2_memory_artifact_hashes(&run, &label, state);
for (path, message) in
public_bundle::unreferenced_run_entries(&state.root, run_path, &run.artifacts)
{
state.fail(path, message);
}
security_snapshot::validate_security_snapshot(&run, &label, state, context);
}
scan_private_json(
&serde_json::to_value(&run).unwrap_or(Value::Null),
run_path,
"$",
state,
);
Some(run.condition)
}
fn validate_v2_memory_artifact_hashes(
run: &MemoryRunArtifact,
label: &str,
state: &mut VerifyState,
) {
if let Some(raw_path) = run.artifacts.get("remem_db_snapshot") {
let Some(path) = resolve_public_path(state, raw_path, raw_path) else {
return;
};
if !security_snapshot::validate_snapshot_size(&path, raw_path, state) {
return;
}
}
if run.artifact_sha256.len() != run.artifacts.len() {
state.fail(
label.to_string(),
"v2 memory run must hash every declared artifact",
);
return;
}
for (key, raw_path) in &run.artifacts {
let Some(expected) = run.artifact_sha256.get(key) else {
state.fail(
label.to_string(),
format!("artifact {key} has no SHA-256 binding"),
);
continue;
};
let Some(path) = resolve_public_path(state, raw_path, raw_path) else {
continue;
};
match state.consume_file(&path, &format!("read artifact {key}")) {
Ok(bytes) if format!("{:x}", Sha256::digest(&bytes)) == *expected => {
if key != "remem_db_snapshot" {
if let Some(message) = public_bundle::private_payload_violation(&bytes) {
state.fail(raw_path.clone(), message);
}
}
}
Ok(_) => state.fail(raw_path.clone(), format!("artifact {key} SHA-256 mismatch")),
Err(()) => {}
}
}
}
fn validate_environment(env: &super::types::RunEnvironment, label: &str, state: &mut VerifyState) {
require_non_blank(&env.os, label, "environment.os", state);
require_non_blank(&env.arch, label, "environment.arch", state);
require_non_blank(&env.remem_commit, label, "environment.remem_commit", state);
require_non_blank(
&env.remem_data_dir,
label,
"environment.remem_data_dir",
state,
);
if !env.remem_data_dir.starts_with("temp://")
&& !env.remem_data_dir.starts_with("/tmp/")
&& !env.remem_data_dir.starts_with("/private/tmp/")
{
state.fail(
label.to_string(),
"environment.remem_data_dir must prove temporary isolation",
);
}
if let Some(digest) = env.docker_image_digest.as_deref() {
require_non_blank(digest, label, "environment.docker_image_digest", state);
}
if let Some(revision) = env.fixture_revision.as_deref() {
require_non_blank(revision, label, "environment.fixture_revision", state);
}
if let Some(commit) = env.repo_base_commit.as_deref() {
require_non_blank(commit, label, "environment.repo_base_commit", state);
}
}
fn validate_artifact_map<const N: usize>(
artifacts: &std::collections::BTreeMap<String, String>,
required_keys: [&str; N],
label: &str,
state: &mut VerifyState,
) {
for key in required_keys {
require_artifact_key(artifacts, key, label, state);
}
}
fn require_artifact_key(
artifacts: &std::collections::BTreeMap<String, String>,
key: &str,
label: &str,
state: &mut VerifyState,
) {
let Some(raw_path) = artifacts.get(key) else {
state.fail(label.to_string(), format!("artifact key {key} is missing"));
return;
};
let Some(path) = resolve_public_path(state, raw_path, raw_path) else {
return;
};
if !path.is_file() {
state.fail(
label.to_string(),
format!("artifact file for {key} is missing"),
);
return;
}
match state.consume_file(&path, &format!("read artifact file for {key}")) {
Ok(bytes) => {
if key != "remem_db_snapshot" {
if let Some(message) = public_bundle::private_payload_violation(&bytes) {
state.fail(raw_path.clone(), message);
}
}
}
Err(()) => return,
}
state.artifact_files.insert(rel_display(&state.root, &path));
}
fn require_non_blank(value: &str, label: &str, field: &str, state: &mut VerifyState) {
if value.trim().is_empty() {
state.fail(label.to_string(), format!("{field} must not be blank"));
}
}
fn validate_condition_list(
conditions: &[String],
layer: BenchmarkLayer,
label: &str,
field: &str,
state: &mut VerifyState,
) {
let mut seen = BTreeSet::new();
for condition in conditions {
require_non_blank(condition, label, field, state);
if !seen.insert(condition.as_str()) {
state.fail(label.to_string(), format!("{field} must be unique"));
}
if layer == BenchmarkLayer::CodingAgentOutcome
&& !coding::is_public_coding_condition(condition)
{
state.fail(
label.to_string(),
format!("{field} contains unknown coding condition identity {condition}"),
);
}
}
}
fn condition_set(conditions: &[String]) -> BTreeSet<String> {
conditions.iter().cloned().collect()
}
fn read_json<T: DeserializeOwned>(path: &Path, state: &mut VerifyState, label: &str) -> Option<T> {
read_json_artifact(path, state, label).map(|artifact| artifact.value)
}
fn read_json_artifact<T: DeserializeOwned>(
path: &Path,
state: &mut VerifyState,
label: &str,
) -> Option<VerifiedArtifact<T>> {
let display = rel_display(&state.root, path);
let bytes = state.consume_file(path, &format!("read {label}")).ok()?;
let value = match serde_json::from_slice::<Value>(&bytes) {
Ok(value) => value,
Err(err) => {
state.fail(display, format!("parse {label} JSON: {err}"));
return None;
}
};
scan_private_json(&value, path, "$", state);
match serde_json::from_value::<T>(value) {
Ok(parsed) => Some(VerifiedArtifact {
path: display,
sha256: format!("{:x}", Sha256::digest(&bytes)),
value: parsed,
}),
Err(err) => {
state.fail(display, format!("validate {label} schema: {err}"));
None
}
}
}
pub(super) fn collect_manifest_paths(root: &Path) -> Result<Vec<PathBuf>> {
let mut paths = Vec::new();
collect_manifest_paths_recursive(root, &mut paths)
.with_context(|| format!("scan benchmark manifests under {}", root.display()))?;
paths.sort();
Ok(paths)
}
fn collect_manifest_paths_recursive(dir: &Path, paths: &mut Vec<PathBuf>) -> Result<()> {
for entry in fs::read_dir(dir).with_context(|| format!("read directory {}", dir.display()))? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
collect_manifest_paths_recursive(&path, paths)?;
} else if path.extension().is_some_and(|ext| ext == "json")
&& path
.parent()
.and_then(Path::file_name)
.is_some_and(|name| name == "manifests")
{
paths.push(path);
}
}
Ok(())
}
fn resolve_public_path(state: &mut VerifyState, raw: &str, label: &str) -> Option<PathBuf> {
let root = state.root.clone();
scan_private_string(raw, &root.join(label), label, state);
let path = Path::new(raw);
if path.is_absolute() {
state.fail(label.to_string(), "artifact path must be relative");
return None;
}
if raw.trim().is_empty() {
state.fail(label.to_string(), "artifact path must not be blank");
return None;
}
if path.components().any(|component| {
matches!(
component,
Component::ParentDir | Component::RootDir | Component::Prefix(_)
)
}) {
state.fail(
label.to_string(),
"artifact path must stay inside benchmark root",
);
return None;
}
let resolved = root.join(path);
if let (Ok(canonical_root), Ok(canonical_target)) =
(fs::canonicalize(&root), fs::canonicalize(&resolved))
{
if !canonical_target.starts_with(canonical_root) {
state.fail(
label.to_string(),
"artifact target must stay inside benchmark root",
);
return None;
}
}
Some(resolved)
}
fn scan_private_json(value: &Value, path: &Path, pointer: &str, state: &mut VerifyState) {
match value {
Value::String(text) => scan_private_string(text, path, pointer, state),
Value::Array(items) => {
for (index, item) in items.iter().enumerate() {
scan_private_json(item, path, &format!("{pointer}/{index}"), state);
}
}
Value::Object(object) => {
for (key, item) in object {
scan_private_json(item, path, &format!("{pointer}/{key}"), state);
}
}
Value::Bool(_) | Value::Number(_) | Value::Null => {}
}
}
fn scan_private_string(text: &str, path: &Path, pointer: &str, state: &mut VerifyState) {
if let Some(message) = public_bundle::private_string_violation(text) {
state.fail(
rel_display(&state.root, path),
format!("{pointer} {message}"),
);
}
}
fn rel_display(root: &Path, path: &Path) -> String {
path.strip_prefix(root)
.unwrap_or(path)
.to_string_lossy()
.replace('\\', "/")
}
struct VerifyState {
root: PathBuf,
manifests_checked: usize,
reports_checked: usize,
run_artifacts_checked: usize,
artifact_files: BTreeSet<String>,
run_artifact_paths: BTreeSet<String>,
coding_run_keys: BTreeSet<String>,
coding_attempt_ids: BTreeSet<String>,
failures: Vec<BenchVerifyFailure>,
verified_artifacts: VerifiedBenchmarkArtifacts,
consumed_bytes: BTreeMap<String, String>,
consumed_file_bytes: BTreeMap<String, Vec<u8>>,
}
impl VerifyState {
fn new(root: PathBuf) -> Self {
Self {
root,
manifests_checked: 0,
reports_checked: 0,
run_artifacts_checked: 0,
artifact_files: BTreeSet::new(),
run_artifact_paths: BTreeSet::new(),
coding_run_keys: BTreeSet::new(),
coding_attempt_ids: BTreeSet::new(),
failures: Vec::new(),
verified_artifacts: VerifiedBenchmarkArtifacts::default(),
consumed_bytes: BTreeMap::new(),
consumed_file_bytes: BTreeMap::new(),
}
}
fn fail(&mut self, path: String, message: impl Into<String>) {
self.failures.push(BenchVerifyFailure {
path,
message: message.into(),
});
}
fn consume_file(&mut self, path: &Path, operation: &str) -> std::result::Result<Vec<u8>, ()> {
let display = rel_display(&self.root, path);
if let Some(bytes) = self.consumed_file_bytes.get(&display) {
return Ok(bytes.clone());
}
match public_bundle::read_bounded(path) {
Ok(bytes) => {
self.consumed_bytes
.insert(display.clone(), format!("{:x}", Sha256::digest(&bytes)));
self.consumed_file_bytes.insert(display, bytes.clone());
Ok(bytes)
}
Err(error) => {
self.fail(display, format!("{operation}: {error}"));
Err(())
}
}
}
fn finish(mut self) -> BenchVerifyReport {
let authority = super::authority::evaluate(
&self.verified_artifacts,
&self.consumed_bytes,
&self.failures,
);
self.failures.extend(authority.failures);
let passed = self.failures.is_empty();
BenchVerifyReport {
schema_version: 1,
root: self.root.to_string_lossy().into_owned(),
passed,
manifests_checked: self.manifests_checked,
reports_checked: self.reports_checked,
run_artifacts_checked: self.run_artifacts_checked,
artifact_files_checked: self.artifact_files.len(),
failures: self.failures,
authority_verdict: authority.verdict,
verified_artifacts: self.verified_artifacts,
}
}
}