use std::{
collections::BTreeSet,
fs,
io::Read,
path::{Component, Path},
};
#[cfg(unix)]
use std::fs::File;
use sha2::{Digest, Sha256};
use crate::interface::CliError;
pub(crate) const PINNED_CAMPAIGN_AUTHORITY_SHA256: &str =
"be3e017e1ef1bcaa26e4e0e1325925c9c9450de32417c3962ae917e1d7668e59";
const CAMPAIGN_AUTHORITY_SCHEMA: &str = "shepherd.campaign-authority/1";
const CAMPAIGN_POLICY: &str = "v657";
const EXPECTED: [(&str, &str, &str, &str); 7] = [
(
"pi-bootstrap",
"pi-bootstrap",
"pi-bootstrap_good.txt",
"pi-bootstrap_bad.txt",
),
(
"least-authority",
"least-authority",
"v656/least-authority_good.txt",
"v656/least-authority_bad.txt",
),
(
"first-run",
"first-run",
"v656/first-run_good.txt",
"v656/first-run_bad.txt",
),
(
"gate-provenance",
"gate-provenance",
"v656/gate-provenance_good.txt",
"v656/gate-provenance_bad.txt",
),
(
"release-trust",
"release-trust",
"v656/release-trust_good.txt",
"v656/release-trust_bad.txt",
),
(
"release-authority",
"release-authority",
"v656/release-authority_good.txt",
"v656/release-authority_bad.txt",
),
(
"harness-semantics",
"harness-semantics",
"v656/harness-semantics_good.txt",
"v656/harness-semantics_bad.txt",
),
];
const V657_EXPECTED: [(&str, &str, &str, &str); 16] = [
(
"pi-bootstrap",
"pi-bootstrap",
"pi-bootstrap_good.txt",
"pi-bootstrap_bad.txt",
),
(
"least-authority",
"least-authority",
"v656/least-authority_good.txt",
"v656/least-authority_bad.txt",
),
(
"first-run",
"first-run",
"v656/first-run_good.txt",
"v656/first-run_bad.txt",
),
(
"spawn-entry",
"spawn-entry",
"v657/spawn-entry_good.txt",
"v657/spawn-entry_bad.txt",
),
(
"ci-cache",
"ci-cache",
"v657/ci-cache_good.txt",
"v657/ci-cache_bad.txt",
),
(
"gate-provenance",
"gate-provenance",
"v656/gate-provenance_good.txt",
"v656/gate-provenance_bad.txt",
),
(
"release-trust",
"release-trust",
"v656/release-trust_good.txt",
"v656/release-trust_bad.txt",
),
(
"release-authority",
"release-authority",
"v656/release-authority_good.txt",
"v656/release-authority_bad.txt",
),
(
"release-runtime",
"release-runtime",
"v657/release-runtime_good.txt",
"v657/release-runtime_bad.txt",
),
(
"harness-semantics",
"harness-semantics",
"v656/harness-semantics_good.txt",
"v656/harness-semantics_bad.txt",
),
(
"version-install-identity",
"version-install-identity",
"v657/version-install-identity_good.txt",
"v657/version-install-identity_bad.txt",
),
(
"native-authority",
"pending-dispatch",
"v657/pending-dispatch_good.txt",
"v657/pending-dispatch_bad.txt",
),
(
"packed-dogfood",
"packed-dogfood",
"v657/packed-dogfood_good.txt",
"v657/packed-dogfood_bad.txt",
),
(
"host-context-boundary",
"host-context-boundary",
"v657/host-context-boundary_good.txt",
"v657/host-context-boundary_bad.txt",
),
(
"candidate-custody",
"candidate-custody",
"v657/candidate-custody_good.txt",
"v657/candidate-custody_bad.txt",
),
(
"plan-readiness",
"plan-readiness",
"v657/plan-readiness_good.txt",
"v657/plan-readiness_bad.txt",
),
];
const LEGACY: [&str; 14] = [
"reflection_good.txt",
"reflection_bad.txt",
"discovery_good.txt",
"discovery_bad.txt",
"dispatch_good.txt",
"dispatch_bad.txt",
"pi_tool_call_id_good.txt",
"pi_tool_call_id_bad.txt",
"content_good.txt",
"content_bad.txt",
"plugin_distribution_good.txt",
"plugin_distribution_bad.txt",
"cargo_native_distribution_good.txt",
"cargo_native_distribution_bad.txt",
];
#[derive(Clone, Debug, Eq, PartialEq)]
struct Row {
owner: String,
kind: String,
good: String,
bad: String,
}
pub(crate) fn validate_v656(
manifest: &Path,
cases: &Path,
rubrics: &Path,
) -> Result<String, CliError> {
validate_versioned(manifest, cases, rubrics, &EXPECTED, "v656")
}
pub(crate) fn validate_v657(
manifest: &Path,
cases: &Path,
rubrics: &Path,
) -> Result<String, CliError> {
validate_versioned(manifest, cases, rubrics, &V657_EXPECTED, "v657")
}
pub(crate) fn validate_manifest(
manifest: &Path,
cases: &Path,
rubrics: &Path,
) -> Result<String, CliError> {
if manifest.file_name().and_then(|name| name.to_str()) == Some("v657-manifest.tsv") {
validate_v657(manifest, cases, rubrics)
} else {
validate_v656(manifest, cases, rubrics)
}
}
fn validate_versioned(
manifest: &Path,
cases: &Path,
rubrics: &Path,
expected: &[(&str, &str, &str, &str)],
label: &str,
) -> Result<String, CliError> {
let raw =
fs::read(manifest).map_err(|e| err(format!("cannot read {}: {e}", manifest.display())))?;
let rows = parse(&raw)?;
contract_for(&rows, expected, label)?;
for row in &rows {
regular(&cases.join(&row.good), "good sibling", &row.good)?;
regular(&cases.join(&row.bad), "bad sibling", &row.bad)?;
regular(
&rubrics.join(format!("{}.rubric.json", row.kind)),
"rubric",
&row.kind,
)?;
}
Ok(rows
.iter()
.map(|r| format!("{}\t{}\t{}\t{}", r.owner, r.kind, r.good, r.bad))
.collect::<Vec<_>>()
.join("\n")
+ "\n")
}
fn parse(raw: &[u8]) -> Result<Vec<Row>, CliError> {
if !raw.is_empty() && !raw.ends_with(b"\n") {
return Err(err("manifest must end with a newline"));
}
let text = std::str::from_utf8(raw).map_err(|e| err(format!("manifest must be UTF-8: {e}")))?;
let mut rows = Vec::new();
for (index, line) in text.lines().enumerate() {
let number = index + 1;
if line.is_empty() {
continue;
}
if line.starts_with('#') {
if number == 1 && line == "# owner\tkind\tgood\tbad" {
continue;
}
return Err(err(if number == 1 {
"manifest header comment is malformed".into()
} else {
format!("unexpected manifest comment on line {number}")
}));
}
let fields = line.split('\t').collect::<Vec<_>>();
if fields.len() != 4 || fields.iter().any(|field| field.is_empty()) {
return Err(err(format!(
"manifest line {number} must contain exactly four non-empty TSV fields"
)));
}
safe(fields[2], "good input")?;
safe(fields[3], "bad input")?;
if fields[2] == fields[3] {
return Err(err(format!(
"same good/bad input for {}: {}",
fields[0], fields[2]
)));
}
rows.push(Row {
owner: fields[0].into(),
kind: fields[1].into(),
good: fields[2].into(),
bad: fields[3].into(),
});
}
Ok(rows)
}
fn safe(value: &str, label: &str) -> Result<(), CliError> {
let path = Path::new(value);
if value.is_empty()
|| value.contains('\\')
|| path.is_absolute()
|| !path.components().all(|c| matches!(c, Component::Normal(_)))
{
return Err(err(format!(
"{label} must be a contained relative POSIX path: {value}"
)));
}
Ok(())
}
fn contract_for(
rows: &[Row],
expected: &[(&str, &str, &str, &str)],
label: &str,
) -> Result<(), CliError> {
if rows.len() != expected.len() {
let expected_count = match expected.len() {
1 => "one".to_owned(),
7 => "seven".to_owned(),
count => count.to_string(),
};
return Err(err(format!(
"{label} manifest must contain exactly {expected_count} owned partition(s), found {}",
rows.len()
)));
}
let inputs = rows
.iter()
.flat_map(|r| [&r.good, &r.bad])
.map(String::as_str)
.collect::<Vec<_>>();
let unique = inputs.iter().copied().collect::<BTreeSet<_>>();
if inputs.len() != unique.len() {
return Err(err(format!("duplicate eval input in {label} manifest")));
}
let legacy = LEGACY.iter().copied().collect::<BTreeSet<_>>();
let collisions = unique.intersection(&legacy).copied().collect::<Vec<_>>();
if !collisions.is_empty() {
return Err(err(format!(
"v656 input collides with legacy pair: {}",
collisions.join(", ")
)));
}
let expected_rows = expected
.iter()
.map(|(o, k, g, b)| Row {
owner: (*o).into(),
kind: (*k).into(),
good: (*g).into(),
bad: (*b).into(),
})
.collect::<Vec<_>>();
if rows != expected_rows {
return Err(err(format!(
"manifest owner/kind/input partitions do not match the closed {label} contract"
)));
}
Ok(())
}
fn regular(path: &Path, label: &str, value: &str) -> Result<(), CliError> {
let components = path.components().collect::<Vec<_>>();
let mut current = Path::new("").to_path_buf();
for (index, component) in components.iter().enumerate() {
current.push(component.as_os_str());
let metadata =
fs::symlink_metadata(¤t).map_err(|_| err(format!("missing {label}: {value}")))?;
if metadata.file_type().is_symlink() {
return Err(err(format!("{label} traverses a symlink: {value}")));
}
if index + 1 < components.len() && !metadata.file_type().is_dir() {
return Err(err(format!("{label} parent is not a directory: {value}")));
}
if index + 1 == components.len() && !metadata.file_type().is_file() {
return Err(err(format!("{label} is not a regular file: {value}")));
}
}
Ok(())
}
fn err(message: impl Into<String>) -> CliError {
CliError::message(format!("eval manifest: {}", message.into()))
}
pub(crate) fn verify_campaign_authority(
manifest: &Path,
policy: &str,
trusted_shepherd: &Path,
trusted_shepherd_sha256: &str,
) -> Result<String, CliError> {
if policy != CAMPAIGN_POLICY {
return Err(CliError::message("campaign policy is not pinned v657"));
}
let manifest = canonical_regular(manifest, "campaign authority manifest")?;
let manifest_bytes = read_verified(&manifest, "campaign authority manifest")?;
let manifest_sha256 = sha256(&manifest_bytes);
if manifest_sha256 != PINNED_CAMPAIGN_AUTHORITY_SHA256 {
return Err(CliError::message(
"campaign authority manifest does not match the compiled v657 native digest",
));
}
let document: serde_json::Value = serde_json::from_slice(&manifest_bytes)
.map_err(|source| CliError::message(format!("campaign authority JSON: {source}")))?;
let object = document
.as_object()
.ok_or_else(|| CliError::message("campaign authority must be a JSON object"))?;
if object.get("schema").and_then(serde_json::Value::as_str) != Some(CAMPAIGN_AUTHORITY_SCHEMA)
|| object.get("policy").and_then(serde_json::Value::as_str) != Some(CAMPAIGN_POLICY)
{
return Err(CliError::message(
"campaign authority schema or policy is invalid",
));
}
let root = manifest
.parent()
.and_then(Path::parent)
.and_then(Path::parent)
.and_then(Path::parent)
.ok_or_else(|| CliError::message("campaign authority has no repository root"))?;
let runner = authority_executable(&object["runner"], root, "runner")?;
let verifier = authority_executable(&object["verifier"], root, "verifier")?;
let shepherd = canonical_regular(trusted_shepherd, "trusted shepherd binary")?;
if shepherd
!= canonical_regular(
&std::env::current_exe().map_err(|source| {
CliError::message(format!("resolve current shepherd binary: {source}"))
})?,
"current shepherd binary",
)?
{
return Err(CliError::message(
"trusted shepherd binary is not the executing installed binary",
));
}
if !is_digest(trusted_shepherd_sha256)
|| sha256(&read_verified(&shepherd, "trusted shepherd binary")?) != trusted_shepherd_sha256
{
return Err(CliError::message(
"trusted shepherd binary hash does not match opened bytes",
));
}
Ok(serde_json::json!({
"schema": "shepherd.campaign-authority-attestation/1",
"policy": CAMPAIGN_POLICY,
"manifest": {"path": manifest, "sha256": manifest_sha256},
"runner": runner,
"verifier": verifier,
"trusted_shepherd": {"path": shepherd, "sha256": trusted_shepherd_sha256},
})
.to_string())
}
fn authority_executable(
value: &serde_json::Value,
root: &Path,
label: &str,
) -> Result<serde_json::Value, CliError> {
let object = value
.as_object()
.ok_or_else(|| CliError::message(format!("campaign authority {label} is not an object")))?;
let path = object
.get("path")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| CliError::message(format!("campaign authority {label} path is missing")))?;
let digest = object
.get("sha256")
.and_then(serde_json::Value::as_str)
.ok_or_else(|| CliError::message(format!("campaign authority {label} hash is missing")))?;
if !is_digest(digest) || !safe_relative(path) {
return Err(CliError::message(format!(
"campaign authority {label} identity is invalid"
)));
}
let absolute = canonical_regular(&root.join(path), label)?;
let actual = sha256(&read_verified(&absolute, label)?);
if actual != digest {
return Err(CliError::message(format!(
"campaign authority {label} hash drifted"
)));
}
if !is_executable(&absolute) {
return Err(CliError::message(format!(
"campaign authority {label} is not executable"
)));
}
Ok(serde_json::json!({"path": absolute, "sha256": digest}))
}
fn is_executable(path: &Path) -> bool {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
path.metadata()
.is_ok_and(|metadata| metadata.permissions().mode() & 0o111 != 0)
}
#[cfg(not(unix))]
{
path.is_file()
}
}
fn canonical_regular(path: &Path, label: &str) -> Result<std::path::PathBuf, CliError> {
if !path.is_absolute() {
return Err(CliError::message(format!("{label} must be absolute")));
}
reject_components(path, label)?;
let canonical = fs::canonicalize(path)
.map_err(|source| CliError::message(format!("canonicalize {label}: {source}")))?;
if canonical != path || !path.is_file() {
return Err(CliError::message(format!(
"{label} is not a canonical regular file"
)));
}
Ok(canonical)
}
fn reject_components(path: &Path, label: &str) -> Result<(), CliError> {
let mut current = std::path::PathBuf::new();
for component in path.components() {
current.push(component.as_os_str());
if matches!(component, Component::Prefix(_) | Component::RootDir) {
continue;
}
let metadata = fs::symlink_metadata(¤t)
.map_err(|source| CliError::message(format!("inspect {label}: {source}")))?;
if metadata.file_type().is_symlink() {
return Err(CliError::message(format!(
"{label} has a symlinked component"
)));
}
}
Ok(())
}
fn read_verified(path: &Path, label: &str) -> Result<Vec<u8>, CliError> {
#[cfg(unix)]
let mut file = {
use rustix::fs::{Mode, OFlags, open};
let fd = open(
path,
OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(|source| CliError::message(format!("open {label}: {source}")))?;
File::from(fd)
};
#[cfg(not(unix))]
let mut file = std::fs::OpenOptions::new()
.read(true)
.open(path)
.map_err(|source| CliError::message(format!("open {label}: {source}")))?;
let before = file
.metadata()
.map_err(|source| CliError::message(format!("inspect {label}: {source}")))?;
const MAX_TRUSTED_BYTES: u64 = 64 * 1024 * 1024;
if !before.is_file() || before.len() > MAX_TRUSTED_BYTES {
return Err(CliError::message(format!(
"{label} exceeds {MAX_TRUSTED_BYTES} bytes or is not regular"
)));
}
let mut bytes = Vec::new();
file.read_to_end(&mut bytes)
.map_err(|source| CliError::message(format!("read {label}: {source}")))?;
let after = file
.metadata()
.map_err(|source| CliError::message(format!("inspect {label}: {source}")))?;
if before.len() != after.len() || before.modified().ok() != after.modified().ok() {
return Err(CliError::message(format!(
"{label} changed while it was read"
)));
}
Ok(bytes)
}
fn sha256(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
fn is_digest(value: &str) -> bool {
value.len() == 64
&& value
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
}
fn safe_relative(value: &str) -> bool {
!value.is_empty()
&& !value.starts_with('/')
&& !value.contains('\\')
&& value
.split('/')
.all(|part| !part.is_empty() && part != "." && part != "..")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn syntax_rejects_traversal_and_missing_newline() {
assert!(
parse(b"o\tk\t../g\tb\n")
.unwrap_err()
.message_text()
.is_some_and(|message| message.contains("contained relative POSIX path"))
);
assert!(
parse(b"o\tk\tg\tb")
.unwrap_err()
.message_text()
.is_some_and(|message| message.contains("must end with a newline"))
);
}
#[test]
fn contract_is_exactly_seven_canonical_rows() {
let rows = EXPECTED
.iter()
.map(|(o, k, g, b)| Row {
owner: (*o).into(),
kind: (*k).into(),
good: (*g).into(),
bad: (*b).into(),
})
.collect::<Vec<_>>();
contract_for(&rows, &EXPECTED, "v656").unwrap();
assert!(contract_for(&rows[..6], &EXPECTED, "v656").is_err());
}
#[test]
fn contract_accepts_only_the_v657_version_identity_partition() {
let rows = V657_EXPECTED
.iter()
.map(|(o, k, g, b)| Row {
owner: (*o).into(),
kind: (*k).into(),
good: (*g).into(),
bad: (*b).into(),
})
.collect::<Vec<_>>();
contract_for(&rows, &V657_EXPECTED, "v657").unwrap();
assert!(contract_for(&[], &V657_EXPECTED, "v657").is_err());
assert!(contract_for(&rows[..rows.len() - 1], &V657_EXPECTED, "v657").is_err());
let mut wrong_order = rows.clone();
wrong_order.swap(2, 3);
assert!(contract_for(&wrong_order, &V657_EXPECTED, "v657").is_err());
let mut wrong_kind = rows;
wrong_kind
.iter_mut()
.find(|row| row.owner == "packed-dogfood")
.unwrap()
.kind = "pending-dispatch".into();
assert!(contract_for(&wrong_kind, &V657_EXPECTED, "v657").is_err());
}
#[test]
fn v657_manifest_requires_the_native_candidate_custody_pair() {
let rows = parse(include_bytes!(
"../../../../services/eval/evals/v657-manifest.tsv"
))
.unwrap();
contract_for(&rows, &V657_EXPECTED, "v657").unwrap();
let candidate = rows
.iter()
.find(|row| row.owner == "candidate-custody")
.unwrap();
assert_eq!(candidate.owner, "candidate-custody");
assert_eq!(candidate.kind, "candidate-custody");
assert_eq!(candidate.good, "v657/candidate-custody_good.txt");
assert_eq!(candidate.bad, "v657/candidate-custody_bad.txt");
}
#[test]
fn v657_manifest_requires_native_plan_readiness_as_an_exact_owned_pair() {
let rows = parse(include_bytes!(
"../../../../services/eval/evals/v657-manifest.tsv"
))
.unwrap();
contract_for(&rows, &V657_EXPECTED, "v657").unwrap();
let readiness = rows
.iter()
.find(|row| row.owner == "plan-readiness")
.unwrap();
assert_eq!(readiness.kind, "plan-readiness");
assert_eq!(readiness.good, "v657/plan-readiness_good.txt");
assert_eq!(readiness.bad, "v657/plan-readiness_bad.txt");
let mut missing = rows.clone();
missing.retain(|row| row.owner != "plan-readiness");
assert!(contract_for(&missing, &V657_EXPECTED, "v657").is_err());
}
#[test]
fn v657_release_runtime_pair_is_an_exact_required_partition() {
assert_eq!(V657_EXPECTED.len(), 16);
let expected = (
"release-runtime",
"release-runtime",
"v657/release-runtime_good.txt",
"v657/release-runtime_bad.txt",
);
assert_eq!(V657_EXPECTED[8], expected);
let rows = V657_EXPECTED
.iter()
.map(|(owner, kind, good, bad)| Row {
owner: (*owner).into(),
kind: (*kind).into(),
good: (*good).into(),
bad: (*bad).into(),
})
.collect::<Vec<_>>();
contract_for(&rows, &V657_EXPECTED, "v657").unwrap();
let mut missing = rows.clone();
missing.remove(8);
assert!(contract_for(&missing, &V657_EXPECTED, "v657").is_err());
let mut substituted = rows;
substituted[8].bad = "v657/ci-cache_bad.txt".into();
assert!(contract_for(&substituted, &V657_EXPECTED, "v657").is_err());
}
}