use std::{
collections::{BTreeMap, BTreeSet},
fs,
io::{Read, Write},
path::{Path, PathBuf},
};
use clap::{Args, Subcommand};
use serde_json::Value;
use sha2::{Digest, Sha256};
use shepherd::digest::{format_digest, sha256_hex};
use crate::{
cmd::dispatch::{ReadSubject, read_regular_refusing_links},
interface::{CliError, CliGlobals},
};
const MAX_FILE_BYTES: u64 = 64 * 1024;
const MAX_BUNDLE_BYTES: u64 = 256 * 1024;
const MAX_BUNDLE_FILES: usize = 128;
const MAX_RENDERED_BYTES: u64 = 256 * 1024;
const MAX_INPUT_BYTES: u64 = 2 * 1024 * 1024;
const MAX_RAW_LINE_BYTES: usize = 512 * 1024;
const MAX_RAW_AGGREGATE_BYTES: u64 = 4 * 1024 * 1024;
const CAMPAIGN_REPETITIONS: u64 = 5;
const CAMPAIGN_GENERATIONS: u64 = 15;
const CAMPAIGN_JUDGES: u64 = 15;
const CAMPAIGN_AUDITORS: u64 = 1;
const CAMPAIGN_MIN_MARGIN: f64 = 15.0;
const CAMPAIGN_MAX_VARIANCE: f64 = 50.0;
const SOURCE_MAP: [(&str, &str); 4] = [
("content/skills/spawn/SKILL.md", "SKILL.md"),
("content/skills/start/SKILL.md", "references/start.md"),
("content/skills/shepherd/SKILL.md", "references/shepherd.md"),
("content/roles/shepherd.md", "references/shepherd-role.md"),
];
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
pub struct EvalCampaignCmd {
#[command(subcommand)]
action: EvalCampaignAction,
}
#[derive(
Clone,
Debug,
Eq,
Hash,
Ord,
PartialEq,
PartialOrd,
Subcommand,
serde::Deserialize,
serde::Serialize,
)]
enum EvalCampaignAction {
Bundle(EvalBundleArgs),
Assemble(EvalAssembleArgs),
VerifyBundle(EvalVerifyBundleArgs),
Canonicalize(EvalCanonicalizeArgs),
Hash(EvalHashArgs),
InspectProvider(EvalInspectProviderArgs),
InspectAuth(EvalInspectAuthArgs),
Verify(EvalVerifyArgs),
}
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
struct EvalBundleArgs {
skill_dir: PathBuf,
#[arg(long)]
render: Option<PathBuf>,
}
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
struct EvalAssembleArgs {
#[arg(long)]
root: PathBuf,
#[arg(long)]
bundle: PathBuf,
#[arg(long)]
manifest: PathBuf,
}
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
struct EvalVerifyBundleArgs {
#[arg(long)]
root: PathBuf,
#[arg(long)]
bundle: PathBuf,
#[arg(long)]
manifest: PathBuf,
}
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
struct EvalCanonicalizeArgs {
source: PathBuf,
destination: PathBuf,
}
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
struct EvalHashArgs {
path: PathBuf,
}
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
struct EvalInspectProviderArgs {
path: PathBuf,
#[arg(default_value = "deterministic")]
execution_mode: String,
}
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
struct EvalInspectAuthArgs {
path: PathBuf,
session: String,
}
#[derive(
Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Args, serde::Deserialize, serde::Serialize,
)]
struct EvalVerifyArgs {
execution_mode: String,
output: PathBuf,
skill_dir: PathBuf,
rubric: PathBuf,
trusted_provider: PathBuf,
trusted_auth: PathBuf,
#[arg(long)]
trusted_shepherd: Option<PathBuf>,
#[arg(long)]
trusted_shepherd_sha256: Option<String>,
}
#[derive(Clone, Debug)]
struct BundleFile {
path: String,
mode: u32,
bytes: Vec<u8>,
}
impl EvalCampaignCmd {
pub(crate) fn run(self, _globals: CliGlobals) -> Result<(), CliError> {
match self.action {
EvalCampaignAction::Bundle(args) => {
let files = collect_bundle(&args.skill_dir)?;
if let Some(render) = args.render {
write_new_file(&render, &render_bundle(&files), 0o600)?;
}
println!(
"{}",
serde_json::to_string(&bundle_metadata(&files))
.map_err(|error| CliError::message(error.to_string()))?
);
Ok(())
}
EvalCampaignAction::Assemble(args) => assemble_bundle(&args),
EvalCampaignAction::VerifyBundle(args) => verify_bundle(&args),
EvalCampaignAction::Canonicalize(args) => {
let bytes = canonical_json(&read_bounded(&args.source, MAX_INPUT_BYTES)?)?;
write_new_file(&args.destination, &bytes, 0o600)
}
EvalCampaignAction::Hash(args) => {
let value = parse_json(&read_bounded(&args.path, MAX_INPUT_BYTES)?)?;
println!("{}", sha256_hex(&canonical_json_value(&value)));
Ok(())
}
EvalCampaignAction::InspectProvider(args) => {
println!(
"{}",
serde_json::to_string(&inspect_provider(&args.path, &args.execution_mode)?)
.map_err(|e| CliError::message(e.to_string()))?
);
Ok(())
}
EvalCampaignAction::InspectAuth(args) => {
println!(
"{}",
serde_json::to_string(&inspect_auth(&args.path, &args.session)?)
.map_err(|e| CliError::message(e.to_string()))?
);
Ok(())
}
EvalCampaignAction::Verify(args) => verify_campaign(&args),
}
}
}
fn err(message: impl Into<String>) -> CliError {
CliError::message(format!("eval campaign: {}", message.into()))
}
fn canonical(path: &Path, label: &str) -> Result<PathBuf, CliError> {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map_err(|e| err(e.to_string()))?
.join(path)
};
let resolved =
fs::canonicalize(&absolute).map_err(|e| err(format!("canonicalize {label}: {e}")))?;
if resolved != absolute {
return Err(err(format!("{label} has a symlinked component")));
}
Ok(resolved)
}
fn read_bounded(path: &Path, limit: u64) -> Result<Vec<u8>, CliError> {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map_err(|e| err(e.to_string()))?
.join(path)
};
let bytes = read_regular_refusing_links(ReadSubject::File, &absolute, limit)?;
Ok(bytes)
}
fn identity(path: &Path, label: &str) -> Result<Value, CliError> {
let metadata =
fs::symlink_metadata(path).map_err(|e| err(format!("cannot inspect {label}: {e}")))?;
if metadata.file_type().is_symlink() {
return Err(err(format!("{label} is a symlink")));
}
Ok(metadata_identity(&metadata))
}
fn stream_hash(path: &Path, limit: u64, label: &str) -> Result<(String, Value), CliError> {
let before = identity(path, label)?;
let metadata = fs::metadata(path).map_err(|e| err(format!("cannot inspect {label}: {e}")))?;
if !metadata.is_file() {
return Err(err(format!("{label} is not a regular file")));
}
if metadata.len() > limit {
return Err(err(format!("{label} exceeds {limit} bytes")));
}
#[cfg(unix)]
let mut file = {
use rustix::fs::{Mode, OFlags, open};
let descriptor = open(
path,
OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)
.map_err(|e| err(format!("read {label}: {e}")))?;
fs::File::from(descriptor)
};
#[cfg(not(unix))]
let mut file = fs::File::open(path).map_err(|e| err(format!("read {label}: {e}")))?;
let mut digest = Sha256::new();
let mut bytes = 0_u64;
let mut buffer = [0_u8; 128 * 1024];
loop {
let count = file
.read(&mut buffer)
.map_err(|e| err(format!("read {label}: {e}")))?;
if count == 0 {
break;
}
bytes = bytes.saturating_add(count as u64);
if bytes > limit {
return Err(err(format!("{label} exceeds {limit} bytes")));
}
digest.update(&buffer[..count]);
}
let after = identity(path, label)?;
if before != after {
return Err(err(format!("{label} identity changed during read")));
}
Ok((format_digest(digest.finalize()), after))
}
fn read_raw_file(path: &Path, label: &str) -> Result<Vec<u8>, CliError> {
let before = identity(path, label)?;
let metadata = fs::metadata(path).map_err(|e| err(format!("cannot inspect {label}: {e}")))?;
if !metadata.is_file() || metadata_mode(&metadata) != 0o600 {
return Err(err(format!("{label} must be a mode-0600 regular file")));
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if metadata.nlink() != 1 {
return Err(err(format!("{label} has hard links")));
}
}
let bytes = read_regular_refusing_links(ReadSubject::File, path, MAX_RAW_AGGREGATE_BYTES)?;
if bytes.len() as u64 > MAX_RAW_AGGREGATE_BYTES {
return Err(err(format!(
"{label} exceeds {MAX_RAW_AGGREGATE_BYTES} bytes"
)));
}
let after = identity(path, label)?;
if before != after {
return Err(err(format!("{label} identity changed during read")));
}
Ok(bytes)
}
fn canonical_child(path: &str, parent: &Path, field: &str) -> Result<String, CliError> {
let name = direct_child(path, parent, field)?;
let expected = parent.join(&name);
if path != expected.to_string_lossy() {
return Err(err(format!("{field} is not a canonical path")));
}
Ok(name)
}
fn write_new_file(path: &Path, bytes: &[u8], mode: u32) -> Result<(), CliError> {
#[cfg(not(unix))]
let _ = mode;
let parent = path.parent().ok_or_else(|| err("output has no parent"))?;
let canonical_parent = canonical(parent, "output parent")?;
if canonical_parent != parent || !canonical_parent.is_dir() {
return Err(err("output parent must be an existing canonical directory"));
}
let mut options = fs::OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(mode).custom_flags(libc::O_NOFOLLOW);
}
let mut file = options
.open(path)
.map_err(|e| err(format!("create {}: {e}", path.display())))?;
file.write_all(bytes)
.map_err(|e| err(format!("write {}: {e}", path.display())))?;
file.sync_all()
.map_err(|e| err(format!("sync {}: {e}", path.display())))?;
#[cfg(unix)]
file.set_permissions(fs::Permissions::from_mode(mode))
.map_err(|e| err(format!("chmod {}: {e}", path.display())))?;
file.sync_all()
.map_err(|e| err(format!("sync {}: {e}", path.display())))?;
Ok(())
}
fn set_mode(path: &Path, mode: u32) -> Result<(), CliError> {
#[cfg(unix)]
{
fs::set_permissions(path, fs::Permissions::from_mode(mode))
.map_err(|e| err(e.to_string()))?;
}
let _ = (path, mode);
Ok(())
}
fn metadata_mode(metadata: &fs::Metadata) -> u32 {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
metadata.permissions().mode() & 0o777
}
#[cfg(not(unix))]
{
if metadata.is_file() { 0o644 } else { 0o755 }
}
}
fn validate_file(path: &Path, relative: &str, expected_mode: u32) -> Result<Vec<u8>, CliError> {
let metadata =
fs::symlink_metadata(path).map_err(|e| err(format!("cannot inspect {relative}: {e}")))?;
if metadata.file_type().is_symlink() {
return Err(err(format!("symlink is not allowed: {relative}")));
}
if !metadata.is_file() {
return Err(err(format!("not a regular file: {relative}")));
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if metadata.nlink() != 1 {
return Err(err(format!("hard link is not allowed: {relative}")));
}
}
let mode = metadata_mode(&metadata);
if mode != expected_mode {
return Err(err(format!(
"{relative} has mode {mode:04o}, expected {expected_mode:04o}"
)));
}
if metadata.len() > MAX_FILE_BYTES {
return Err(err(format!("{relative} exceeds {MAX_FILE_BYTES} bytes")));
}
let before = metadata_identity(&metadata);
let bytes = read_regular_refusing_links(ReadSubject::File, path, MAX_FILE_BYTES)?;
let after_metadata = fs::symlink_metadata(path)
.map_err(|e| err(format!("cannot inspect {relative} after read: {e}")))?;
if metadata_identity(&after_metadata) != before {
return Err(err(format!("{relative} identity changed during read")));
}
if bytes.len() as u64 > MAX_FILE_BYTES {
return Err(err(format!("{relative} exceeds {MAX_FILE_BYTES} bytes")));
}
std::str::from_utf8(&bytes).map_err(|_| err(format!("{relative} is not UTF-8")))?;
Ok(bytes)
}
fn collect_bundle(root: &Path) -> Result<Vec<BundleFile>, CliError> {
let root = canonical(root, "skill directory")?;
let metadata = fs::metadata(&root).map_err(|e| err(e.to_string()))?;
if !metadata.is_dir() {
return Err(err("skill path is not a directory"));
}
if metadata_mode(&metadata) & 0o022 != 0 {
return Err(err("bundle directory is writable by group or other"));
}
let root_identity =
metadata_identity(&fs::symlink_metadata(&root).map_err(|e| err(e.to_string()))?);
let mut names = BTreeMap::<String, PathBuf>::new();
for entry in fs::read_dir(&root).map_err(|e| err(e.to_string()))? {
let entry = entry.map_err(|e| err(e.to_string()))?;
names.insert(
entry.file_name().to_string_lossy().into_owned(),
entry.path(),
);
}
let allowed = ["SKILL.md", "assets", "references", "scripts"];
for name in names.keys() {
if !allowed.contains(&name.as_str()) {
return Err(err(format!("unsupported top-level entry: {name}")));
}
}
let mut files = Vec::new();
let skill = names
.get("SKILL.md")
.ok_or_else(|| err("missing SKILL.md"))?;
files.push(BundleFile {
path: "SKILL.md".into(),
mode: 0o644,
bytes: validate_file(skill, "SKILL.md", 0o644)?,
});
for category in ["assets", "references", "scripts"] {
let Some(directory) = names.get(category) else {
continue;
};
let metadata = fs::symlink_metadata(directory).map_err(|e| err(e.to_string()))?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(err(format!("not a regular directory: {category}")));
}
if metadata_mode(&metadata) & 0o022 != 0 {
return Err(err(format!("{category} is writable by group or other")));
}
let category_identity = metadata_identity(&metadata);
let expected = if category == "scripts" { 0o755 } else { 0o644 };
let mut entries = fs::read_dir(directory)
.map_err(|e| err(e.to_string()))?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| err(e.to_string()))?;
entries.sort_by_key(|entry| entry.file_name());
for entry in entries {
let name = entry.file_name().to_string_lossy().into_owned();
let relative = format!("{category}/{name}");
let metadata = fs::symlink_metadata(entry.path()).map_err(|e| err(e.to_string()))?;
if metadata.is_dir() {
return Err(err(format!("not a regular file: {relative}")));
}
if files.len() + 1 > MAX_BUNDLE_FILES {
return Err(err(format!(
"bundle contains more than {MAX_BUNDLE_FILES} files"
)));
}
let bytes = validate_file(&entry.path(), &relative, expected)?;
files.push(BundleFile {
path: relative,
mode: expected,
bytes,
});
}
if metadata_identity(&fs::symlink_metadata(directory).map_err(|e| err(e.to_string()))?)
!= category_identity
{
return Err(err(format!(
"{category} identity changed during bundle read"
)));
}
}
files.sort_by(|a, b| a.path.cmp(&b.path));
let total: u64 = files.iter().map(|item| item.bytes.len() as u64).sum();
if total > MAX_BUNDLE_BYTES {
return Err(err(format!("bundle exceeds {MAX_BUNDLE_BYTES} bytes")));
}
if rendered_bytes(&files) > MAX_RENDERED_BYTES {
return Err(err(format!(
"rendered bundle exceeds {MAX_RENDERED_BYTES} bytes"
)));
}
if metadata_identity(&fs::symlink_metadata(&root).map_err(|e| err(e.to_string()))?)
!= root_identity
{
return Err(err("bundle directory identity changed during bundle read"));
}
Ok(files)
}
fn bundle_digest(files: &[BundleFile]) -> String {
let mut bytes = Vec::new();
for item in files {
bytes.extend(item.path.as_bytes());
bytes.push(0);
bytes.extend(format!("{:04o}", item.mode).as_bytes());
bytes.push(0);
bytes.extend(&item.bytes);
bytes.push(0);
}
sha256_hex(&bytes)
}
fn rendered_bytes(files: &[BundleFile]) -> u64 {
files
.iter()
.map(|item| {
format!("=== {} ({:04o}) ===\n", item.path, item.mode).len() as u64
+ item.bytes.len() as u64
+ 1
})
.sum()
}
fn render_bundle(files: &[BundleFile]) -> Vec<u8> {
let mut output = Vec::new();
for item in files {
output.extend(format!("=== {} ({:04o}) ===\n", item.path, item.mode).as_bytes());
output.extend(&item.bytes);
if !item.bytes.ends_with(b"\n") {
output.push(b'\n');
}
}
output
}
fn bundle_metadata(files: &[BundleFile]) -> Value {
serde_json::json!({"schema":"shepherd.skill-bundle/1","sha256":bundle_digest(files),"bytes":files.iter().map(|item|item.bytes.len()).sum::<usize>(),"rendered_bytes":rendered_bytes(files),"files":files.iter().map(|item|serde_json::json!({"path":item.path,"mode":item.mode,"bytes":item.bytes.len()})).collect::<Vec<_>>()})
}
fn canonical_json_value(value: &Value) -> Vec<u8> {
serde_json::to_vec(value).expect("JSON values are serializable")
}
fn canonical_json(bytes: &[u8]) -> Result<Vec<u8>, CliError> {
Ok(canonical_json_value(&parse_json(bytes)?))
}
fn parse_json(bytes: &[u8]) -> Result<Value, CliError> {
serde_json::from_slice(bytes).map_err(|e| err(format!("invalid JSON: {e}")))
}
fn direct_child(path: &str, raw_dir: &Path, field: &str) -> Result<String, CliError> {
let value = PathBuf::from(path);
let canonical_raw = canonical(raw_dir, "raw campaign directory")?;
let absolute = canonical(&value, field)?;
if absolute.parent() != Some(canonical_raw.as_path()) {
return Err(err(format!("{field} is not a raw directory child")));
}
absolute
.file_name()
.and_then(|name| name.to_str())
.map(ToOwned::to_owned)
.ok_or_else(|| err(format!("{field} has no valid basename")))
}
fn terminal_record(bytes: &[u8], path: &str) -> Result<(Value, String), CliError> {
if bytes.len() as u64 > MAX_RAW_AGGREGATE_BYTES {
return Err(err(format!(
"{path} exceeds {MAX_RAW_AGGREGATE_BYTES} bytes"
)));
}
let body = bytes.strip_suffix(b"\n").unwrap_or(bytes);
let lines = body.split(|byte| *byte == b'\n').collect::<Vec<_>>();
if lines.is_empty() || lines.len() > 16_384 {
return Err(err(format!("{path} has an invalid line count")));
}
if lines
.iter()
.any(|line| line.is_empty() || line.len() > MAX_RAW_LINE_BYTES)
{
return Err(err(format!(
"{path} contains an empty or oversized JSONL line"
)));
}
let records = lines
.iter()
.map(|line| parse_json(line))
.collect::<Result<Vec<_>, _>>()?;
let successes = records
.iter()
.filter(|record| {
record.get("type").and_then(Value::as_str) == Some("result")
&& record.get("subtype").and_then(Value::as_str) == Some("success")
&& record.get("is_error") == Some(&Value::Bool(false))
&& record
.get("result")
.and_then(Value::as_str)
.is_some_and(|value| !value.is_empty())
})
.collect::<Vec<_>>();
if successes.len() != 1 || records.last() != Some(successes[0]) {
return Err(err(format!(
"{path} must contain exactly one terminal successful result"
)));
}
let session = successes[0]
.get("session_id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.ok_or_else(|| err(format!("{path} session_id is missing")))?;
Ok((successes[0].clone(), session.to_owned()))
}
fn json_hash(value: &Value, field: &str) -> Result<String, CliError> {
let value = value
.get(field)
.and_then(Value::as_str)
.ok_or_else(|| err(format!("{field} is missing")))?;
if value.len() != 64
|| !value
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
{
return Err(err(format!("{field} is not a SHA-256 digest")));
}
Ok(value.to_owned())
}
fn verify_arm_stats(arm: &Value) -> Result<(), CliError> {
let results = arm
.get("results")
.and_then(Value::as_array)
.ok_or_else(|| err("campaign arm results are missing"))?;
let scores = results
.iter()
.map(|result| {
result
.get("overall")
.and_then(Value::as_f64)
.ok_or_else(|| err("campaign score is missing"))
})
.collect::<Result<Vec<_>, _>>()?;
if scores.len() != 5 {
return Err(err("campaign arm must contain five scores"));
}
if !scores.iter().all(|score| score.is_finite()) {
return Err(err("campaign scores must be finite"));
}
let mut sorted = scores.clone();
sorted.sort_by(|left, right| left.partial_cmp(right).unwrap());
let mean = sorted.iter().sum::<f64>() / 5.0;
let variance = sorted
.iter()
.map(|score| (score - mean) * (score - mean))
.sum::<f64>()
/ 5.0;
let stats = arm
.get("stats")
.ok_or_else(|| err("campaign arm statistics are missing"))?;
let pass_rate = results
.iter()
.filter(|result| result.get("passed") == Some(&Value::Bool(true)))
.count() as f64
* 100.0
/ CAMPAIGN_REPETITIONS as f64;
if stats.get("min").and_then(Value::as_f64) != Some(sorted[0])
|| stats.get("median").and_then(Value::as_f64) != Some(sorted[2])
|| stats.get("max").and_then(Value::as_f64) != Some(sorted[4])
|| stats.get("pass_rate").and_then(Value::as_f64) != Some(pass_rate)
|| stats.get("variance").and_then(Value::as_f64) != Some(variance)
{
return Err(err("campaign arm statistics do not reproduce"));
}
Ok(())
}
fn expected_verdict(
rubric: &Value,
judge_response: &[u8],
kind: &str,
model: &str,
) -> Result<Value, CliError> {
let response = parse_json(judge_response)?;
let response_object = response
.as_object()
.ok_or_else(|| err("judge response is not an object"))?;
let scores = response_object
.get("scores")
.and_then(Value::as_object)
.ok_or_else(|| err("judge response scores are missing"))?;
let dimensions = rubric
.get("dimensions")
.and_then(Value::as_array)
.ok_or_else(|| err("rubric dimensions are missing"))?;
let scale = rubric
.get("scale")
.and_then(Value::as_u64)
.ok_or_else(|| err("rubric scale is missing"))?;
if scale == 0 {
return Err(err("rubric scale must be positive"));
}
let threshold = rubric
.get("threshold")
.and_then(Value::as_u64)
.ok_or_else(|| err("rubric threshold is missing"))?;
let mut normalized = serde_json::Map::new();
let mut weighted = 0_u64;
let mut total_weight = 0_u64;
let mut dimension_keys = BTreeSet::new();
for dimension in dimensions {
let key = dimension
.get("key")
.and_then(Value::as_str)
.filter(|key| !key.is_empty())
.ok_or_else(|| err("rubric dimension key is missing"))?;
if !dimension_keys.insert(key.to_owned()) {
return Err(err(format!("rubric dimension is duplicated: {key}")));
}
let weight = dimension
.get("weight")
.and_then(Value::as_u64)
.ok_or_else(|| err(format!("rubric dimension weight is invalid: {key}")))?;
let score = scores
.get(key)
.and_then(Value::as_u64)
.filter(|score| *score >= 1 && *score <= scale)
.ok_or_else(|| err(format!("judge score is invalid or out of range: {key}")))?;
normalized.insert(key.to_owned(), Value::from(score));
weighted = weighted
.checked_add(
score
.checked_mul(weight)
.ok_or_else(|| err("judge score weight overflow"))?,
)
.ok_or_else(|| err("judge weighted score overflow"))?;
total_weight = total_weight
.checked_add(weight)
.ok_or_else(|| err("judge total weight overflow"))?;
}
if total_weight == 0
|| scores.keys().collect::<BTreeSet<_>>() != dimension_keys.iter().collect::<BTreeSet<_>>()
{
return Err(err("judge response has unexpected dimensions"));
}
let denominator = scale
.checked_mul(total_weight)
.ok_or_else(|| err("judge score denominator overflow"))?;
let overall = (weighted
.checked_mul(200)
.ok_or_else(|| err("judge score rounding overflow"))?
.checked_add(denominator)
.ok_or_else(|| err("judge score rounding overflow"))?)
/ (2 * denominator);
Ok(serde_json::json!({
"kind": kind,
"model": model,
"overall": overall,
"threshold": threshold,
"passed": overall >= threshold,
"scale": scale,
"scores": normalized,
"rationale": response_object.get("rationale").cloned().unwrap_or_else(|| Value::String(String::new()))
}))
}
fn assemble_bundle(args: &EvalAssembleArgs) -> Result<(), CliError> {
let root = canonical(&args.root, "root")?;
let bundle = if args.bundle.is_absolute() {
args.bundle.clone()
} else {
std::env::current_dir()
.map_err(|e| err(e.to_string()))?
.join(&args.bundle)
};
let manifest = if args.manifest.is_absolute() {
args.manifest.clone()
} else {
std::env::current_dir()
.map_err(|e| err(e.to_string()))?
.join(&args.manifest)
};
for (path, label) in [(&bundle, "bundle output"), (&manifest, "manifest output")] {
let parent = path
.parent()
.ok_or_else(|| err(format!("{label} has no parent")))?;
if canonical(parent, label)? != parent || !parent.is_dir() {
return Err(err(format!(
"{label} parent must be an existing canonical directory"
)));
}
}
if bundle.exists() || manifest.exists() {
return Err(err("bundle or manifest output already exists"));
}
let mut files = Vec::new();
for (source, target) in SOURCE_MAP {
files.push((
source,
target,
read_regular_refusing_links(ReadSubject::File, &root.join(source), MAX_FILE_BYTES)?,
));
}
fs::create_dir(&bundle).map_err(|e| err(format!("create bundle: {e}")))?;
set_mode(&bundle, 0o700)?;
fs::create_dir(bundle.join("references")).map_err(|e| err(e.to_string()))?;
set_mode(&bundle.join("references"), 0o700)?;
let mut manifest_files = Vec::new();
let mut bundle_files = Vec::new();
for (source, target, bytes) in files {
let destination = bundle.join(target);
write_new_file(&destination, &bytes, 0o644)?;
manifest_files.push(serde_json::json!({"source":source,"bundle_path":target,"sha256":sha256_hex(&bytes),"bytes":bytes.len(),"mode":0o644}));
bundle_files.push(BundleFile {
path: target.into(),
mode: 0o644,
bytes,
});
}
bundle_files.sort_by(|left, right| left.path.cmp(&right.path));
let doc = serde_json::json!({"schema":"shepherd.spawn-skill-source-custody/1","root":root,"bundle":bundle,"bundle_sha256":bundle_digest(&bundle_files),"files":manifest_files});
write_new_file(&manifest, &canonical_json_value(&doc), 0o600)
}
fn verify_bundle(args: &EvalVerifyBundleArgs) -> Result<(), CliError> {
let root = canonical(&args.root, "root")?;
let bundle = canonical(&args.bundle, "bundle")?;
let manifest = canonical(&args.manifest, "manifest")?;
let document = parse_json(&read_bounded(&manifest, MAX_INPUT_BYTES)?)?;
if document.get("schema").and_then(Value::as_str)
!= Some("shepherd.spawn-skill-source-custody/1")
{
return Err(err("source-custody manifest schema is invalid"));
}
if document.get("root") != Some(&Value::String(root.to_string_lossy().into_owned()))
|| document.get("bundle") != Some(&Value::String(bundle.to_string_lossy().into_owned()))
{
return Err(err(
"source-custody manifest selected a different root or bundle",
));
}
let rows = document
.get("files")
.and_then(Value::as_array)
.ok_or_else(|| err("source-custody manifest file set is incomplete"))?;
if rows.len() != SOURCE_MAP.len() {
return Err(err(
"source-custody manifest must contain exactly four files",
));
}
for ((expected_source, expected_target), row) in SOURCE_MAP.iter().zip(rows) {
let object = row
.as_object()
.ok_or_else(|| err("source-custody row is not an object"))?;
let keys = object
.keys()
.map(String::as_str)
.collect::<std::collections::BTreeSet<_>>();
let expected_keys = ["bytes", "bundle_path", "mode", "sha256", "source"]
.into_iter()
.collect::<std::collections::BTreeSet<_>>();
if keys != expected_keys {
return Err(err("source-custody row fields are not exact"));
}
if row.get("source").and_then(Value::as_str) != Some(*expected_source)
|| row.get("bundle_path").and_then(Value::as_str) != Some(*expected_target)
|| row.get("mode").and_then(Value::as_u64) != Some(0o644)
{
return Err(err("source-custody SOURCE_MAP was substituted"));
}
let source_path = root.join(expected_source);
let bundle_path = bundle.join(expected_target);
let authored =
read_regular_refusing_links(ReadSubject::File, &source_path, MAX_FILE_BYTES)?;
let copied = read_regular_refusing_links(ReadSubject::File, &bundle_path, MAX_FILE_BYTES)?;
if row.get("bytes").and_then(Value::as_u64) != Some(authored.len() as u64)
|| row.get("sha256").and_then(Value::as_str) != Some(sha256_hex(&authored).as_str())
{
return Err(err(
"source-custody row hash or byte count does not reproduce",
));
}
if authored != copied {
return Err(err(format!(
"bundle content does not match authored source: {expected_target}"
)));
}
}
let files = collect_bundle(&bundle)?;
if document.get("bundle_sha256").and_then(Value::as_str) != Some(bundle_digest(&files).as_str())
{
return Err(err(
"source-custody manifest bundle hash does not match assembled bytes",
));
}
Ok(())
}
#[cfg(unix)]
fn mode_uid(metadata: &fs::Metadata) -> (u32, u32) {
use std::os::unix::fs::MetadataExt;
(metadata.uid(), metadata.mode() & 0o777)
}
#[cfg(not(unix))]
fn mode_uid(metadata: &fs::Metadata) -> (u32, u32) {
(0, if metadata.is_file() { 0o755 } else { 0o755 })
}
fn inspect_provider(path: &Path, execution_mode: &str) -> Result<Value, CliError> {
if !path.is_absolute() {
return Err(err("provider must be absolute"));
}
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()
.map_err(|e| err(e.to_string()))?
.join(path)
};
let parent = absolute
.parent()
.ok_or_else(|| err("provider has no parent"))?;
let canonical_parent = canonical(parent, "provider parent")?;
if canonical_parent != parent {
return Err(err("provider parent has a symlinked component"));
}
let alias = fs::symlink_metadata(&absolute).map_err(|e| err(e.to_string()))?;
if !alias.file_type().is_symlink() && !alias.is_file() {
return Err(err(
"configured Claude alias is not a symlink or regular file",
));
}
let alias_target = fs::canonicalize(&absolute).map_err(|e| err(e.to_string()))?;
let _ = canonical(&alias_target, "provider target")?;
if execution_mode == "live" && absolute.file_name().and_then(|n| n.to_str()) != Some("claude") {
return Err(err("configured Claude alias must end in claude"));
}
let target = fs::metadata(&alias_target).map_err(|e| err(e.to_string()))?;
if !target.is_file() {
return Err(err("configured Claude target is not a file"));
}
if target_mode(&target) & 0o022 != 0 {
return Err(err("configured Claude target has unsafe write permissions"));
}
if target_mode(&target) & 0o111 == 0 {
return Err(err("configured Claude target is not executable"));
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if target.uid() != rustix::process::geteuid().as_raw() {
return Err(err(
"configured Claude target owner is not the current user",
));
}
}
if target.len() > 512 * 1024 * 1024 {
return Err(err("configured Claude target exceeds 536870912 bytes"));
}
if execution_mode == "live" {
let lowered = alias_target.to_string_lossy().to_lowercase();
for marker in [
"fake",
"mock",
"fixture",
"/tmp/",
"/private/tmp/",
"/var/folders/",
"/private/var/folders/",
"/checkout/",
"/worktree/",
"/.git/",
] {
if lowered.contains(marker) {
return Err(err("live Claude target is fixture or PATH-shadowed"));
}
}
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if target.nlink() != 1 {
return Err(err("configured Claude target has hard links"));
}
}
let (target_sha256, target_identity) =
stream_hash(&alias_target, 512 * 1024 * 1024, "configured Claude target")?;
let link_target = if alias.file_type().is_symlink() {
fs::read_link(&absolute)
.ok()
.map(|p| p.to_string_lossy().into_owned())
} else {
None
};
let target_after = identity(&alias_target, "configured Claude target")?;
if target_after != target_identity {
return Err(err(
"configured Claude target identity changed during inspection",
));
}
if metadata_identity(&fs::symlink_metadata(&absolute).map_err(|e| err(e.to_string()))?)
!= metadata_identity(&alias)
{
return Err(err(
"configured Claude alias identity changed during inspection",
));
}
Ok(serde_json::json!({
"alias_path":absolute,
"alias_target":alias_target,
"alias_sha256":link_target.as_deref().map(|v|sha256_hex(v.as_bytes())).unwrap_or_else(||target_sha256.clone()),
"alias":{"kind":if alias.file_type().is_symlink(){"symlink"}else{"regular"},"identity":metadata_identity(&alias)},
"target_path":alias_target,
"target_sha256":target_sha256,
"target":target_identity
}))
}
fn target_mode(metadata: &fs::Metadata) -> u32 {
metadata_mode(metadata)
}
fn metadata_identity(metadata: &fs::Metadata) -> Value {
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
serde_json::json!({
"dev": metadata.dev(), "ino": metadata.ino(),
"mode": metadata.mode() & 0o777, "size": metadata.len(),
"mtime_ns": metadata.mtime_nsec(), "ctime_ns": metadata.ctime_nsec(),
"nlink": metadata.nlink(), "uid": metadata.uid(), "gid": metadata.gid()
})
}
#[cfg(not(unix))]
{
serde_json::json!({"mode": metadata_mode(metadata), "size": metadata.len()})
}
}
fn inspect_auth(path: &Path, session: &str) -> Result<Value, CliError> {
if !path.is_absolute() {
return Err(err("auth must be absolute"));
}
let absolute = canonical(path, "auth")?;
let metadata = fs::metadata(&absolute).map_err(|e| err(e.to_string()))?;
let auth_identity =
metadata_identity(&fs::symlink_metadata(&absolute).map_err(|e| err(e.to_string()))?);
let (uid, mode) = mode_uid(&metadata);
if mode != 0o600 {
return Err(err("trusted auth document must be mode 0600"));
}
#[cfg(unix)]
{
if uid != rustix::process::geteuid().as_raw() {
return Err(err("trusted auth document owner is not the current user"));
}
}
if session.is_empty() {
return Err(err("auth session identity is empty"));
}
let bytes = read_bounded(&absolute, MAX_INPUT_BYTES)?;
if metadata_identity(&fs::symlink_metadata(&absolute).map_err(|e| err(e.to_string()))?)
!= auth_identity
{
return Err(err("trusted auth document identity changed during read"));
}
let document = parse_json(&bytes)?;
let object = document
.as_object()
.ok_or_else(|| err("auth document is not an object"))?;
let valid = object.len() == 1
&& object
.get("claudeAiOauth")
.and_then(Value::as_object)
.is_some_and(|oauth| {
let allowed = [
"accessToken",
"expiresAt",
"rateLimitTier",
"refreshToken",
"scopes",
"subscriptionType",
]
.into_iter()
.collect::<std::collections::BTreeSet<_>>();
oauth.keys().all(|key| allowed.contains(key.as_str()))
&& oauth
.get("accessToken")
.and_then(Value::as_str)
.is_some_and(|token| !token.is_empty())
&& oauth.get("expiresAt").is_none_or(Value::is_number)
&& oauth.get("rateLimitTier").is_none_or(Value::is_string)
&& oauth.get("refreshToken").is_none_or(Value::is_string)
&& oauth.get("scopes").is_none_or(|value| {
value
.as_array()
.is_some_and(|items| items.iter().all(Value::is_string))
})
&& oauth.get("subscriptionType").is_none_or(Value::is_string)
});
if !valid {
return Err(err(
"auth document is not a recognized auth-only Claude document",
));
}
Ok(
serde_json::json!({"path":absolute,"sha256":sha256_hex(&bytes),"owner_uid":uid,"mode":mode,"schema":"claude.auth-only/1","session":session,"identity":{"uid":uid,"mode":mode,"size":metadata.len()},"document":document}),
)
}
fn verify_campaign(args: &EvalVerifyArgs) -> Result<(), CliError> {
if !matches!(args.execution_mode.as_str(), "deterministic" | "live") {
return Err(err("invalid execution mode"));
}
if args.trusted_shepherd.is_some() != args.trusted_shepherd_sha256.is_some() {
return Err(err(
"trusted Shepherd path and SHA-256 must be supplied together",
));
}
if let (Some(path), Some(expected)) = (&args.trusted_shepherd, &args.trusted_shepherd_sha256) {
let trusted = canonical(path, "trusted shepherd")?;
if metadata_mode(&fs::metadata(&trusted).map_err(|e| err(e.to_string()))?) & 0o111 == 0 {
return Err(err("trusted shepherd is not executable"));
}
if expected.len() != 64
|| !expected
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
{
return Err(err("trusted Shepherd SHA-256 is invalid"));
}
if sha256_hex(&read_bounded(&trusted, 64 * 1024 * 1024)?) != *expected {
return Err(err("trusted Shepherd SHA-256 does not match opened bytes"));
}
}
let output_path = canonical(&args.output, "campaign output")?;
let output_identity = identity(&output_path, "campaign output")?;
let artifact = parse_json(&read_bounded(&output_path, 2 * 1024 * 1024)?)?;
if identity(&output_path, "campaign output")? != output_identity {
return Err(err("campaign output identity changed during read"));
}
if artifact.get("schema").and_then(Value::as_str) != Some("shepherd.skill-campaign/2") {
return Err(err("campaign schema mismatch"));
}
if artifact.get("status").and_then(Value::as_str) != Some("complete") {
return Err(err("campaign status is not complete"));
}
if artifact.get("phase").and_then(Value::as_str) != Some("post-audit")
|| artifact
.get("kind")
.and_then(Value::as_str)
.is_none_or(str::is_empty)
|| artifact
.get("campaign_id")
.and_then(Value::as_str)
.is_none_or(str::is_empty)
|| artifact.get("repetitions").and_then(Value::as_u64) != Some(CAMPAIGN_REPETITIONS)
{
return Err(err("campaign object is not the exact post-audit contract"));
}
let policy = artifact
.get("policy")
.and_then(Value::as_object)
.ok_or_else(|| err("campaign policy is not the fixed v657 policy"))?;
if policy.len() != 2
|| policy.get("min_margin").and_then(Value::as_f64) != Some(CAMPAIGN_MIN_MARGIN)
|| policy.get("max_variance").and_then(Value::as_f64) != Some(CAMPAIGN_MAX_VARIANCE)
{
return Err(err("campaign policy is not the fixed v657 policy"));
}
if artifact.get("execution_mode").and_then(Value::as_str) != Some(args.execution_mode.as_str())
{
return Err(err("execution_mode does not match verifier mode"));
}
if artifact.get("executed_count").and_then(Value::as_u64) != Some(CAMPAIGN_GENERATIONS)
|| artifact.get("judge_executed_count").and_then(Value::as_u64) != Some(CAMPAIGN_JUDGES)
|| artifact
.get("auditor_executed_count")
.and_then(Value::as_u64)
!= Some(CAMPAIGN_AUDITORS)
{
return Err(err("campaign call counts are incomplete"));
}
let provenance = artifact
.get("provenance")
.and_then(Value::as_object)
.ok_or_else(|| err("campaign provenance is missing"))?;
let authority = provenance
.get("native_attestation")
.and_then(Value::as_object)
.ok_or_else(|| err("native campaign attestation is missing"))?;
let authority_manifest = provenance
.get("authority_manifest")
.and_then(Value::as_object)
.ok_or_else(|| err("campaign authority manifest binding is missing"))?;
if authority.get("schema").and_then(Value::as_str)
!= Some("shepherd.campaign-authority-attestation/1")
|| authority
.get("manifest")
.and_then(|value| value.get("sha256"))
!= authority_manifest.get("sha256")
{
return Err(err("campaign native provenance mismatch"));
}
let authority_path = authority_manifest
.get("path")
.and_then(Value::as_str)
.ok_or_else(|| err("campaign authority manifest path is missing"))?;
let trusted = provenance
.get("trusted_shepherd")
.and_then(Value::as_object)
.ok_or_else(|| err("trusted Shepherd provenance is missing"))?;
let trusted_path = trusted
.get("path")
.and_then(Value::as_str)
.ok_or_else(|| err("trusted Shepherd path is missing"))?;
let trusted_sha = trusted
.get("sha256")
.and_then(Value::as_str)
.ok_or_else(|| err("trusted Shepherd hash is missing"))?;
let trusted_path = canonical(Path::new(trusted_path), "trusted shepherd")?;
let supplied_trusted = args
.trusted_shepherd
.as_ref()
.ok_or_else(|| err("trusted Shepherd path is required"))?;
let supplied_trusted = canonical(supplied_trusted, "trusted shepherd argument")?;
if supplied_trusted != trusted_path
|| args.trusted_shepherd_sha256.as_deref() != Some(trusted_sha)
{
return Err(err(
"trusted Shepherd argument does not match artifact provenance",
));
}
let trusted_meta = fs::metadata(&trusted_path).map_err(|e| err(e.to_string()))?;
if !trusted_meta.is_file() || metadata_mode(&trusted_meta) & 0o111 == 0 {
return Err(err("trusted Shepherd is not an executable regular file"));
}
let (trusted_hash_before, trusted_identity) =
stream_hash(&trusted_path, 512 * 1024 * 1024, "trusted Shepherd")?;
let (trusted_hash_after, _) =
stream_hash(&trusted_path, 512 * 1024 * 1024, "trusted Shepherd")?;
if trusted_hash_before != trusted_hash_after || trusted_hash_before != trusted_sha {
return Err(err("trusted Shepherd SHA-256 changed or does not match"));
}
if identity(&trusted_path, "trusted Shepherd")? != trusted_identity {
return Err(err("trusted Shepherd identity changed during verification"));
}
let attestation_text = super::eval_manifest::verify_campaign_authority(
Path::new(authority_path),
"v657",
&trusted_path,
trusted_sha,
)?;
let attestation: Value = parse_json(attestation_text.as_bytes())?;
if attestation != Value::Object(authority.clone()) {
return Err(err("campaign authority attestation was substituted"));
}
let raw_dir = artifact
.get("raw_dir")
.and_then(Value::as_str)
.ok_or_else(|| err("raw campaign directory is missing"))?;
let output = output_path;
if raw_dir != format!("{}.raw", output.display()) {
return Err(err(
"raw campaign directory is not the canonical output sibling",
));
}
let arms = artifact
.get("arms")
.and_then(Value::as_object)
.ok_or_else(|| err("campaign arms are missing"))?;
let threshold = artifact
.get("rubric")
.and_then(|value| value.get("threshold"))
.and_then(Value::as_f64)
.ok_or_else(|| err("rubric threshold is missing"))?;
for arm in ["baseline", "guided", "adversarial"] {
let results = arms
.get(arm)
.and_then(|value| value.get("results"))
.and_then(Value::as_array)
.ok_or_else(|| err(format!("{arm} results are missing")))?;
if results.len() != 5
|| results.iter().enumerate().any(|(index, result)| {
result.get("repetition").and_then(Value::as_u64) != Some(index as u64 + 1)
})
{
return Err(err(format!("{arm} repetitions are incomplete")));
}
verify_arm_stats(arms.get(arm).expect("validated arm"))?;
for result in results {
let overall = result
.get("overall")
.and_then(Value::as_f64)
.ok_or_else(|| err("campaign overall score is missing"))?;
let passed = result
.get("passed")
.and_then(Value::as_bool)
.ok_or_else(|| err("campaign passed verdict is missing"))?;
if result
.get("verdict")
.and_then(|v| v.get("overall"))
.and_then(Value::as_f64)
!= Some(overall)
|| result
.get("verdict")
.and_then(|v| v.get("passed"))
.and_then(Value::as_bool)
!= Some(passed)
|| passed != (overall >= threshold)
{
return Err(err(format!(
"{arm} judge verdict contradicts overall score"
)));
}
}
if (arm == "baseline" && results.iter().any(|r| r["passed"] == Value::Bool(true)))
|| (arm != "baseline" && results.iter().any(|r| r["passed"] != Value::Bool(true)))
{
return Err(err(format!("{arm} pass/fail policy is violated")));
}
}
let raw_path = canonical(Path::new(raw_dir), "raw campaign directory")?;
let raw_metadata = fs::metadata(&raw_path).map_err(|e| err(e.to_string()))?;
if !raw_metadata.is_dir() || metadata_mode(&raw_metadata) != 0o700 {
return Err(err("raw campaign directory has unsafe mode or type"));
}
let raw_identity =
metadata_identity(&fs::symlink_metadata(&raw_path).map_err(|e| err(e.to_string()))?);
let mut expected_files = BTreeMap::new();
for arm in ["baseline", "guided", "adversarial"] {
let results = arms[arm]["results"].as_array().expect("validated results");
for result in results {
let repetition = result["repetition"].as_u64().expect("validated repetition");
for (field, suffix) in [
(
"generation_prompt_path",
format!("{arm}-{repetition}.generation.prompt.txt"),
),
(
"generation_raw_jsonl",
format!("{arm}-{repetition}.generation.jsonl"),
),
("judge_raw_jsonl", format!("{arm}-{repetition}.judge.jsonl")),
(
"judge_system_path",
format!("{arm}-{repetition}.judge.system.txt"),
),
(
"judge_prompt_path",
format!("{arm}-{repetition}.judge.prompt.txt"),
),
(
"judge_response_path",
format!("{arm}-{repetition}.judge.response.txt"),
),
] {
let path = result
.get(field)
.and_then(Value::as_str)
.ok_or_else(|| err(format!("{field} is missing")))?;
let name = direct_child(path, &raw_path, field)?;
if name != suffix {
return Err(err(format!("{field} is not canonical")));
}
expected_files.insert(name, path.to_owned());
}
}
}
for name in [
"campaign-payload.json",
"auditor-invocation.json",
"auditor-response.jsonl",
"auditor-review.json",
] {
expected_files.insert(name.to_owned(), format!("{raw_dir}/{name}"));
}
let actual_names = fs::read_dir(&raw_path)
.map_err(|e| err(e.to_string()))?
.map(|entry| {
entry
.map(|entry| entry.file_name().to_string_lossy().into_owned())
.map_err(|e| err(e.to_string()))
})
.collect::<Result<std::collections::BTreeSet<_>, _>>()?;
if actual_names.len() != 94 || actual_names != expected_files.keys().cloned().collect() {
return Err(err(
"raw campaign evidence file set is incomplete or substituted",
));
}
let mut raw_data = BTreeMap::new();
for name in actual_names {
let file = raw_path.join(&name);
let bytes = read_raw_file(&file, &format!("raw evidence {name}"))?;
if bytes.is_empty() {
return Err(err(format!("raw evidence is empty: {name}")));
}
raw_data.insert(name.clone(), bytes);
}
let skill = artifact
.get("skill")
.and_then(Value::as_object)
.ok_or_else(|| err("skill evidence is missing"))?;
let bundle = collect_bundle(&args.skill_dir)?;
if skill.get("path").and_then(Value::as_str)
!= Some(
canonical(&args.skill_dir, "skill directory")?
.to_string_lossy()
.as_ref(),
)
|| skill.get("bundle") != Some(&bundle_metadata(&bundle))
|| skill.get("bundle_sha256").and_then(Value::as_str)
!= Some(bundle_digest(&bundle).as_str())
{
return Err(err("skill bundle identity mismatch"));
}
let rubric_path = canonical(&args.rubric, "rubric")?;
let rubric_bytes = read_bounded(&rubric_path, MAX_INPUT_BYTES)?;
let rubric = parse_json(&rubric_bytes)?;
let rubric_meta = artifact
.get("rubric")
.and_then(Value::as_object)
.ok_or_else(|| err("rubric evidence is missing"))?;
if rubric_meta.get("path").and_then(Value::as_str)
!= Some(rubric_path.to_string_lossy().as_ref())
|| rubric_meta.get("sha256").and_then(Value::as_str)
!= Some(sha256_hex(&rubric_bytes).as_str())
|| rubric.get("kind") != artifact.get("kind")
|| rubric.get("threshold") != rubric_meta.get("threshold")
{
return Err(err("rubric identity mismatch"));
}
let kind = artifact
.get("kind")
.and_then(Value::as_str)
.expect("validated kind");
if rubric.get("kind").and_then(Value::as_str) != Some(kind) {
return Err(err("campaign kind does not match rubric kind"));
}
for arm in ["baseline", "guided", "adversarial"] {
let recorded_case = artifact
.get("cases")
.and_then(|cases| cases.get(arm))
.and_then(Value::as_object)
.ok_or_else(|| err(format!("{arm} case identity is missing")))?;
let case_path = recorded_case
.get("path")
.and_then(Value::as_str)
.ok_or_else(|| err(format!("{arm} case path is missing")))?;
let case_path = canonical(Path::new(case_path), &format!("{arm} case"))?;
let (case_sha, _) = stream_hash(&case_path, MAX_INPUT_BYTES, &format!("{arm} case"))?;
if recorded_case.get("sha256").and_then(Value::as_str) != Some(case_sha.as_str())
|| arms.get(arm).and_then(|value| value.get("case_path"))
!= Some(&Value::String(case_path.to_string_lossy().into_owned()))
|| arms.get(arm).and_then(|value| value.get("case_sha256"))
!= Some(&Value::String(case_sha))
{
return Err(err(format!(
"{arm} case identity changed or was substituted"
)));
}
}
let mut calls = Vec::new();
for arm in ["baseline", "guided", "adversarial"] {
for result in arms[arm]["results"].as_array().expect("validated results") {
let generation_name = Path::new(result["generation_raw_jsonl"].as_str().expect("path"))
.file_name()
.unwrap()
.to_string_lossy();
let judge_name = Path::new(result["judge_raw_jsonl"].as_str().expect("path"))
.file_name()
.unwrap()
.to_string_lossy();
let generation = raw_data
.get(generation_name.as_ref())
.ok_or_else(|| err("generation evidence missing"))?;
let judge = raw_data
.get(judge_name.as_ref())
.ok_or_else(|| err("judge evidence missing"))?;
let generation_hash = json_hash(result, "generation_raw_sha256")?;
let judge_hash = json_hash(result, "judge_raw_sha256")?;
if generation_hash != sha256_hex(generation) || judge_hash != sha256_hex(judge) {
return Err(err("raw result hash mismatch"));
}
let prompt_name = Path::new(result["generation_prompt_path"].as_str().expect("path"))
.file_name()
.unwrap()
.to_string_lossy();
let prompt = raw_data
.get(prompt_name.as_ref())
.ok_or_else(|| err("generation prompt evidence missing"))?;
if result.get("prompt_sha256").and_then(Value::as_str)
!= Some(sha256_hex(prompt).as_str())
|| result
.get("generation_prompt_sha256")
.and_then(Value::as_str)
!= result.get("prompt_sha256").and_then(Value::as_str)
{
return Err(err("generation prompt hash mismatch"));
}
for (field, name_field) in [
("judge_system_sha256", "judge_system_path"),
("judge_prompt_sha256", "judge_prompt_path"),
("judge_response_sha256", "judge_response_path"),
] {
let name = Path::new(result[name_field].as_str().expect("path"))
.file_name()
.unwrap()
.to_string_lossy();
let bytes = raw_data
.get(name.as_ref())
.ok_or_else(|| err("judge prompt evidence missing"))?;
if result.get(field).and_then(Value::as_str) != Some(sha256_hex(bytes).as_str()) {
return Err(err(format!("{field} does not reproduce")));
}
}
let (generation_record, generation_session) =
terminal_record(generation, &generation_name)?;
let (judge_record, judge_session) = terminal_record(judge, &judge_name)?;
if result.get("generation_session_id").and_then(Value::as_str)
!= Some(&generation_session)
|| result.get("generation_call_id").and_then(Value::as_str)
!= Some(&generation_session)
|| result.get("judge_session_id").and_then(Value::as_str) != Some(&judge_session)
|| result.get("judge_call_id").and_then(Value::as_str) != Some(&judge_session)
{
return Err(err("campaign call identity mismatch"));
}
calls.extend([generation_session, judge_session]);
let response = generation_record
.get("result")
.and_then(Value::as_str)
.ok_or_else(|| err("generation result missing"))?;
if result.get("response_sha256").and_then(Value::as_str)
!= Some(sha256_hex(response.as_bytes()).as_str())
{
return Err(err("generation response hash mismatch"));
}
let judge_response = judge_record
.get("result")
.and_then(Value::as_str)
.ok_or_else(|| err("judge result missing"))?;
let expected = expected_verdict(
&rubric,
judge_response.as_bytes(),
kind,
artifact
.get("model")
.and_then(Value::as_str)
.ok_or_else(|| err("campaign model is missing"))?,
)?;
if result.get("verdict") != Some(&expected)
|| result.get("overall") != expected.get("overall")
|| result.get("passed") != expected.get("passed")
{
return Err(err(
"stored judge verdict does not reproduce normalized evaluator verdict",
));
}
let response_name = Path::new(result["judge_response_path"].as_str().expect("path"))
.file_name()
.unwrap()
.to_string_lossy();
if raw_data.get(response_name.as_ref()).map(Vec::as_slice)
!= Some(judge_response.as_bytes())
{
return Err(err("judge response evidence differs from raw JSONL"));
}
}
}
let raw_identity_after =
metadata_identity(&fs::symlink_metadata(&raw_path).map_err(|e| err(e.to_string()))?);
if raw_identity != raw_identity_after {
return Err(err(
"raw campaign directory identity changed during verification",
));
}
if calls.len() != 30
|| calls
.iter()
.collect::<std::collections::BTreeSet<_>>()
.len()
!= 30
{
return Err(err("campaign call IDs are duplicated"));
}
let generation_ids = ["baseline", "guided", "adversarial"]
.into_iter()
.flat_map(|name| arms[name]["results"].as_array().into_iter().flatten())
.map(|result| {
result["generation_call_id"]
.as_str()
.unwrap_or_default()
.to_owned()
})
.collect::<Vec<_>>();
let judge_ids = ["baseline", "guided", "adversarial"]
.into_iter()
.flat_map(|name| arms[name]["results"].as_array().into_iter().flatten())
.map(|result| {
result["judge_call_id"]
.as_str()
.unwrap_or_default()
.to_owned()
})
.collect::<Vec<_>>();
let generation_hashes = ["baseline", "guided", "adversarial"]
.into_iter()
.flat_map(|name| arms[name]["results"].as_array().into_iter().flatten())
.map(|result| {
result["generation_raw_sha256"]
.as_str()
.unwrap_or_default()
.to_owned()
})
.collect::<Vec<_>>();
let judge_hashes = ["baseline", "guided", "adversarial"]
.into_iter()
.flat_map(|name| arms[name]["results"].as_array().into_iter().flatten())
.map(|result| {
result["judge_raw_sha256"]
.as_str()
.unwrap_or_default()
.to_owned()
})
.collect::<Vec<_>>();
let calls_doc = artifact
.get("calls")
.and_then(Value::as_object)
.ok_or_else(|| err("campaign call ledger is missing"))?;
if calls_doc
.get("generation")
.and_then(|v| v.get("count"))
.and_then(Value::as_u64)
!= Some(CAMPAIGN_GENERATIONS)
|| calls_doc
.get("judge")
.and_then(|v| v.get("count"))
.and_then(Value::as_u64)
!= Some(CAMPAIGN_JUDGES)
|| calls_doc.get("generation").and_then(|v| v.get("ids"))
!= Some(&serde_json::json!(generation_ids))
|| calls_doc.get("judge").and_then(|v| v.get("ids")) != Some(&serde_json::json!(judge_ids))
|| calls_doc.get("generation").and_then(|v| v.get("hashes"))
!= Some(&serde_json::json!(generation_hashes))
|| calls_doc.get("judge").and_then(|v| v.get("hashes"))
!= Some(&serde_json::json!(judge_hashes))
{
return Err(err("generation or judge call ledger is substituted"));
}
let services = artifact
.get("services")
.and_then(Value::as_object)
.ok_or_else(|| err("campaign service identity schema is missing"))?;
let service_names = [
"llm",
"eval",
"bundle",
"runner",
"verifier",
"verifier_impl",
"acceptance",
"contract",
];
if services.get("schema").and_then(Value::as_str)
!= Some("shepherd.campaign-service-identity/1")
|| services
.keys()
.filter(|key| key.as_str() != "schema")
.count()
!= service_names.len()
|| service_names.iter().any(|name| {
let Some(service) = services.get(*name).and_then(Value::as_object) else {
return true;
};
let keys = service
.keys()
.map(String::as_str)
.collect::<std::collections::BTreeSet<_>>();
keys != ["path", "sha256"].into_iter().collect()
})
{
return Err(err("campaign service identity schema is invalid"));
}
for name in service_names {
let service = services[name].as_object().expect("validated service");
let path = service["path"]
.as_str()
.ok_or_else(|| err("service path is missing"))?;
let path = canonical(Path::new(path), &format!("service {name}"))?;
let (sha, _) = stream_hash(&path, 512 * 1024 * 1024, &format!("service {name}"))?;
if service["sha256"].as_str() != Some(sha.as_str()) {
return Err(err(format!("service {name} identity changed")));
}
}
let auditor = artifact
.get("auditor_review")
.and_then(Value::as_object)
.ok_or_else(|| err("Auditor review is missing"))?;
if auditor.get("role").and_then(Value::as_str) != Some("auditor")
|| auditor.get("verdict").and_then(Value::as_str) != Some("approved")
{
return Err(err("Auditor review is not approved"));
}
let auditor_fields = [
("path", "auditor-review.json"),
("raw_invocation_path", "auditor-invocation.json"),
("raw_response_path", "auditor-response.jsonl"),
];
for (field, name) in auditor_fields {
let path = auditor
.get(field)
.and_then(Value::as_str)
.ok_or_else(|| err(format!("Auditor {field} is missing")))?;
if canonical_child(path, &raw_path, &format!("Auditor {field}"))? != name {
return Err(err(format!("Auditor {field} is not canonical")));
}
}
let review_bytes = raw_data
.get("auditor-review.json")
.expect("fixed Auditor evidence");
let invocation_bytes = raw_data
.get("auditor-invocation.json")
.expect("fixed Auditor evidence");
let response_bytes = raw_data
.get("auditor-response.jsonl")
.expect("fixed Auditor evidence");
let payload_sha = artifact
.get("pre_audit_payload")
.and_then(|value| value.get("sha256"))
.and_then(Value::as_str)
.ok_or_else(|| err("pre-audit payload binding is missing"))?;
let payload_path = artifact["pre_audit_payload"]["path"]
.as_str()
.ok_or_else(|| err("pre-audit payload path is missing"))?;
if canonical_child(payload_path, &raw_path, "pre-audit payload")? != "campaign-payload.json"
|| sha256_hex(
raw_data
.get("campaign-payload.json")
.expect("fixed payload"),
) != payload_sha
{
return Err(err("pre-audit payload path or hash does not reproduce"));
}
let payload_bytes = raw_data
.get("campaign-payload.json")
.expect("fixed payload");
let payload = parse_json(payload_bytes)?;
if payload.get("schema").and_then(Value::as_str) != Some("shepherd.skill-campaign-pre-audit/1")
|| payload.get("phase").and_then(Value::as_str) != Some("post-campaign")
|| payload.get("campaign_id") != artifact.get("campaign_id")
|| canonical_json_value(&payload) != *payload_bytes
{
return Err(err("pre-audit payload is not canonical or fully bound"));
}
for (field, bytes) in [
("sha256", review_bytes),
("raw_invocation_sha256", invocation_bytes),
("raw_response_sha256", response_bytes),
] {
let expected = sha256_hex(bytes);
let recorded = if field == "sha256" {
auditor.get("sha256")
} else {
auditor.get(field)
};
if recorded.and_then(Value::as_str) != Some(expected.as_str()) {
return Err(err(format!("Auditor {field} does not reproduce")));
}
}
let invocation = parse_json(invocation_bytes)?;
if invocation.get("schema").and_then(Value::as_str)
!= Some("shepherd.skill-campaign-auditor-invocation/1")
|| invocation.get("phase").and_then(Value::as_str) != Some("post-campaign")
|| invocation.get("campaign_id") != artifact.get("campaign_id")
|| invocation.get("campaign_payload_path")
!= artifact
.get("pre_audit_payload")
.and_then(|v| v.get("path"))
|| invocation.get("campaign_payload_sha256")
!= artifact
.get("pre_audit_payload")
.and_then(|v| v.get("sha256"))
|| invocation.get("raw_campaign_dir").and_then(Value::as_str) != Some(raw_dir)
|| invocation.get("sequence").and_then(Value::as_u64) != Some(2)
|| invocation.get("raw_id").and_then(Value::as_str)
!= auditor.get("raw_id").and_then(Value::as_str)
{
return Err(err("Auditor invocation is not fully bound"));
}
let (auditor_record, auditor_session) =
terminal_record(response_bytes, "auditor-response.jsonl")?;
if auditor.get("call_id").and_then(Value::as_str) != Some(&auditor_session)
|| auditor.get("raw_id").and_then(Value::as_str) != Some(&auditor_session)
|| calls_doc
.get("auditor")
.and_then(|v| v.get("count"))
.and_then(Value::as_u64)
!= Some(CAMPAIGN_AUDITORS)
|| calls_doc.get("auditor").and_then(|v| v.get("ids"))
!= Some(&serde_json::json!([auditor_session.clone()]))
|| calls_doc.get("auditor").and_then(|v| v.get("hashes"))
!= Some(&serde_json::json!([sha256_hex(response_bytes)]))
{
return Err(err("Auditor call ledger is not exact"));
}
if calls.contains(&auditor_session) {
return Err(err("Auditor call ID reuses a generation or judge call"));
}
if auditor
.get("call_ledger")
.and_then(Value::as_object)
.is_none_or(|ledger| {
ledger.get("generation") != calls_doc.get("generation")
|| ledger.get("judge") != calls_doc.get("judge")
|| ledger.get("auditor") != calls_doc.get("auditor")
})
{
return Err(err(
"Auditor raw call ledger is not the exact 31-call ledger",
));
}
let auditor_review = auditor_record
.get("result")
.and_then(Value::as_str)
.ok_or_else(|| err("Auditor result is missing"))?;
let auditor_review: Value = parse_json(auditor_review.as_bytes())?;
let review_document = parse_json(review_bytes)?;
if auditor_review != review_document {
return Err(err("Auditor response and review payload differ"));
}
if review_document.get("schema").and_then(Value::as_str)
!= Some("shepherd.skill-campaign-auditor-review/2")
|| review_document
.get("campaign_schema")
.and_then(Value::as_str)
!= Some("shepherd.skill-campaign/2")
|| review_document.get("phase").and_then(Value::as_str) != Some("post-campaign")
|| review_document.get("campaign_id") != artifact.get("campaign_id")
|| review_document.get("campaign_payload_sha256")
!= Some(&Value::String(payload_sha.to_owned()))
|| review_document
.get("claims")
.and_then(Value::as_array)
.is_none_or(Vec::is_empty)
|| review_document
.get("citations")
.and_then(Value::as_array)
.is_none_or(Vec::is_empty)
|| review_document["reviewer"]["id"]
.as_str()
.is_none_or(str::is_empty)
|| review_document["reviewer"]["status"].as_str() != Some("complete")
|| review_document["reviewer"]["phase_sequence"].as_u64() != Some(3)
|| !((args.execution_mode == "deterministic"
&& review_document["reviewer"]["execution_mode"].as_str()
== Some("deterministic-fake"))
|| (args.execution_mode == "live"
&& review_document["reviewer"]["execution_mode"].as_str() == Some("live")))
{
return Err(err("Auditor review schema or payload binding is invalid"));
}
let citation_ids = review_document["citations"]
.as_array()
.expect("validated citations")
.iter()
.filter_map(|citation| citation.get("id").and_then(Value::as_str))
.collect::<std::collections::BTreeSet<_>>();
for claim in review_document["claims"]
.as_array()
.expect("validated claims")
{
if claim
.get("id")
.and_then(Value::as_str)
.is_none_or(str::is_empty)
|| claim
.get("assertion")
.and_then(Value::as_str)
.is_none_or(str::is_empty)
|| !matches!(
claim.get("result").and_then(Value::as_str),
Some("approved" | "rejected")
)
|| claim
.get("citations")
.and_then(Value::as_array)
.is_none_or(|refs| {
refs.is_empty()
|| refs.iter().any(|reference| {
reference
.as_str()
.is_none_or(|id| !citation_ids.contains(id))
})
})
{
return Err(err("Auditor claim is incomplete"));
}
}
for citation in review_document["citations"]
.as_array()
.expect("validated citations")
{
if citation
.get("id")
.and_then(Value::as_str)
.is_none_or(str::is_empty)
|| citation
.get("path")
.and_then(Value::as_str)
.is_none_or(str::is_empty)
|| citation
.get("predicate")
.and_then(Value::as_str)
.is_none_or(str::is_empty)
{
return Err(err("Auditor citation is incomplete"));
}
}
for field in [
"schema",
"campaign_schema",
"role",
"verdict",
"reviewer",
"claims",
"citations",
"campaign_id",
"campaign_payload_sha256",
"bindings",
] {
if auditor.get(field) != review_document.get(field) {
return Err(err(format!("Auditor review field {field} was substituted")));
}
}
if artifact
.get("auditor_executed_count")
.and_then(Value::as_u64)
!= Some(1)
|| auditor.get("review_sequence").and_then(Value::as_u64) != Some(4)
|| auditor
.get("reviewer")
.and_then(|value| value.get("phase_sequence"))
.and_then(Value::as_u64)
!= Some(3)
{
return Err(err("Auditor review execution binding is invalid"));
}
if auditor
.get("campaign_payload_sha256")
.and_then(Value::as_str)
!= Some(payload_sha)
|| auditor
.get("bindings")
.and_then(|value| value.get("campaign_payload_sha256"))
.and_then(Value::as_str)
!= Some(payload_sha)
{
return Err(err("Auditor payload binding is substituted"));
}
let bindings = auditor
.get("bindings")
.and_then(Value::as_object)
.ok_or_else(|| err("Auditor bindings are missing"))?;
if bindings.get("campaign_payload_path") != Some(&Value::String(payload_path.to_owned()))
|| bindings.get("raw_campaign_dir") != Some(&Value::String(raw_dir.to_owned()))
|| bindings.get("input_hashes") != artifact.get("input_hashes")
|| bindings.get("auth") != artifact.get("auth")
{
return Err(err("Auditor bindings do not cover all campaign inputs"));
}
let bound_calls = bindings
.get("calls")
.and_then(Value::as_object)
.ok_or_else(|| err("Auditor call binding is missing"))?;
if bound_calls.get("generation") != calls_doc.get("generation")
|| bound_calls.get("judge") != calls_doc.get("judge")
|| bound_calls
.get("auditor")
.and_then(|v| v.get("count"))
.and_then(Value::as_u64)
!= Some(1)
|| bound_calls
.get("auditor")
.and_then(|v| v.get("ids"))
.and_then(Value::as_array)
.is_none_or(|ids| !ids.is_empty())
{
return Err(err("Auditor pre-audit call binding is not exact"));
}
let _ = collect_bundle(&args.skill_dir)?;
let _ = parse_json(&read_bounded(&args.rubric, MAX_INPUT_BYTES)?)?;
let provider = inspect_provider(&args.trusted_provider, &args.execution_mode)?;
let recorded = artifact
.get("claude")
.and_then(Value::as_object)
.ok_or_else(|| err("Claude provider evidence is missing"))?;
for field in [
"alias_path",
"alias_target",
"alias_sha256",
"target_path",
"target_sha256",
] {
if recorded.get(field) != provider.get(field) {
return Err(err(format!(
"Claude provider {field} changed during campaign"
)));
}
}
if recorded.get("alias") != provider.get("alias")
|| recorded.get("target") != provider.get("target")
{
return Err(err("Claude provider file identity changed during campaign"));
}
if artifact.get("executable")
!= Some(&serde_json::json!({
"command":"claude",
"path":provider["target_path"],
"sha256":provider["target_sha256"],
"alias_path":provider["alias_path"],
"alias_target":provider["alias_target"],
"alias_sha256":provider["alias_sha256"],
"alias":provider["alias"],
"target_path":provider["target_path"],
"target_sha256":provider["target_sha256"],
"target":provider["target"]
}))
{
return Err(err("executable provider identity was substituted"));
}
let campaign_id = artifact
.get("campaign_id")
.and_then(Value::as_str)
.ok_or_else(|| err("campaign identity is missing"))?;
let auth = inspect_auth(&args.trusted_auth, campaign_id)?;
let recorded_auth = artifact
.get("auth")
.and_then(Value::as_object)
.ok_or_else(|| err("auth evidence is missing"))?;
for field in ["path", "sha256", "owner_uid", "schema", "session"] {
if recorded_auth.get(field) != auth.get(field) {
return Err(err(format!("auth {field} changed during campaign")));
}
}
if recorded_auth.get("document_mode") != auth.get("mode") {
return Err(err("auth document mode changed during campaign"));
}
let expected_auth_mode = if args.execution_mode == "live" {
"auth-only"
} else {
"trusted-fake"
};
if artifact["auth"]["mode"].as_str() != Some(expected_auth_mode)
|| artifact["auth"]["authenticated"].as_bool() != Some(args.execution_mode == "live")
|| recorded_auth.get("document") != auth.get("document")
{
return Err(err(
"auth execution mode or full document binding is invalid",
));
}
let baseline_median = arms["baseline"]["stats"]["median"]
.as_f64()
.ok_or_else(|| err("baseline median is missing"))?;
let guided_median = arms["guided"]["stats"]["median"]
.as_f64()
.ok_or_else(|| err("guided median is missing"))?;
let adversarial_median = arms["adversarial"]["stats"]["median"]
.as_f64()
.ok_or_else(|| err("adversarial median is missing"))?;
let margins = artifact
.get("margins")
.and_then(Value::as_object)
.ok_or_else(|| err("campaign margins are missing"))?;
if margins.get("guided_over_baseline").and_then(Value::as_f64)
!= Some(guided_median - baseline_median)
|| margins
.get("adversarial_over_baseline")
.and_then(Value::as_f64)
!= Some(adversarial_median - baseline_median)
{
return Err(err("campaign margins do not reproduce"));
}
if guided_median - baseline_median < CAMPAIGN_MIN_MARGIN
|| adversarial_median - baseline_median < CAMPAIGN_MIN_MARGIN
|| ["baseline", "guided", "adversarial"].iter().any(|arm| {
arms[*arm]["stats"]["variance"]
.as_f64()
.is_none_or(|variance| variance > CAMPAIGN_MAX_VARIANCE)
})
{
return Err(err("campaign margin or variance policy is violated"));
}
println!("ok: verified shepherd.skill-campaign/2");
Ok(())
}
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[cfg(test)]
mod tests {
use super::*;
fn rubric() -> Value {
serde_json::json!({
"scale": 5,
"threshold": 80,
"dimensions": [
{"key":"quality","weight":1},
{"key":"custody","weight":2}
]
})
}
#[test]
fn expected_verdict_normalizes_raw_scores_with_half_up_rounding() {
let verdict = expected_verdict(
&rubric(),
br#"{"scores":{"quality":4,"custody":5},"rationale":"fixture"}"#,
"skill-test",
"fixture-opus",
)
.expect("normalized verdict");
assert_eq!(verdict["overall"], 93);
assert_eq!(verdict["passed"], true);
assert_eq!(verdict["kind"], "skill-test");
assert_eq!(verdict["model"], "fixture-opus");
assert_eq!(verdict["rationale"], "fixture");
}
#[test]
fn expected_verdict_rejects_missing_or_extra_dimensions() {
let missing = expected_verdict(&rubric(), br#"{"scores":{"quality":5}}"#, "k", "m");
assert!(missing.is_err());
let extra = expected_verdict(
&rubric(),
br#"{"scores":{"quality":5,"custody":5,"extra":1}}"#,
"k",
"m",
);
assert!(extra.is_err());
}
#[test]
fn expected_verdict_rejects_non_integer_and_out_of_range_scores() {
assert!(
expected_verdict(
&rubric(),
br#"{"scores":{"quality":1.5,"custody":5}}"#,
"k",
"m"
)
.is_err()
);
assert!(
expected_verdict(
&rubric(),
br#"{"scores":{"quality":0,"custody":5}}"#,
"k",
"m"
)
.is_err()
);
assert!(
expected_verdict(
&rubric(),
br#"{"scores":{"quality":6,"custody":5}}"#,
"k",
"m"
)
.is_err()
);
}
}