use std::collections::{BTreeMap, BTreeSet, HashMap};
use serde_json::{Value, json};
use skald_spec::{Prompt, ResponseType};
use super::extract::ExtractOutput;
use super::reference::{OrderBy, Reference, ReferenceRoot, SourceField};
use super::schema_path::{admits, fits};
use super::{
Action, AgentCall, Check, CheckEvidence, CodeCall, CodeSource, Condition, Decide, DriftCheck,
InputSlot, MAX_STEP_RETRIES, Requirement, Step, VerifierSpec,
};
use crate::card::agent::{AgentSpec, BuiltinTool};
use crate::card::drift::DriftSignal;
use crate::card::prompt::{ParameterName, is_valid_parameter_name};
use crate::envelope::CardKind;
use crate::error::WyrdError;
use crate::ids::{FeatureName, SlotName};
use crate::reference::InlineableRef;
use crate::verification::execution::Verdict;
use crate::verification::{ExecutionPlan, TaskId, plan_stages};
#[derive(Debug, Clone, Copy)]
pub struct ResolvedAgent<'a> {
pub agent: &'a AgentSpec,
pub prompt: &'a Prompt,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Position<'a> {
Step(&'a TaskId),
StepCheck(&'a TaskId),
VerifierCheck,
}
fn invalid(rule: &str, path: &str, message: impl std::fmt::Display) -> WyrdError {
WyrdError::Validation {
message: format!("{path}: {message}"),
details: json!({ "rule": rule, "path": path }),
}
}
impl VerifierSpec {
pub fn validate(&self) -> Result<(), WyrdError> {
self.validate_with(|agent| {
let spec = agent.as_inline()?;
Some(ResolvedAgent {
agent: spec,
prompt: spec.prompt.as_inline()?,
})
})
}
pub fn validate_with<'a, F>(&'a self, resolve: F) -> Result<(), WyrdError>
where
F: Fn(&'a InlineableRef<AgentSpec>) -> Option<ResolvedAgent<'a>>,
{
Validator {
spec: self,
resolve,
owners: self.requirement_owners()?,
ancestors: BTreeMap::new(),
}
.run()
}
pub fn plan(&self) -> Result<ExecutionPlan, WyrdError> {
let dependencies = self.dependencies();
plan_stages(
dependencies
.iter()
.map(|(id, deps)| (*id, deps.iter().copied())),
)
.map_err(|error| WyrdError::Validation {
message: format!("steps: {error}"),
details: json!({ "rule": "graph", "path": "steps", "error": error }),
})
}
fn dependencies(&self) -> BTreeMap<&TaskId, BTreeSet<&TaskId>> {
self.steps
.iter()
.map(|step| {
let mut deps: BTreeSet<&TaskId> = step.depends_on.iter().collect();
deps.extend(step_references(step).filter_map(Reference::step));
(&step.id, deps)
})
.collect()
}
fn requirement_owners(&self) -> Result<HashMap<&TaskId, &TaskId>, WyrdError> {
let mut seen = BTreeSet::new();
let mut owners = HashMap::new();
let ids = self
.steps
.iter()
.flat_map(|step| {
std::iter::once((&step.id, None)).chain(
step.requirements
.iter()
.map(|req| (&req.id, Some(&step.id))),
)
})
.chain(self.requirements.iter().map(|req| (&req.id, None)));
for (id, owner) in ids {
if !seen.insert(id) {
return Err(invalid(
"duplicate_id",
id.as_str(),
"step and requirement IDs must be unique across the Verifier",
));
}
if let Some(owner) = owner {
owners.insert(id, owner);
}
}
Ok(owners)
}
}
fn step_references(step: &Step) -> impl Iterator<Item = &Reference> {
let action: Vec<&Reference> = match &step.action {
Action::Extract { from } => vec![from],
Action::Agent(call) => call
.inputs
.values()
.chain(call.workspace.values())
.collect(),
Action::Code(call) => call.inputs.values().collect(),
};
action
.into_iter()
.chain(step.when.as_ref().map(|when| &when.path))
}
struct Validator<'a, F> {
spec: &'a VerifierSpec,
resolve: F,
owners: HashMap<&'a TaskId, &'a TaskId>,
ancestors: BTreeMap<&'a TaskId, BTreeSet<&'a TaskId>>,
}
impl<'a, F> Validator<'a, F>
where
F: Fn(&'a InlineableRef<AgentSpec>) -> Option<ResolvedAgent<'a>>,
{
fn run(mut self) -> Result<(), WyrdError> {
let spec = self.spec;
for (slot, input) in &spec.inputs {
validate_slot(slot, input)?;
}
if let Some(order) = &spec.order_by {
validate_order(spec, order)?;
}
self.validate_evidence_mode()?;
if let Decide::PassRate { threshold } = spec.decide
&& !(threshold.is_finite() && threshold > 0.0 && threshold <= 1.0)
{
return Err(invalid(
"pass_rate",
"decide.pass_rate.threshold",
"threshold must be finite, above 0, and at most 1",
));
}
for (index, step) in spec.steps.iter().enumerate() {
let path = format!("steps[{index}]");
self.validate_step_shape(step, &path)?;
}
let plan = spec.plan()?;
self.ancestors = ancestors(spec, &plan);
for (index, step) in spec.steps.iter().enumerate() {
for (req_index, requirement) in step.requirements.iter().enumerate() {
let path = format!("steps[{index}].requirements[{req_index}]");
self.validate_requirement(requirement, Position::StepCheck(&step.id), &path)?;
}
}
for (index, requirement) in spec.requirements.iter().enumerate() {
let path = format!("requirements[{index}]");
self.validate_requirement(requirement, Position::VerifierCheck, &path)?;
}
Ok(())
}
fn validate_evidence_mode(&self) -> Result<(), WyrdError> {
let spec = self.spec;
let samples = spec
.inputs
.values()
.filter(|slot| matches!(slot, InputSlot::Samples { .. }))
.count();
if samples > 0 && samples < spec.inputs.len() {
return Err(invalid(
"mixed_evidence",
"inputs",
"samples slots cannot be combined with other slot types",
));
}
let mut modes = spec
.steps
.iter()
.flat_map(|step| &step.requirements)
.chain(&spec.requirements)
.map(|requirement| requirement.check.evidence());
let first = modes.next();
if modes.any(|mode| Some(mode) != first) {
return Err(invalid(
"mixed_evidence",
"requirements",
"drift checks cannot be combined with run-evidence checks",
));
}
if first == Some(CheckEvidence::Window) || samples > 0 {
if first == Some(CheckEvidence::Run) {
return Err(invalid(
"mixed_evidence",
"requirements",
"samples slots are judged only by drift checks",
));
}
if !spec.steps.is_empty() {
return Err(invalid(
"window_steps",
"steps",
"a scheduled-window Verifier has no preparation steps",
));
}
}
Ok(())
}
fn validate_step_shape(&self, step: &'a Step, path: &str) -> Result<(), WyrdError> {
if step.timeout_seconds == Some(0) {
return Err(invalid(
"step_budget",
&format!("{path}.timeout_seconds"),
"timeout must be positive",
));
}
if step.retries > MAX_STEP_RETRIES {
return Err(invalid(
"step_budget",
&format!("{path}.retries"),
format!("at most {MAX_STEP_RETRIES} retries"),
));
}
for dep in &step.depends_on {
if !self.spec.steps.iter().any(|other| &other.id == dep) {
return Err(invalid(
"unknown_reference",
&format!("{path}.depends_on"),
format!("unknown step {}", dep.as_str()),
));
}
}
let position = Position::Step(&step.id);
if let Some(Condition {
path: when,
operator,
..
}) = &step.when
{
let when_path = format!("{path}.when");
self.validate_reference(when, position, false, &when_path)?;
validate_operator(operator, &when_path)?;
}
let action_path = format!("{path}.action");
match &step.action {
Action::Extract { from } => {
self.validate_reference(from, position, false, &action_path)?;
let Some(InputSlot::Media { formats, .. }) = self.whole_slot(from) else {
return Err(invalid(
"extract_source",
&action_path,
"extract reads one whole media slot",
));
};
if let Some(format) = formats.iter().find(|format| !format.is_extractable()) {
return Err(invalid(
"extract_format",
&action_path,
format!(
"{format} is not extractable; accept an OOXML, text, or PDF format"
),
));
}
}
Action::Agent(call) => self.validate_agent(call, position, &action_path)?,
Action::Code(call) => self.validate_code(call, position, &action_path)?,
}
Ok(())
}
fn validate_requirement(
&self,
requirement: &'a Requirement,
position: Position<'a>,
path: &str,
) -> Result<(), WyrdError> {
let path = format!("{path}.check");
match &requirement.check {
Check::Assert {
path: one,
each,
operator,
..
} => {
let (reference, wildcard) = match (one, each) {
(Some(reference), None) => (reference, false),
(None, Some(reference)) => (reference, true),
_ => {
return Err(invalid(
"assert_selector",
&path,
"set exactly one of path or each",
));
}
};
self.validate_reference(reference, position, wildcard, &path)?;
validate_operator(operator, &path)
}
Check::Trace(trace) => {
self.validate_reference(&trace.on, position, false, &path)?;
if !matches!(self.whole_slot(&trace.on), Some(InputSlot::Trace { .. })) {
return Err(invalid(
"trace_source",
&path,
"on must be one whole trace slot",
));
}
validate_operator(&trace.operator, &path)
}
Check::Agent(call) => self.validate_agent(call, position, &path),
Check::Code(call) => self.validate_code(call, position, &path),
Check::Drift(drift) => self.validate_drift(drift, &path),
}
}
fn validate_drift(&self, drift: &DriftCheck, path: &str) -> Result<(), WyrdError> {
let Some(InputSlot::Samples { features, .. }) = self.whole_slot(&drift.on) else {
return Err(invalid(
"drift_source",
path,
"on must be one whole samples slot",
));
};
drift.spec.validate()?;
let names: Vec<&str> = match &drift.spec.signal {
DriftSignal::Distribution { features, .. } => {
features.iter().map(FeatureName::as_str).collect()
}
DriftSignal::Metric { name } => vec![name.as_str()],
};
if let Some(missing) = names
.into_iter()
.find(|name| !features.iter().any(|feature| feature.as_str() == *name))
{
return Err(invalid(
"drift_features",
path,
format!("{missing} is not a measurement of the samples slot"),
));
}
Ok(())
}
fn validate_code(
&self,
call: &CodeCall,
position: Position<'a>,
path: &str,
) -> Result<(), WyrdError> {
match &call.source {
CodeSource::Inline(source) if source.trim().is_empty() => {
return Err(invalid("code_source", path, "inline source is empty"));
}
CodeSource::Artifact { card, .. }
if card
.as_card_ref()
.is_some_and(|card| card.kind != CardKind::Artifact) =>
{
return Err(invalid(
"code_source",
path,
"source must reference an Artifact Card",
));
}
_ => {}
}
if !is_valid_parameter_name(&call.entrypoint) {
return Err(invalid(
"code_entrypoint",
path,
"entrypoint must be an identifier",
));
}
for reference in call.inputs.values() {
self.validate_reference(reference, position, false, path)?;
}
Ok(())
}
fn validate_agent(
&self,
call: &'a AgentCall,
position: Position<'a>,
path: &str,
) -> Result<(), WyrdError> {
if call
.agent
.as_card_ref()
.is_some_and(|card| card.kind != CardKind::Agent)
{
return Err(invalid(
"agent_kind",
path,
"agent must reference an Agent Card",
));
}
for reference in call.inputs.values().chain(call.workspace.values()) {
self.validate_reference(reference, position, false, path)?;
}
let in_step = matches!(position, Position::Step(_));
if !in_step && !call.workspace.is_empty() {
return Err(invalid(
"check_agent_workspace",
path,
"check Agents have no workspace",
));
}
if let Some(bad) = call.workspace.keys().find(|key| !valid_workspace_path(key)) {
return Err(invalid(
"workspace_path",
path,
format!("{bad} must be /inputs/<name>[/<name>...] without . or .. segments"),
));
}
let Some(resolved) = (self.resolve)(&call.agent) else {
return Ok(());
};
self.validate_placeholders(call, resolved.prompt, path)?;
let tools = &resolved.agent.tool_names;
if in_step {
let mut builtins = Vec::with_capacity(tools.len());
for name in tools {
let Some(tool) = BuiltinTool::from_name(name) else {
return Err(invalid(
"agent_tools",
path,
format!("{name} is not a built-in tool"),
));
};
builtins.push(tool);
}
let fs_tool = builtins.iter().any(|tool| tool.is_filesystem());
if fs_tool && call.workspace.is_empty() {
return Err(invalid(
"agent_tools",
path,
"filesystem tools require a workspace",
));
}
let workspace_tool = fs_tool || builtins.contains(&BuiltinTool::CodeRun);
if !call.workspace.is_empty() && !workspace_tool {
return Err(invalid(
"agent_tools",
path,
"a workspace requires a filesystem or code.run tool",
));
}
} else {
if tools
.iter()
.any(|name| name != BuiltinTool::BifrostQuery.as_str())
{
return Err(invalid(
"check_agent_tools",
path,
"check Agents may declare only the bifrost.query tool",
));
}
let produces_verdict = match &resolved.prompt.response_type {
ResponseType::JsonSchema { schema, .. } => fits(schema, &verdict_schema()),
ResponseType::Text => false,
};
if !produces_verdict {
return Err(invalid(
"check_agent_schema",
path,
"check Agent response schema must produce a Verdict: outcome passed, failed, or inconclusive; a string reason; and citations of slot and locator",
));
}
}
Ok(())
}
fn validate_placeholders(
&self,
call: &AgentCall,
prompt: &Prompt,
path: &str,
) -> Result<(), WyrdError> {
let expected: BTreeSet<&str> = prompt
.variables
.iter()
.chain(&prompt.media_variables)
.map(String::as_str)
.collect();
let actual: BTreeSet<&str> = call.inputs.keys().map(ParameterName::as_str).collect();
if expected != actual {
return Err(WyrdError::Validation {
message: format!("{path}: inputs must match the Prompt placeholders"),
details: json!({
"rule": "agent_inputs", "path": path,
"expected": expected, "actual": actual,
}),
});
}
for (name, reference) in &call.inputs {
let media_slot = match self.whole_slot(reference) {
Some(InputSlot::Media { formats, .. }) => Some(formats),
_ => None,
};
let is_media_placeholder = prompt.media_variables.iter().any(|m| m == name.as_str());
match (is_media_placeholder, media_slot) {
(true, Some(formats)) => {
if let Some(format) = formats.iter().find(|f| f.native_kind().is_none()) {
return Err(invalid(
"native_media",
path,
format!(
"{format} is not provider-native media; read it through a step workspace or supply a PDF or image rendition"
),
));
}
if let Some(block) = prompt.media_kind(name.as_str())
&& let Some(format) =
formats.iter().find(|f| f.native_kind() != Some(block))
{
return Err(invalid(
"native_media",
path,
format!(
"{format} cannot fill the {block:?} block bound to media placeholder {name}"
),
));
}
}
(true, None) => {
return Err(invalid(
"native_media",
path,
format!("media placeholder {name} must bind one whole media slot"),
));
}
(false, Some(_)) => {
return Err(invalid(
"media_as_text",
path,
format!("text placeholder {name} cannot bind a media slot"),
));
}
(false, None) => {}
}
}
Ok(())
}
fn validate_reference(
&self,
reference: &Reference,
position: Position<'a>,
wildcard: bool,
path: &str,
) -> Result<(), WyrdError> {
let refuse =
|rule, message: &str| Err(invalid(rule, path, format!("{reference}: {message}")));
if reference.has_wildcard() && !wildcard {
return refuse(
"wildcard_position",
"[*] is allowed only in an assertion's each",
);
}
match &reference.root {
ReferenceRoot::Input(slot) => {
let Some(input) = self.spec.inputs.get(slot) else {
return refuse("unknown_reference", "unknown input slot");
};
if let InputSlot::Json {
schema: Some(schema),
..
} = input
&& !admits(schema, schema, &reference.path)
{
return refuse("missing_field", "the slot schema has no such field");
}
Ok(())
}
ReferenceRoot::RequirementId | ReferenceRoot::RequirementStatement => {
if matches!(position, Position::Step(_)) {
return refuse(
"requirement_position",
"requirement.* is readable only in checks",
);
}
Ok(())
}
ReferenceRoot::StepOutput(step)
| ReferenceRoot::StepStatus(step)
| ReferenceRoot::StepVerdict { step, .. } => {
let Some(target) = self.spec.steps.iter().find(|s| &s.id == step) else {
return refuse("unknown_reference", "unknown step");
};
if let ReferenceRoot::StepVerdict { requirement, .. } = &reference.root {
if self.owners.get(requirement) != Some(&step) {
return refuse("unknown_reference", "the step has no such requirement");
}
if !matches!(position, Position::Step(_)) {
return refuse(
"verdict_position",
"checks cannot read requirement verdicts",
);
}
}
if let Position::StepCheck(own) = position {
let upstream = self
.ancestors
.get(own)
.is_some_and(|set| set.contains(step));
if step != own && !upstream {
return refuse(
"step_position",
"a step requirement reads only its own step and upstream steps",
);
}
}
if matches!(reference.root, ReferenceRoot::StepOutput(_))
&& let Some(schema) = self.output_schema(target)
&& !admits(&schema, &schema, &reference.path)
{
return refuse("missing_field", "the step output has no such field");
}
Ok(())
}
}
}
fn whole_slot(&self, reference: &Reference) -> Option<&'a InputSlot> {
match &reference.root {
ReferenceRoot::Input(slot) if reference.path.is_empty() => self.spec.inputs.get(slot),
_ => None,
}
}
fn output_schema(&self, step: &'a Step) -> Option<Value> {
match &step.action {
Action::Extract { from } => match self.whole_slot(from)? {
InputSlot::Media { formats, .. } => Some(ExtractOutput::schema_for(formats)),
_ => None,
},
Action::Agent(call) => match &(self.resolve)(&call.agent)?.prompt.response_type {
ResponseType::JsonSchema { schema, .. } => Some(schema.clone()),
ResponseType::Text => Some(json!({ "type": "string" })),
},
Action::Code(_) => None,
}
}
}
fn verdict_schema() -> Value {
serde_json::to_value(crate::schema::schema_for::<Verdict>())
.expect("a generated JSON Schema serializes to JSON")
}
fn validate_slot(slot: &SlotName, input: &InputSlot) -> Result<(), WyrdError> {
let path = format!("inputs.{slot}");
if slot.as_str() == "group_id" {
return Err(invalid(
"slot",
&path,
"group_id is reserved for the verify/submit group_id argument; rename the slot",
));
}
match input {
InputSlot::Media {
formats, max_bytes, ..
} => {
if formats.is_empty() {
return Err(invalid(
"slot",
&path,
"media slots accept at least one format",
));
}
if *max_bytes == Some(0) {
return Err(invalid("slot", &path, "max_bytes must be positive"));
}
}
InputSlot::Samples { features, .. } if features.is_empty() => {
return Err(invalid(
"slot",
&path,
"samples slots name at least one measurement",
));
}
InputSlot::Source(source) => {
source
.parse()
.map_err(|error| invalid("source", &path, error))?;
}
InputSlot::Json {
schema: Some(schema),
..
} if !schema.is_object() && !schema.is_boolean() => {
return Err(invalid(
"slot",
&path,
"schema must be a JSON Schema object",
));
}
_ => {}
}
Ok(())
}
fn validate_order(spec: &VerifierSpec, order: &OrderBy) -> Result<(), WyrdError> {
let refuse = |message| invalid("order_by", "order_by", message);
if !spec
.inputs
.values()
.any(|input| matches!(input, InputSlot::Source(_)))
{
return Err(refuse(
"order_by orders group source lists; declare a source slot",
));
}
let OrderBy::Input(order) = order else {
return Ok(());
};
let slot = order
.slot()
.ok_or_else(|| refuse("order_by must read inputs.<slot>"))?;
let Some(InputSlot::Source(source)) = spec.inputs.get(slot) else {
return Err(refuse("order_by must name a declared source slot"));
};
if !matches!(
source.parse().map(|source| source.field),
Ok(SourceField::Input(_))
) {
return Err(refuse("order_by must name a source slot reading an input"));
}
if order.has_wildcard() {
return Err(refuse("order_by selects one value, not [*]"));
}
Ok(())
}
fn validate_operator(
operator: &crate::verification::ComparisonOperator,
path: &str,
) -> Result<(), WyrdError> {
operator
.validate()
.map_err(|error| invalid("operator", path, error))
}
fn valid_workspace_path(key: &str) -> bool {
key.strip_prefix("/inputs/").is_some_and(|rest| {
rest.split('/')
.all(|segment| !segment.is_empty() && segment != "." && segment != "..")
})
}
fn ancestors<'a>(
spec: &'a VerifierSpec,
plan: &ExecutionPlan,
) -> BTreeMap<&'a TaskId, BTreeSet<&'a TaskId>> {
let dependencies = spec.dependencies();
let mut ancestors: BTreeMap<&TaskId, BTreeSet<&TaskId>> = BTreeMap::new();
for id in plan.stages.iter().flat_map(|stage| &stage.tasks) {
let Some((&id, deps)) = dependencies.get_key_value(id) else {
continue;
};
let mut all = deps.clone();
for dep in deps {
if let Some(upstream) = ancestors.get(dep) {
all.extend(upstream.iter().copied());
}
}
ancestors.insert(id, all);
}
ancestors
}
#[cfg(test)]
mod tests {
use serde_json::{Value, json};
use super::super::VerifierSpec;
use crate::error::WyrdError;
fn parse(body: &Value) -> VerifierSpec {
serde_json::from_value(body.clone()).expect("authored Verifier deserializes")
}
fn rule(body: &Value) -> String {
match parse(body).validate() {
Err(WyrdError::Validation { details, .. }) => details["rule"]
.as_str()
.expect("refusal names its rule")
.to_owned(),
other => panic!("expected a validation refusal for {body}, got {other:?}"),
}
}
fn with(mut body: Value, pointer: &str, value: Value) -> Value {
let (parent, key) = pointer.rsplit_once('/').expect("pointer has a parent");
match body.pointer_mut(parent).expect(pointer) {
Value::Object(fields) => {
fields.insert(key.to_owned(), value);
}
target => {
*target
.get_mut(key.parse::<usize>().expect("array index"))
.expect(pointer) = value;
}
}
body
}
fn agent(tools: &[&str], content: Value, response_format: Option<Value>) -> Value {
let mut prompt = json!({
"provider": "openai",
"model": "gpt-4o",
"messages": [{"role": "user", "content": content}],
});
if let Some(schema) = response_format {
prompt["response_format"] = schema;
}
let mut agent = json!({
"prompt": prompt,
"execution_route": {"kind": "wyrd_gateway", "deployment": "openai-primary"},
});
if !tools.is_empty() {
agent["tool_names"] = json!(tools);
}
agent
}
fn verdict_schema() -> Value {
let locator = |key: &str, value: Value| json!({"type": "object", "properties": {key: value}, "required": [key], "additionalProperties": false});
json!({
"type": "object",
"properties": {
"outcome": {"type": "string", "enum": ["passed", "failed", "inconclusive"]},
"reason": {"type": "string"},
"citations": {"type": "array", "items": {
"type": "object",
"properties": {
"slot": {"type": "string"},
"locator": {"anyOf": [
locator("page", json!({"type": "integer"})),
locator("lines", json!({"type": "object", "properties": {"start": {"type": "integer"}, "end": {"type": "integer"}}, "required": ["start", "end"], "additionalProperties": false})),
locator("json_pointer", json!({"type": "string"}))
]}
},
"required": ["slot", "locator"],
"additionalProperties": false
}}
},
"required": ["outcome", "reason", "citations"],
"additionalProperties": false
})
}
fn judge(schema: Value) -> Value {
agent(
&[],
json!([{"type": "document", "source": "${media:policy}"}, {"type": "text", "text": "{{claim}} {{answer}} {{findings}}"}]),
Some(schema),
)
}
fn review() -> Value {
json!({
"inputs": {
"answer": {"type": "json", "schema": {"type": "object", "properties": {"text": {"type": "string"}}, "additionalProperties": false}},
"policy": {"type": "media", "formats": ["pdf"]},
"deck": {"type": "media", "formats": ["pptx"], "required": false},
"trace": {"type": "trace"}
},
"steps": [
{"id": "structure", "action": {"kind": "extract", "from": "inputs.deck"},
"requirements": [{"id": "titles", "statement": "Every slide has a title",
"check": {"kind": "assert", "each": "steps.structure.output.slides[*].title", "operator": "is_not_null"}}]},
{"id": "findings",
"when": {"path": "steps.structure.requirements.titles.outcome", "operator": "equals", "expected": "passed"},
"action": {"kind": "agent",
"agent": agent(&["fs.read"], json!("Review {{outline}}"), Some(json!({"type": "object", "properties": {"score": {"type": "integer"}}, "required": ["score"], "additionalProperties": false}))),
"inputs": {"outline": "steps.structure.output"},
"workspace": {"/inputs/policy.pdf": "inputs.policy"}},
"timeout_seconds": 60, "retries": 2,
"requirements": [{"id": "score", "statement": "Score at least 4",
"check": {"kind": "assert", "path": "steps.findings.output.score", "operator": "greater_than_or_equals", "expected": 4}}]},
{"id": "count", "depends_on": ["findings"],
"action": {"kind": "code", "entrypoint": "count", "source": {"inline": "def count(text):\n return len(text)"}, "inputs": {"text": "inputs.answer.text"}}}
],
"requirements": [
{"id": "no-guarantee", "statement": "No guaranteed refunds",
"check": {"kind": "assert", "path": "inputs.answer.text", "operator": "not_contains", "expected": "guaranteed refund"}},
{"id": "traced", "statement": "A retrieval span exists",
"check": {"kind": "trace", "on": "inputs.trace", "span_selector": "$.spans[0].name", "operator": "equals", "expected": "retrieve"}},
{"id": "judged", "statement": "The answer follows the policy",
"check": {"kind": "agent",
"agent": agent(&[], json!([{"type": "document", "source": "${media:policy}"}, {"type": "text", "text": "{{claim}} {{answer}} {{findings}}"}]), Some(verdict_schema())),
"inputs": {"claim": "requirement.statement", "answer": "inputs.answer.text", "policy": "inputs.policy", "findings": "steps.findings.output"}}},
{"id": "counted", "statement": "Counting code agrees",
"check": {"kind": "code", "entrypoint": "check", "source": {"artifact": {"card": {"kind": "Artifact", "space": "support", "name": "checks", "version": "1.0.0"}, "path": "check.py"}},
"inputs": {"count": "steps.count.output", "statement": "requirement.statement"}}}
],
"decide": {"pass_rate": {"threshold": 0.75}}
})
}
fn drift() -> Value {
json!({
"inputs": {"current": {"type": "samples", "features": ["income", "dti"]}},
"requirements": [{"id": "stable", "statement": "Inputs are stable",
"check": {"kind": "drift", "on": "inputs.current", "method": "Spc",
"signal": {"kind": "Distribution", "baseline_ref": {"kind": "Data", "space": "risk", "name": "train", "version": "1.0.0"}, "features": ["income"]},
"condition": {"kind": "Statistical"},
"profile": {"kind": "Spc", "sample_size": 5}}}]
})
}
#[test]
fn verifier_spec_round_trips_and_validates() {
for body in [
review(),
drift(),
json!({}),
json!({"inputs": {"a": {"type": "json"}}}),
] {
let spec = parse(&body);
spec.validate().expect("valid Verifier");
assert_eq!(serde_json::to_value(&spec).expect("serialize"), body);
}
let spec = parse(&review());
assert_eq!(
spec.inputs.values().filter(|slot| slot.required()).count(),
3
);
let plan = spec.plan().expect("plan");
let stages: Vec<Vec<&str>> = plan
.stages
.iter()
.map(|stage| {
stage
.tasks
.iter()
.map(crate::verification::TaskId::as_str)
.collect()
})
.collect();
assert_eq!(
stages,
vec![vec!["structure"], vec!["findings"], vec!["count"]]
);
let explicit =
json!({"inputs": {"a": {"type": "json", "required": true}}, "decide": "all"});
assert_eq!(
serde_json::to_value(parse(&explicit)).expect("serialize"),
json!({"inputs": {"a": {"type": "json"}}})
);
}
#[test]
fn verifier_spec_pass_rate_bounds() {
for (threshold, valid) in [
(1.0, true),
(0.01, true),
(0.0, false),
(1.5, false),
(-0.1, false),
] {
let body = json!({"decide": {"pass_rate": {"threshold": threshold}}});
assert_eq!(parse(&body).validate().is_ok(), valid, "{threshold}");
}
}
#[test]
fn verifier_spec_reserves_group_id_slot() {
for slot in [json!({"type": "json"}), json!({"type": "trace"})] {
let body = json!({"inputs": {"group_id": slot}});
match parse(&body).validate() {
Err(WyrdError::Validation { details, message }) => {
assert_eq!(details["rule"], "slot", "{body}");
assert_eq!(details["path"], "inputs.group_id", "{body}");
assert!(message.contains("group_id"), "{message}");
}
other => panic!("expected a slot refusal for {body}, got {other:?}"),
}
}
assert!(
parse(&json!({"inputs": {"group": {"type": "json"}}}))
.validate()
.is_ok()
);
}
#[test]
fn verifier_spec_declares_source_slots() {
let body = json!({"inputs": {
"turns": "verifiers.turn-verifier.inputs.turn",
"checks": "verifiers.turn-verifier.requirements.on-topic.outcome",
}});
let spec = parse(&body);
spec.validate().expect("source slots validate");
assert_eq!(
serde_json::to_value(&spec).expect("serializes")["inputs"],
body["inputs"]
);
for source in [
"verifiers.turn-verifier.inputs.turn.text",
"verifiers.turn-verifier.inputs.turn[0]",
"verifiers.turn-verifier.inputs.turn[*]",
"verifiers.turn-verifier.requirements.on-topic.reason",
"steps.a.output",
] {
let body = json!({"inputs": {"turns": source}});
match parse(&body).validate() {
Err(WyrdError::Validation { details, .. }) => {
assert_eq!(details["rule"], "source", "{source}");
assert_eq!(details["path"], "inputs.turns", "{source}");
}
other => panic!("expected a source refusal for {source}, got {other:?}"),
}
}
assert_eq!(
rule(&json!({"inputs": {"group_id": "verifiers.turn-verifier.inputs.turn"}})),
"slot"
);
}
#[test]
fn verifier_spec_orders_source_lists() {
let body = |order: &str| {
json!({
"inputs": {
"turns": "verifiers.turn-verifier.inputs.turn",
"checks": "verifiers.turn-verifier.requirements.on-topic.outcome",
"plain": {"type": "json"},
},
"order_by": order,
})
};
for order in [
"recorded_at",
"inputs.turns",
"inputs.turns.turn",
"inputs.turns.at[0]",
] {
let spec = parse(&body(order));
spec.validate().expect("order validates");
assert_eq!(
serde_json::to_value(&spec).expect("serializes")["order_by"],
order
);
}
for order in [
"inputs.checks",
"inputs.plain",
"inputs.missing",
"inputs.turns[*].turn",
"steps.a.output",
] {
assert_eq!(
parse(&body(order))
.validate()
.map_err(|error| match error {
WyrdError::Validation { details, .. } => details,
other => panic!("expected a validation refusal, got {other:?}"),
})
.expect_err(order),
json!({"rule": "order_by", "path": "order_by"}),
"{order}"
);
}
assert!(serde_json::from_value::<VerifierSpec>(body("turns")).is_err());
assert_eq!(
rule(&json!({"inputs": {"plain": {"type": "json"}}, "order_by": "recorded_at"})),
"order_by"
);
}
#[test]
fn verifier_spec_refuses_unknown_fields() {
for body in [
json!({"implementation": {}}),
json!({"inputs": {"a": {"type": "json", "extra": 1}}}),
json!({"inputs": {"a": {"type": "blob"}}}),
json!({"requirements": [{"id": "a", "statement": "s", "check": {"kind": "assert", "path": "inputs.a", "operator": "equals", "expected": 1, "extra": 1}}]}),
json!({"inputs": {"s": {"type": "samples", "features": ["x"]}}, "requirements": [{"id": "a", "statement": "s", "check": {"kind": "drift", "on": "inputs.s", "method": "Spc", "signal": {"kind": "Metric", "name": "x"}, "condition": {"kind": "Statistical"}, "extra": 1}}]}),
json!({"steps": [{"id": "a", "action": {"kind": "fetch"}}]}),
json!({"decide": {"majority": {}}}),
] {
assert!(
serde_json::from_value::<VerifierSpec>(body.clone()).is_err(),
"{body}"
);
}
}
#[test]
fn verifier_spec_refuses_invalid_graphs_and_references() {
let with = |pointer: &str, value: Value| with(review(), pointer, value);
let cases = [
(with("/requirements/0/id", json!("titles")), "duplicate_id"),
(with("/steps/2/id", json!("structure")), "duplicate_id"),
(
with("/requirements/0/check/path", json!("inputs.missing")),
"unknown_reference",
),
(
with("/requirements/0/check/path", json!("inputs.answer.missing")),
"missing_field",
),
(
with(
"/requirements/0/check/path",
json!("steps.findings.output.nope"),
),
"missing_field",
),
(
with(
"/steps/0/requirements/0/check/each",
json!("steps.structure.output.slides[*].heading"),
),
"missing_field",
),
(
with("/requirements/0/check/path", json!("steps.ghost.output")),
"unknown_reference",
),
(
with(
"/requirements/0/check/path",
json!("steps.findings.requirements.titles.outcome"),
),
"unknown_reference",
),
(
with(
"/requirements/0/check/path",
json!("steps.structure.requirements.titles.outcome"),
),
"verdict_position",
),
(
with("/requirements/0/check/path", json!("inputs.answer[*]")),
"wildcard_position",
),
(
with(
"/steps/0/requirements/0/check/each",
json!("steps.count.output"),
),
"step_position",
),
(with("/steps/2/depends_on", json!(["count"])), "graph"),
(with("/steps/0/depends_on", json!(["count"])), "graph"),
(
with("/steps/2/depends_on", json!(["ghost"])),
"unknown_reference",
),
(
with(
"/steps/2/action/inputs/text",
json!("requirement.statement"),
),
"requirement_position",
),
(
with("/requirements/0/check/each", json!("inputs.answer.text")),
"assert_selector",
),
(
with(
"/requirements/0/check/operator",
json!({"kind": "matches_regex", "pattern": "("}),
),
"operator",
),
(
with(
"/inputs/trace",
json!({"type": "samples", "features": ["x"]}),
),
"mixed_evidence",
),
(with("/inputs/policy/formats", json!([])), "slot"),
(with("/inputs/policy/max_bytes", json!(0)), "slot"),
(with("/steps/1/timeout_seconds", json!(0)), "step_budget"),
(with("/steps/1/retries", json!(6)), "step_budget"),
(
with("/steps/0/action/from", json!("inputs.policy.x")),
"extract_source",
),
(
with("/inputs/deck/formats", json!(["ppt"])),
"extract_format",
),
(
with("/inputs/deck/formats", json!(["zip"])),
"extract_format",
),
(
with("/requirements/1/check/on", json!("inputs.answer")),
"trace_source",
),
(
with("/steps/2/action/entrypoint", json!("not valid")),
"code_entrypoint",
),
(
with("/steps/2/action/source/inline", json!(" ")),
"code_source",
),
(
with(
"/requirements/3/check/source/artifact/card/kind",
json!("Data"),
),
"code_source",
),
(
with(
"/steps/1/action/workspace",
json!({"/inputs/../etc": "inputs.policy"}),
),
"workspace_path",
),
];
for (body, expected) in cases {
assert_eq!(rule(&body), expected, "{body}");
}
let mut drift = drift();
drift["requirements"][0]["check"]["signal"]["features"] = json!(["fico"]);
assert_eq!(rule(&drift), "drift_features");
let mut mixed = drift.clone();
mixed["requirements"][0]["check"]["signal"]["features"] = json!(["income"]);
mixed["requirements"]
.as_array_mut()
.expect("requirements")
.push(json!({"id": "x", "statement": "s", "check": {"kind": "assert", "path": "inputs.current", "operator": "is_not_null"}}));
assert_eq!(rule(&mixed), "mixed_evidence");
let mut stepped = self::drift();
stepped["steps"] = json!([{"id": "a", "action": {"kind": "code", "entrypoint": "f", "source": {"inline": "def f(): pass"}}}]);
assert_eq!(rule(&stepped), "window_steps");
let mut bad = self::drift();
bad["requirements"][0]["check"]["on"] = json!("inputs.current.income");
assert_eq!(rule(&bad), "drift_source");
}
#[test]
fn verifier_spec_enforces_agent_positions_and_media() {
let with = |pointer: &str, value: Value| with(review(), pointer, value);
let text_judge = agent(
&[],
json!("{{claim}} {{answer}} {{policy}} {{findings}}"),
Some(verdict_schema()),
);
let cases = [
(
with("/requirements/2/check/inputs/extra", json!("inputs.answer")),
"agent_inputs",
),
(with("/steps/1/action/inputs", json!({})), "agent_inputs"),
(
with(
"/requirements/2/check/inputs/policy",
json!("inputs.answer.text"),
),
"native_media",
),
(
with("/inputs/policy/formats", json!(["pdf", "docx"])),
"native_media",
),
(
with(
"/requirements/2/check/agent",
agent(
&[],
json!([{"type": "image", "source": "${media:policy}"}, {"type": "text", "text": "{{claim}} {{answer}} {{findings}}"}]),
Some(verdict_schema()),
),
),
"native_media",
),
(
with("/requirements/2/check/agent", text_judge),
"media_as_text",
),
(
with(
"/requirements/2/check/agent",
agent(
&["fs.read"],
json!([{"type": "document", "source": "${media:policy}"}, {"type": "text", "text": "{{claim}} {{answer}} {{findings}}"}]),
Some(verdict_schema()),
),
),
"check_agent_tools",
),
(
with(
"/requirements/2/check/agent",
agent(
&[],
json!([{"type": "document", "source": "${media:policy}"}, {"type": "text", "text": "{{claim}} {{answer}} {{findings}}"}]),
None,
),
),
"check_agent_schema",
),
(
with(
"/requirements/2/check/workspace",
json!({"/inputs/p.pdf": "inputs.policy"}),
),
"check_agent_workspace",
),
(
with("/steps/1/action/agent/tool_names", json!(["http.fetch"])),
"agent_tools",
),
(with("/steps/1/action/workspace", json!({})), "agent_tools"),
(
with("/steps/1/action/agent/tool_names", json!(["cards.get"])),
"agent_tools",
),
(
with(
"/steps/1/action/agent",
json!({"kind": "Prompt", "space": "support", "name": "answer-prompt", "version": "1.0.0"}),
),
"agent_kind",
),
];
for (body, expected) in cases {
assert_eq!(rule(&body), expected, "{body}");
}
let registered = with(
"/requirements/2/check/agent",
json!({"kind": "Agent", "space": "support", "name": "judge", "version": "1.0.0"}),
);
parse(®istered)
.validate()
.expect("an unresolved registered Agent is validated by its resolver");
let live_query = with(
"/requirements/2/check/agent",
agent(
&["bifrost.query"],
json!([{"type": "document", "source": "${media:policy}"}, {"type": "text", "text": "{{claim}} {{answer}} {{findings}}"}]),
Some(verdict_schema()),
),
);
parse(&live_query)
.validate()
.expect("a check Agent may query Bifrost live");
}
#[test]
fn verifier_spec_admits_every_builtin_step_tool() {
use crate::card::agent::BuiltinTool;
for tool in BuiltinTool::ALL {
let mut body = with(
review(),
"/steps/1/action/agent/tool_names",
json!([tool.as_str()]),
);
if !tool.is_filesystem() && tool != BuiltinTool::CodeRun {
body = with(body, "/steps/1/action/workspace", json!({}));
}
parse(&body)
.validate()
.unwrap_or_else(|error| panic!("{}: {error:?}", tool.as_str()));
}
for name in ["cards", "bifrost.*", "fs.read/x", "FS.READ", "http.fetch"] {
let body = with(review(), "/steps/1/action/agent/tool_names", json!([name]));
assert_eq!(rule(&body), "agent_tools", "{name}");
}
}
#[test]
fn verifier_spec_requires_check_agents_to_produce_a_verdict() {
let schema = |pointer: &str, value: Value| with(verdict_schema(), pointer, value);
let mut variant = verdict_schema();
variant["definitions"] =
json!({"Citation": verdict_schema()["properties"]["citations"]["items"].clone()});
variant["properties"]["citations"]["items"] = json!({"$ref": "#/definitions/Citation"});
variant["properties"]["outcome"] =
json!({"description": "verdict", "type": "string", "enum": ["failed", "passed"]});
variant["properties"]["reason"] = json!({"type": "string", "maxLength": 500});
variant["description"] = json!("A judgment");
let compatible = [
verdict_schema(),
variant,
super::verdict_schema(),
schema(
"/properties/reason",
json!({"type": "string", "allOf": [{"maxLength": 500}]}),
),
schema("/properties/reason", json!({"const": "fixed"})),
schema("/properties/reason", json!({"enum": ["short", "long"]})),
schema(
"/properties/outcome",
json!({"type": "string", "enum": ["passed", "failed"], "allOf": [{"minLength": 1}]}),
),
schema(
"/patternProperties",
json!({"^reason$": {"maxLength": 500}}),
),
];
for compatible in compatible {
let body = with(
review(),
"/requirements/2/check/agent",
judge(compatible.clone()),
);
parse(&body)
.validate()
.unwrap_or_else(|error| panic!("{compatible}: {error:?}"));
}
let incompatible = [
schema("/properties/reason", json!({"const": 7})),
schema("/properties/reason", json!({"enum": ["short", 1]})),
schema("/patternProperties", json!({"^score$": {"type": "number"}})),
schema("/properties/outcome", json!({"type": "integer"})),
schema("/properties/outcome", json!({"type": "string"})),
schema(
"/properties/outcome",
json!({"type": "string", "const": "yes"}),
),
schema(
"/properties/outcome",
json!({"type": "string", "enum": ["passed", "yes"]}),
),
schema("/properties/reason", json!({"type": "boolean"})),
schema("/properties/citations", json!({"type": "string"})),
schema("/properties/citations", json!({"type": "array"})),
schema("/properties/citations/items/required", json!(["slot"])),
schema(
"/properties/citations/items/properties/locator",
json!({"type": "object", "properties": {"page": {"type": "string"}}, "required": ["page"], "additionalProperties": false}),
),
schema(
"/properties/citations/items/properties/locator",
json!({"type": "object", "properties": {"chapter": {"type": "integer"}}, "required": ["chapter"], "additionalProperties": false}),
),
schema("/properties/score", json!({"type": "integer"})),
schema(
"/required",
json!(["outcome", "reason", "citations", "score"]),
),
schema("/required", json!(["outcome", "reason"])),
schema("/additionalProperties", json!(true)),
schema(
"/properties/outcome",
json!({"oneOf": [{"$ref": "#/properties/outcome"}]}),
),
];
for declared in &incompatible {
let body = with(
review(),
"/requirements/2/check/agent",
judge(declared.clone()),
);
assert_eq!(rule(&body), "check_agent_schema", "{declared}");
}
let mut stepped = review();
stepped["steps"][1]["requirements"]
.as_array_mut()
.expect("step requirements")
.push(json!({"id": "grounded", "statement": "Findings are grounded",
"check": {"kind": "agent", "agent": agent(&[], json!("{{findings}}"), Some(verdict_schema())),
"inputs": {"findings": "steps.findings.output"}}}));
parse(&stepped).validate().expect("a Verdict step check");
stepped["steps"][1]["requirements"][1]["check"]["agent"] =
agent(&[], json!("{{findings}}"), Some(incompatible[0].clone()));
assert_eq!(rule(&stepped), "check_agent_schema");
let card =
json!({"kind": "Agent", "space": "support", "name": "judge", "version": "1.0.0"});
let registered = with(review(), "/requirements/2/check/agent", card);
for (declared, valid) in [(verdict_schema(), true), (incompatible[0].clone(), false)] {
let resolved: crate::card::agent::AgentSpec =
serde_json::from_value(judge(declared)).expect("Agent");
let prompt = resolved.prompt.as_inline().expect("inline Prompt");
let spec = parse(®istered);
let outcome = spec.validate_with(|agent| {
Some(match agent.as_inline() {
Some(inline) => super::ResolvedAgent {
agent: inline,
prompt: inline.prompt.as_inline()?,
},
None => super::ResolvedAgent {
agent: &resolved,
prompt,
},
})
});
assert_eq!(outcome.is_ok(), valid, "{outcome:?}");
}
}
#[test]
fn verifier_spec_bounds_cyclic_schema_references() {
let cyclic = [
json!({"type": "object", "properties": {"x": {"oneOf": [{"$ref": "#/properties/x"}]}}}),
json!({"type": "object", "definitions": {"a": {"anyOf": [{"$ref": "#/definitions/b"}]}, "b": {"anyOf": [{"$ref": "#/definitions/a"}]}}, "properties": {"x": {"$ref": "#/definitions/a"}}}),
];
for schema in &cyclic {
let input = with(review(), "/inputs/answer/schema", schema.clone());
let input = with(
input,
"/requirements/0/check/path",
json!("inputs.answer.x.y"),
);
parse(&input)
.validate()
.unwrap_or_else(|error| panic!("{schema}: {error:?}"));
let output = with(
review(),
"/steps/1/action/agent",
agent(
&["fs.read"],
json!("Review {{outline}}"),
Some(schema.clone()),
),
);
let output = with(
output,
"/steps/1/requirements/0/check/path",
json!("steps.findings.output.x.y"),
);
parse(&output)
.validate()
.unwrap_or_else(|error| panic!("{schema}: {error:?}"));
}
let tree = json!({
"definitions": {"node": {"type": "object", "properties": {"child": {"$ref": "#/definitions/node"}, "text": {"type": "string"}}, "additionalProperties": false}},
"$ref": "#/definitions/node"
});
let body = with(review(), "/inputs/answer/schema", tree);
let path =
|selector: &str| with(body.clone(), "/requirements/0/check/path", json!(selector));
parse(&path("inputs.answer.child.child.text"))
.validate()
.expect("finite recursion");
assert_eq!(
rule(&path("inputs.answer.child.child.nope")),
"missing_field"
);
let literal = json!({
"type": "object",
"properties": {"text": {"type": "string"}},
"patternProperties": {"^score$": {"type": "object", "properties": {"value": {"type": "number"}}, "additionalProperties": false}},
"additionalProperties": false
});
let body = with(review(), "/inputs/answer/schema", literal.clone());
let path =
|selector: &str| with(body.clone(), "/requirements/0/check/path", json!(selector));
for selector in ["inputs.answer.score", "inputs.answer.score.value"] {
parse(&path(selector))
.validate()
.unwrap_or_else(|error| panic!("{selector}: {error:?}"));
}
for selector in ["inputs.answer.points", "inputs.answer.score.nope"] {
assert_eq!(rule(&path(selector)), "missing_field", "{selector}");
}
let open = with(
body.clone(),
"/inputs/answer/schema/patternProperties/^p.*$",
json!({"type": "number"}),
);
parse(&with(
open,
"/requirements/0/check/path",
json!("inputs.answer.points"),
))
.validate()
.expect("a non-literal pattern leaves the field unknown");
let step = |agent: Value, selector: &str| {
let body = with(review(), "/steps/1/action/agent", agent);
with(body, "/steps/1/requirements/0/check/path", json!(selector))
};
let inline = |selector: &str| {
step(
agent(
&["fs.read"],
json!("Review {{outline}}"),
Some(literal.clone()),
),
selector,
)
};
parse(&inline("steps.findings.output.score.value"))
.validate()
.expect("an inline Agent's literal-pattern output field");
assert_eq!(
rule(&inline("steps.findings.output.points")),
"missing_field"
);
let resolved: crate::card::agent::AgentSpec = serde_json::from_value(agent(
&["fs.read"],
json!("Review {{outline}}"),
Some(literal),
))
.expect("Agent");
let prompt = resolved.prompt.as_inline().expect("inline Prompt");
let card =
json!({"kind": "Agent", "space": "support", "name": "reviewer", "version": "1.0.0"});
for (selector, valid) in [
("steps.findings.output.score.value", true),
("steps.findings.output.points", false),
] {
let spec = parse(&step(card.clone(), selector));
let outcome = spec.validate_with(|agent| {
Some(match agent.as_inline() {
Some(inline) => super::ResolvedAgent {
agent: inline,
prompt: inline.prompt.as_inline()?,
},
None => super::ResolvedAgent {
agent: &resolved,
prompt,
},
})
});
assert_eq!(outcome.is_ok(), valid, "{selector}: {outcome:?}");
}
}
}