use std::collections::BTreeMap;
use schemars::JsonSchema;
use serde::de::Error as _;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
use skald_spec::MediaFormat;
use crate::card::agent::AgentSpec;
use crate::card::drift::DriftSpec;
use crate::card::operator::OperatorSpec;
use crate::card::prompt::ParameterName;
use crate::card::trigger::TriggerSpec;
use crate::ids::{BindingId, FeatureName, SlotName};
use crate::reference::{CardRef, InlineableRef, Ref};
use crate::verification::ContextCapture;
use crate::verification::VerificationError;
use crate::verification::{ComparisonOperator, JsonPath, TaskId};
pub mod extract;
pub mod reference;
mod schema_path;
mod validate;
pub use reference::{
OrderBy, PathSegment, Reference, ReferenceError, ReferenceRoot, SourceField, SourceReference,
SourceText, VerdictField,
};
pub use validate::ResolvedAgent;
pub const MAX_STEP_RETRIES: u32 = 5;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(deny_unknown_fields)]
pub struct VerifierSpec {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
#[schemars(with = "BTreeMap<SlotName, InputSlotSchema>")]
#[cfg_attr(feature = "server", schema(value_type = BTreeMap<String, InputSlotSchema>))]
pub inputs: BTreeMap<SlotName, InputSlot>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub steps: Vec<Step>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub requirements: Vec<Requirement>,
#[serde(default, skip_serializing_if = "Decide::is_all")]
pub decide: Decide,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[cfg_attr(feature = "server", schema(value_type = Option<Object>))]
pub context_capture: Option<ContextCapture>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub order_by: Option<OrderBy>,
}
impl VerifierSpec {
#[must_use]
pub fn agent_calls(&self) -> Vec<(String, &AgentCall)> {
fn checks<'a>(
requirements: &'a [Requirement],
prefix: &str,
calls: &mut Vec<(String, &'a AgentCall)>,
) {
for (i, requirement) in requirements.iter().enumerate() {
if let Check::Agent(call) = &requirement.check {
calls.push((format!("{prefix}[{i}].check"), call));
}
}
}
let mut calls = Vec::new();
for (i, step) in self.steps.iter().enumerate() {
if let Action::Agent(call) = &step.action {
calls.push((format!("spec.steps[{i}].action"), call));
}
checks(
&step.requirements,
&format!("spec.steps[{i}].requirements"),
&mut calls,
);
}
checks(&self.requirements, "spec.requirements", &mut calls);
calls
}
pub fn all_requirements(&self) -> impl Iterator<Item = &Requirement> {
self.steps
.iter()
.flat_map(|step| &step.requirements)
.chain(&self.requirements)
}
pub fn drift_checks(&self) -> impl Iterator<Item = (&TaskId, &DriftCheck)> {
self.all_requirements()
.filter_map(|requirement| match &requirement.check {
Check::Drift(drift) => Some((&requirement.id, drift)),
_ => None,
})
}
}
enum InputSlotSchema {}
impl JsonSchema for InputSlotSchema {
fn schema_name() -> String {
"InputSlot".to_owned()
}
fn json_schema(generator: &mut schemars::r#gen::SchemaGenerator) -> schemars::schema::Schema {
use schemars::schema::{Metadata, SchemaObject, SubschemaValidation};
SchemaObject {
metadata: Some(Box::new(Metadata {
description: Some(
"One named evidence slot. Slots are required unless `required: false`."
.to_owned(),
),
..Metadata::default()
})),
subschemas: Some(Box::new(SubschemaValidation {
one_of: Some(vec![
generator.subschema_for::<InputSlot>(),
generator.subschema_for::<SourceText>(),
]),
..SubschemaValidation::default()
})),
..SchemaObject::default()
}
.into()
}
}
#[cfg(feature = "server")]
impl utoipa::PartialSchema for InputSlotSchema {
fn schema() -> utoipa::openapi::RefOr<utoipa::openapi::schema::Schema> {
use utoipa::openapi::RefOr;
use utoipa::openapi::schema::Schema;
let mut slot = <InputSlot as utoipa::PartialSchema>::schema();
if let RefOr::T(Schema::OneOf(one_of)) = &mut slot {
one_of
.items
.retain(|item| !matches!(item, RefOr::T(Schema::AllOf(_))));
one_of
.items
.push(<SourceText as utoipa::PartialSchema>::schema());
}
slot
}
}
#[cfg(feature = "server")]
impl utoipa::ToSchema for InputSlotSchema {
fn name() -> std::borrow::Cow<'static, str> {
"InputSlot".into()
}
fn schemas(
schemas: &mut Vec<(
String,
utoipa::openapi::RefOr<utoipa::openapi::schema::Schema>,
)>,
) {
<InputSlot as utoipa::ToSchema>::schemas(schemas);
}
}
#[derive(Debug, Clone, PartialEq, Serialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
#[schemars(rename = "TypedInputSlot")]
pub enum InputSlot {
Json {
#[serde(default, skip_serializing_if = "Option::is_none")]
schema: Option<Value>,
#[serde(default = "required_default", skip_serializing_if = "is_true")]
required: bool,
},
Media {
#[cfg_attr(feature = "server", schema(value_type = Vec<String>))]
formats: Vec<MediaFormat>,
#[serde(default, skip_serializing_if = "Option::is_none")]
max_bytes: Option<u64>,
#[serde(default = "required_default", skip_serializing_if = "is_true")]
required: bool,
},
Trace {
#[serde(default = "required_default", skip_serializing_if = "is_true")]
required: bool,
},
Samples {
features: Vec<FeatureName>,
},
#[serde(untagged)]
#[schemars(skip)]
Source(SourceText),
}
#[derive(Deserialize)]
#[serde(
remote = "InputSlot",
tag = "type",
rename_all = "snake_case",
deny_unknown_fields
)]
enum TypedInputSlot {
Json {
#[serde(default)]
schema: Option<Value>,
#[serde(default = "required_default")]
required: bool,
},
Media {
formats: Vec<MediaFormat>,
#[serde(default)]
max_bytes: Option<u64>,
#[serde(default = "required_default")]
required: bool,
},
Trace {
#[serde(default = "required_default")]
required: bool,
},
Samples {
features: Vec<FeatureName>,
},
}
impl<'de> Deserialize<'de> for InputSlot {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct Slot;
impl<'de> serde::de::Visitor<'de> for Slot {
type Value = InputSlot;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("a typed input slot or a source reference string")
}
fn visit_str<E: serde::de::Error>(self, text: &str) -> Result<InputSlot, E> {
SourceText::deserialize(serde::de::value::StrDeserializer::new(text))
.map(InputSlot::Source)
}
fn visit_map<A: serde::de::MapAccess<'de>>(
self,
map: A,
) -> Result<InputSlot, A::Error> {
TypedInputSlot::deserialize(serde::de::value::MapAccessDeserializer::new(map))
}
}
deserializer.deserialize_any(Slot)
}
}
impl InputSlot {
#[must_use]
pub fn required(&self) -> bool {
match self {
Self::Json { required, .. }
| Self::Media { required, .. }
| Self::Trace { required } => *required,
Self::Samples { .. } | Self::Source(_) => true,
}
}
}
fn required_default() -> bool {
true
}
#[expect(
clippy::trivially_copy_pass_by_ref,
reason = "serde skip_serializing_if passes the field by reference"
)]
fn is_true(value: &bool) -> bool {
*value
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(deny_unknown_fields)]
pub struct Step {
pub id: TaskId,
pub action: Action,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub depends_on: Vec<TaskId>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub when: Option<Condition>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub requirements: Vec<Requirement>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub timeout_seconds: Option<u32>,
#[serde(default, skip_serializing_if = "is_zero")]
pub retries: u32,
}
#[expect(
clippy::trivially_copy_pass_by_ref,
reason = "serde skip_serializing_if passes the field by reference"
)]
fn is_zero(value: &u32) -> bool {
*value == 0
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum Action {
Extract {
from: Reference,
},
Agent(AgentCall),
Code(CodeCall),
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(deny_unknown_fields)]
pub struct AgentCall {
pub agent: InlineableRef<AgentSpec>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub inputs: BTreeMap<ParameterName, Reference>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub workspace: BTreeMap<String, Reference>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(deny_unknown_fields)]
pub struct CodeCall {
#[serde(default, skip_serializing_if = "CodeRuntime::is_default")]
pub runtime: CodeRuntime,
pub source: CodeSource,
pub entrypoint: String,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub inputs: BTreeMap<ParameterName, Reference>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(rename_all = "snake_case")]
pub enum CodeRuntime {
#[default]
Python,
}
impl CodeRuntime {
#[must_use]
pub fn is_default(&self) -> bool {
*self == Self::default()
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub enum CodeSource {
Inline(String),
Artifact {
card: Ref,
#[serde(default, skip_serializing_if = "Option::is_none")]
path: Option<String>,
},
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(deny_unknown_fields)]
pub struct Requirement {
pub id: TaskId,
pub statement: String,
pub check: Check,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum Check {
Assert {
#[serde(default, skip_serializing_if = "Option::is_none")]
path: Option<Reference>,
#[serde(default, skip_serializing_if = "Option::is_none")]
each: Option<Reference>,
operator: ComparisonOperator,
#[serde(default, skip_serializing_if = "Value::is_null")]
expected: Value,
},
Trace(TraceCheck),
Agent(AgentCall),
Code(CodeCall),
Drift(DriftCheck),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CheckEvidence {
Run,
Window,
}
impl Check {
#[must_use]
pub fn evidence(&self) -> CheckEvidence {
match self {
Self::Drift(_) => CheckEvidence::Window,
Self::Assert { .. } | Self::Trace(_) | Self::Agent(_) | Self::Code(_) => {
CheckEvidence::Run
}
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(deny_unknown_fields)]
pub struct TraceCheck {
pub on: Reference,
pub span_selector: JsonPath,
pub operator: ComparisonOperator,
#[serde(default, skip_serializing_if = "Value::is_null")]
pub expected: Value,
}
#[derive(Debug, Clone, PartialEq, JsonSchema)]
pub struct DriftCheck {
pub on: Reference,
#[schemars(flatten)]
pub spec: DriftSpec,
}
#[cfg(feature = "server")]
impl utoipa::PartialSchema for DriftCheck {
fn schema() -> utoipa::openapi::RefOr<utoipa::openapi::schema::Schema> {
use utoipa::openapi::schema::{AllOfBuilder, ObjectBuilder};
AllOfBuilder::new()
.item(utoipa::openapi::Ref::from_schema_name("DriftSpec"))
.item(
ObjectBuilder::new()
.property("on", <Reference as utoipa::PartialSchema>::schema())
.required("on"),
)
.into()
}
}
#[cfg(feature = "server")]
impl utoipa::ToSchema for DriftCheck {
fn schemas(
schemas: &mut Vec<(
String,
utoipa::openapi::RefOr<utoipa::openapi::schema::Schema>,
)>,
) {
schemas.push((
<DriftSpec as utoipa::ToSchema>::name().into(),
<DriftSpec as utoipa::PartialSchema>::schema(),
));
<DriftSpec as utoipa::ToSchema>::schemas(schemas);
}
}
impl Serialize for DriftCheck {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut value = serde_json::to_value(&self.spec).map_err(serde::ser::Error::custom)?;
if let Value::Object(fields) = &mut value {
fields.insert("on".to_owned(), Value::String(self.on.to_string()));
}
value.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for DriftCheck {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let mut value = Value::deserialize(deserializer)?;
let on = value
.as_object_mut()
.and_then(|fields| fields.remove("on"))
.ok_or_else(|| D::Error::missing_field("on"))?;
Ok(Self {
on: Reference::deserialize(on).map_err(D::Error::custom)?,
spec: DriftSpec::deserialize(value).map_err(D::Error::custom)?,
})
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(deny_unknown_fields)]
pub struct Condition {
pub path: Reference,
pub operator: ComparisonOperator,
#[serde(default, skip_serializing_if = "Value::is_null")]
pub expected: Value,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub enum Decide {
#[default]
All,
PassRate {
threshold: f64,
},
}
impl Decide {
#[must_use]
pub fn is_all(&self) -> bool {
matches!(self, Self::All)
}
}
pub const OWNER_OCCURRENCE_KEY: &str = "$owner";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
pub struct VerificationStatus {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub binding_ids: Vec<BindingId>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub baseline: Option<DriftBaselineStatus>,
}
impl VerificationStatus {
#[must_use]
pub fn derived(
binding_ids: Vec<BindingId>,
baseline: Option<DriftBaselineStatus>,
) -> Option<Self> {
(!binding_ids.is_empty() || baseline.is_some()).then_some(Self {
binding_ids,
baseline,
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(rename_all = "snake_case")]
pub enum DriftBaselineState {
Pending,
Building,
Ready,
Failed,
}
impl DriftBaselineState {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Pending => "pending",
Self::Building => "building",
Self::Ready => "ready",
Self::Failed => "failed",
}
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
match value {
"pending" => Some(Self::Pending),
"building" => Some(Self::Building),
"ready" => Some(Self::Ready),
"failed" => Some(Self::Failed),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
pub struct DriftBaselineStatus {
pub state: DriftBaselineState,
pub data: CardRef,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error: Option<VerificationError>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
#[cfg_attr(feature = "server", derive(utoipa::ToSchema))]
#[serde(deny_unknown_fields)]
pub struct VerificationBinding {
pub verifier: Ref,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub runs_on: Option<InlineableRef<TriggerSpec>>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub on_failure: Vec<InlineableRef<OperatorSpec>>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub on_verdict_change: Vec<InlineableRef<OperatorSpec>>,
}
impl VerificationBinding {
#[must_use]
pub fn operator_lists(&self) -> [(&'static str, &[InlineableRef<OperatorSpec>]); 2] {
[
("on_failure", &self.on_failure),
("on_verdict_change", &self.on_verdict_change),
]
}
pub fn operator_lists_mut(
&mut self,
) -> [(&'static str, &mut Vec<InlineableRef<OperatorSpec>>); 2] {
[
("on_failure", &mut self.on_failure),
("on_verdict_change", &mut self.on_verdict_change),
]
}
}
#[cfg(test)]
mod tests {
use serde_json::{Value, json};
use super::{InputSlot, VerifierSpec};
#[test]
fn input_slot_schema_matches_decoding() {
let schema = serde_json::to_value(schemars::schema_for!(VerifierSpec))
.expect("invariant: a schema serializes");
let definitions = &schema["definitions"];
assert_eq!(
definitions["InputSlot"]["oneOf"],
json!([
{"$ref": "#/definitions/TypedInputSlot"},
{"$ref": "#/definitions/SourceText"}
])
);
let tags: Vec<&Value> = definitions["TypedInputSlot"]["oneOf"]
.as_array()
.expect("invariant: typed slots are a oneOf")
.iter()
.map(|variant| &variant["properties"]["type"]["enum"][0])
.collect();
assert_eq!(
tags,
[
&json!("json"),
&json!("media"),
&json!("trace"),
&json!("samples")
]
);
assert_eq!(definitions["SourceText"]["type"], json!("string"));
let decode = |value| serde_json::from_value::<InputSlot>(value);
assert!(matches!(
decode(json!("verifiers.turn.inputs.message")),
Ok(InputSlot::Source(_))
));
assert!(decode(json!({"type": "trace"})).is_ok());
assert!(decode(json!({"type": "source"})).is_err());
}
#[cfg(feature = "server")]
#[test]
fn served_input_slot_schema_matches_decoding() {
let schema =
serde_json::to_value(<super::InputSlotSchema as utoipa::PartialSchema>::schema())
.expect("invariant: a schema serializes");
let forms: Vec<Value> = schema["oneOf"]
.as_array()
.expect("invariant: slots are a oneOf")
.iter()
.map(|form| form["properties"]["type"]["enum"][0].clone())
.collect();
assert_eq!(
forms,
[
json!("json"),
json!("media"),
json!("trace"),
json!("samples"),
Value::Null
]
);
assert_eq!(schema["oneOf"][4]["type"], json!("string"));
}
}