use crate::consistency::ConsistencyGate;
use crate::decision::DecisionGate;
use crate::judge::JudgeGate;
use crate::provider::{Auth, ModelRequest, ModelResponse, ProviderRegistry};
use crate::subprocess::SubprocessGate;
use async_trait::async_trait;
use firstpass_core::{AbstainPolicy, GateDef, GateResult, Verdict, cost::PriceTable};
use std::collections::VecDeque;
use std::sync::Mutex;
use std::time::Duration;
#[async_trait]
pub trait Gate: Send + Sync + std::fmt::Debug {
fn id(&self) -> &str;
async fn evaluate(&self, req: &ModelRequest, resp: &ModelResponse) -> GateResult;
fn abstain_fails_closed(&self) -> bool {
false
}
}
fn request_has_declared_tools(req: &ModelRequest) -> bool {
req.tools.as_array().is_some_and(|a| !a.is_empty())
}
#[derive(Debug, Clone, Copy)]
pub struct NonEmptyGate;
#[async_trait]
impl Gate for NonEmptyGate {
fn id(&self) -> &str {
"non-empty"
}
async fn evaluate(&self, req: &ModelRequest, resp: &ModelResponse) -> GateResult {
let verdict = if !resp.text.trim().is_empty() {
Verdict::Pass
} else if resp.has_tool_calls() && request_has_declared_tools(req) {
Verdict::Pass
} else {
Verdict::Fail
};
GateResult::deterministic(self.id(), verdict, 0)
}
}
#[derive(Debug, Clone, Copy)]
pub struct JsonValidGate;
#[async_trait]
impl Gate for JsonValidGate {
fn id(&self) -> &str {
"json-valid"
}
async fn evaluate(&self, req: &ModelRequest, resp: &ModelResponse) -> GateResult {
let has_tools = request_has_declared_tools(req);
if resp.has_tool_calls() && has_tools {
return GateResult::deterministic(self.id(), Verdict::Pass, 0);
}
let ok = serde_json::from_str::<serde_json::Value>(resp.text.trim()).is_ok();
GateResult::deterministic(self.id(), if ok { Verdict::Pass } else { Verdict::Fail }, 0)
}
}
fn schema_violation(
value: &serde_json::Value,
schema: &serde_json::Value,
path: &str,
) -> Option<String> {
use serde_json::Value;
fn type_matches(v: &Value, ty: &str) -> bool {
match ty {
"object" => v.is_object(),
"array" => v.is_array(),
"string" => v.is_string(),
"integer" => v.is_i64() || v.is_u64() || v.as_f64().is_some_and(|f| f.fract() == 0.0),
"number" => v.is_number(),
"boolean" => v.is_boolean(),
"null" => v.is_null(),
_ => true, }
}
if let Some(type_kw) = schema.get("type") {
let ok = match type_kw {
Value::String(ty) => type_matches(value, ty),
Value::Array(types) => types
.iter()
.filter_map(Value::as_str)
.any(|ty| type_matches(value, ty)),
_ => true,
};
if !ok {
return Some(format!("{path} has wrong type"));
}
}
if let Some(Value::Array(variants)) = schema.get("enum")
&& !variants.contains(value)
{
return Some(format!("{path} is not one of the allowed enum values"));
}
if let Some(obj) = value.as_object() {
if let Some(Value::Array(req)) = schema.get("required") {
for field in req.iter().filter_map(Value::as_str) {
if !obj.contains_key(field) {
return Some(format!("{path}.{field:?} is required but missing"));
}
}
}
let ap_false = schema.get("additionalProperties").and_then(Value::as_bool) == Some(false);
let defined_props: std::collections::HashSet<&str> = schema
.get("properties")
.and_then(Value::as_object)
.map(|p| p.keys().map(String::as_str).collect())
.unwrap_or_default();
if ap_false {
for key in obj.keys() {
if !defined_props.contains(key.as_str()) {
return Some(format!(
"{path}.{key:?} is an additional property not allowed by the schema"
));
}
}
} else if let Some(extra_schema) =
schema.get("additionalProperties").filter(|v| v.is_object())
{
for (key, child) in obj {
if !defined_props.contains(key.as_str())
&& let Some(v) = schema_violation(child, extra_schema, &format!("{path}.{key}"))
{
return Some(v);
}
}
}
if let Some(props) = schema.get("properties").and_then(Value::as_object) {
for (name, subschema) in props {
if let Some(child) = obj.get(name) {
let child_path = format!("{path}.{name}");
if let Some(v) = schema_violation(child, subschema, &child_path) {
return Some(v);
}
}
}
}
}
if let (Some(arr), Some(items_schema)) = (value.as_array(), schema.get("items")) {
for (i, elem) in arr.iter().enumerate() {
let child_path = format!("{path}[{i}]");
if let Some(v) = schema_violation(elem, items_schema, &child_path) {
return Some(v);
}
}
}
None
}
#[derive(Debug, Clone)]
pub struct SchemaGate {
id: String,
schema: serde_json::Value,
}
impl SchemaGate {
#[must_use]
pub fn new(id: impl Into<String>, schema: serde_json::Value) -> Self {
Self {
id: id.into(),
schema,
}
}
pub(crate) fn violation(&self, value: &serde_json::Value) -> Option<String> {
schema_violation(value, &self.schema, "root")
}
}
#[async_trait]
impl Gate for SchemaGate {
fn id(&self) -> &str {
&self.id
}
async fn evaluate(&self, req: &ModelRequest, resp: &ModelResponse) -> GateResult {
let has_tools = request_has_declared_tools(req);
if resp.has_tool_calls() && has_tools {
return GateResult::deterministic(self.id(), Verdict::Pass, 0);
}
let Ok(value) = serde_json::from_str::<serde_json::Value>(resp.text.trim()) else {
let mut r = GateResult::deterministic(self.id(), Verdict::Fail, 0);
r.reason = Some("candidate is not valid JSON".to_owned());
return r;
};
match self.violation(&value) {
None => GateResult::deterministic(self.id(), Verdict::Pass, 0),
Some(reason) => {
let mut r = GateResult::deterministic(self.id(), Verdict::Fail, 0);
r.reason = Some(reason);
r
}
}
}
}
fn extract_response_tool_calls(raw: &serde_json::Value) -> Vec<(String, String)> {
let mut out = Vec::new();
if let Some(blocks) = raw.get("content").and_then(serde_json::Value::as_array) {
for b in blocks {
if b.get("type").and_then(serde_json::Value::as_str) == Some("tool_use") {
let name = b
.get("name")
.and_then(serde_json::Value::as_str)
.unwrap_or("")
.to_owned();
let args = b
.get("input")
.map_or_else(String::new, std::string::ToString::to_string);
out.push((name, args));
}
}
return out;
}
if let Some(calls) = raw
.pointer("/choices/0/message/tool_calls")
.and_then(serde_json::Value::as_array)
{
for call in calls {
let (name, args) = if let Some(func) = call.get("function") {
(
func.get("name")
.and_then(serde_json::Value::as_str)
.unwrap_or("")
.to_owned(),
func.get("arguments")
.and_then(serde_json::Value::as_str)
.unwrap_or("")
.to_owned(),
)
} else {
(String::new(), String::new())
};
out.push((name, args));
}
}
out
}
fn validate_tool_args_against_tools(
name: &str,
args: &serde_json::Value,
tools: &serde_json::Value,
) -> Option<String> {
let arr = tools.as_array()?;
for tool in arr {
if let (Some(n), Some(schema)) = (
tool.get("name").and_then(serde_json::Value::as_str),
tool.get("input_schema"),
) && n == name
{
return schema_violation(args, schema, "args");
}
if let (Some(n), Some(schema)) = (
tool.get("name").and_then(serde_json::Value::as_str),
tool.get("parameters"),
) && n == name
{
return schema_violation(args, schema, "args");
}
if let Some(func) = tool.get("function")
&& let (Some(n), Some(schema)) = (
func.get("name").and_then(serde_json::Value::as_str),
func.get("parameters"),
)
&& n == name
{
return schema_violation(args, schema, "args");
}
}
None }
fn tool_name_is_known(name: &str, tools: &serde_json::Value) -> bool {
let Some(arr) = tools.as_array() else {
return false;
};
for tool in arr {
if tool.get("name").and_then(serde_json::Value::as_str) == Some(name) {
return true;
}
if tool
.pointer("/function/name")
.and_then(serde_json::Value::as_str)
== Some(name)
{
return true;
}
}
false
}
#[derive(Debug, Clone, Copy)]
pub struct ToolArgsValidGate;
#[async_trait]
impl Gate for ToolArgsValidGate {
fn id(&self) -> &str {
"tool-args-valid"
}
async fn evaluate(&self, req: &ModelRequest, resp: &ModelResponse) -> GateResult {
let calls = extract_response_tool_calls(&resp.raw);
if calls.is_empty() {
return GateResult::deterministic(self.id(), Verdict::Pass, 0);
}
let has_declared_tools = req.tools.as_array().is_some_and(|a| !a.is_empty());
if !has_declared_tools {
let mut r = GateResult::deterministic(self.id(), Verdict::Fail, 0);
r.reason = Some(
"response contains tool calls but request declared no tools (hallucinated)"
.to_owned(),
);
return r;
}
for (name, args_str) in &calls {
if !tool_name_is_known(name, &req.tools) {
let mut r = GateResult::deterministic(self.id(), Verdict::Fail, 0);
r.reason = Some(format!(
"tool call {name:?}: tool not declared in request (hallucinated)"
));
return r;
}
let args = match serde_json::from_str::<serde_json::Value>(args_str) {
Ok(v) => v,
Err(_) => {
let mut r = GateResult::deterministic(self.id(), Verdict::Fail, 0);
r.reason = Some(format!("tool call {name:?}: arguments is not valid JSON"));
return r;
}
};
if let Some(violation) = validate_tool_args_against_tools(name, &args, &req.tools) {
let mut r = GateResult::deterministic(self.id(), Verdict::Fail, 0);
r.reason = Some(format!("tool call {name:?}: {violation}"));
return r;
}
}
GateResult::deterministic(self.id(), Verdict::Pass, 0)
}
}
#[derive(Debug)]
pub struct GateHealth {
window: usize,
max_error_rate: f64,
outcomes: Mutex<VecDeque<bool>>, disabled: std::sync::atomic::AtomicBool,
}
impl GateHealth {
#[must_use]
pub fn new(window: usize, max_error_rate: f64) -> Self {
Self {
window: window.max(1),
max_error_rate,
outcomes: Mutex::new(VecDeque::new()),
disabled: std::sync::atomic::AtomicBool::new(false),
}
}
#[must_use]
pub fn is_enabled(&self) -> bool {
!self.disabled.load(std::sync::atomic::Ordering::Relaxed)
}
pub fn record(&self, gate_id: &str, errored: bool) {
let Ok(mut q) = self.outcomes.lock() else {
return; };
q.push_back(errored);
while q.len() > self.window {
q.pop_front();
}
if q.len() == self.window {
let errors = q.iter().filter(|e| **e).count();
let rate = errors as f64 / self.window as f64;
if rate > self.max_error_rate && self.is_enabled() {
self.disabled
.store(true, std::sync::atomic::Ordering::Relaxed);
tracing::error!(
gate = %gate_id,
error_rate = rate,
"gate exceeded its error budget — auto-disabled (ALARM)"
);
}
}
}
}
#[derive(Debug, Default)]
pub struct GateHealthRegistry {
budgets: std::collections::HashMap<String, (usize, f64)>,
state: Mutex<std::collections::HashMap<(String, String), GateHealth>>,
}
impl GateHealthRegistry {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn with_budget(
mut self,
gate_id: impl Into<String>,
window: usize,
max_error_rate: f64,
) -> Self {
self.budgets
.insert(gate_id.into(), (window, max_error_rate));
self
}
#[must_use]
pub fn enabled(&self, tenant: &str, gate_id: &str) -> bool {
let Some(&(window, max_error_rate)) = self.budgets.get(gate_id) else {
return true;
};
let Ok(mut state) = self.state.lock() else {
return true;
};
state
.entry((tenant.to_owned(), gate_id.to_owned()))
.or_insert_with(|| GateHealth::new(window, max_error_rate))
.is_enabled()
}
pub fn record(&self, tenant: &str, gate_id: &str, errored: bool) {
let Some(&(window, max_error_rate)) = self.budgets.get(gate_id) else {
return;
};
let Ok(mut state) = self.state.lock() else {
return;
};
state
.entry((tenant.to_owned(), gate_id.to_owned()))
.or_insert_with(|| GateHealth::new(window, max_error_rate))
.record(gate_id, errored);
}
}
#[derive(Debug)]
struct FailClosed(Box<dyn Gate>);
#[async_trait]
impl Gate for FailClosed {
fn id(&self) -> &str {
self.0.id()
}
async fn evaluate(&self, req: &ModelRequest, resp: &ModelResponse) -> GateResult {
self.0.evaluate(req, resp).await
}
fn abstain_fails_closed(&self) -> bool {
true
}
}
#[must_use]
pub fn resolve_gates(
names: &[String],
defs: &[GateDef],
registry: &ProviderRegistry,
auth: &Auth,
prices: &PriceTable,
http: &reqwest::Client,
) -> Vec<Box<dyn Gate>> {
let mut gates: Vec<Box<dyn Gate>> = Vec::new();
let mut push = |gate: Box<dyn Gate>, def: Option<&GateDef>| {
if def.is_some_and(|d| d.on_abstain == AbstainPolicy::FailClosed) {
gates.push(Box::new(FailClosed(gate)));
} else {
gates.push(gate);
}
};
for name in names {
match name.as_str() {
"non-empty" => push(Box::new(NonEmptyGate), None),
"json-valid" => push(Box::new(JsonValidGate), None),
"tool-args-valid" => push(Box::new(ToolArgsValidGate), None),
other => match defs.iter().find(|d| d.id == other) {
Some(def) if def.judge.is_some() => {
if let Some(judge) = def.judge.as_ref() {
let provider_id = judge.model.split('/').next().unwrap_or_default();
match registry.get(provider_id) {
Some(provider) => push(
Box::new(JudgeGate::new(
def.id.clone(),
provider,
judge.model.clone(),
auth.clone(),
judge.threshold,
judge.rubric.clone().unwrap_or_default(),
prices.clone(),
)),
Some(def),
),
None => tracing::warn!(
gate = %other, provider = %provider_id,
"judge gate provider not registered — skipped"
),
}
}
}
Some(def) if def.consistency.is_some() => {
if let Some(cons) = def.consistency.as_ref() {
let provider_id = cons.model.split('/').next().unwrap_or_default();
match registry.get(provider_id) {
Some(provider) => push(
Box::new(ConsistencyGate::new(
def.id.clone(),
provider,
cons.model.clone(),
auth.clone(),
cons.k,
cons.threshold,
prices.clone(),
)),
Some(def),
),
None => tracing::warn!(
gate = %other, provider = %provider_id,
"consistency gate provider not registered — skipped"
),
}
}
}
Some(def) if def.schema.is_some() => {
if let Some(schema) = def.schema.as_ref() {
push(
Box::new(SchemaGate::new(def.id.clone(), schema.clone())),
Some(def),
);
}
}
Some(def) if def.decision.is_some() => {
if let Some(decision) = def.decision.as_ref() {
match std::env::var(&decision.api_key_env) {
Ok(api_key) => push(
Box::new(DecisionGate::new(
def.id.clone(),
http.clone(),
decision,
api_key,
)),
Some(def),
),
Err(_) => tracing::warn!(
gate = %other, env_var = %decision.api_key_env,
"decision gate API key env var is unset — skipped"
),
}
}
}
Some(def) => {
let Some((program, args)) = def.cmd.split_first() else {
tracing::warn!(gate = %other, "configured gate has empty cmd — skipped");
continue;
};
push(
Box::new(SubprocessGate::new(
def.id.clone(),
program.clone(),
args.to_vec(),
Duration::from_millis(def.timeout_ms),
)),
Some(def),
);
}
None => tracing::warn!(
gate = %other,
"unknown gate id — not a built-in and not defined in [[gate]]; skipped"
),
},
}
}
gates
}
#[must_use]
pub fn aggregate_with_policy(
results: &[GateResult],
fail_closed_ids: &std::collections::HashSet<&str>,
) -> Verdict {
let blocking = |r: &GateResult| {
r.verdict == Verdict::Fail
|| (r.verdict == Verdict::Abstain && fail_closed_ids.contains(r.gate_id.as_str()))
};
if results.iter().any(blocking) {
Verdict::Fail
} else {
Verdict::Pass
}
}
#[must_use]
pub fn aggregate(results: &[GateResult]) -> Verdict {
aggregate_with_policy(results, &std::collections::HashSet::new())
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::{Value, json};
fn resp(text: &str) -> ModelResponse {
ModelResponse {
model: "anthropic/claude-haiku-4-5".to_owned(),
text: text.to_owned(),
in_tokens: 1,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 1,
raw: Value::Null,
}
}
fn req() -> ModelRequest {
ModelRequest {
model: "anthropic/claude-haiku-4-5".to_owned(),
system: None,
messages: vec![],
max_tokens: 16,
tools: Value::Null,
raw: Value::Null,
cache_prefix: false,
}
}
#[tokio::test]
async fn non_empty_gate() {
assert_eq!(
NonEmptyGate.evaluate(&req(), &resp("hi")).await.verdict,
Verdict::Pass
);
assert_eq!(
NonEmptyGate.evaluate(&req(), &resp(" ")).await.verdict,
Verdict::Fail
);
}
#[tokio::test]
async fn json_valid_gate() {
assert_eq!(
JsonValidGate
.evaluate(&req(), &resp(r#"{"ok":true}"#))
.await
.verdict,
Verdict::Pass
);
assert_eq!(
JsonValidGate.evaluate(&req(), &resp("nope")).await.verdict,
Verdict::Fail
);
}
#[tokio::test]
async fn schema_gate_type_and_required() {
let g = SchemaGate::new(
"schema",
json!({
"type": "object",
"required": ["name", "age"],
"properties": { "name": {"type": "string"}, "age": {"type": "integer"} }
}),
);
assert_eq!(
g.evaluate(&req(), &resp(r#"{"name":"a","age":3}"#))
.await
.verdict,
Verdict::Pass
);
assert_eq!(
g.evaluate(&req(), &resp(r#"{"name":"a"}"#)).await.verdict,
Verdict::Fail
);
assert_eq!(
g.evaluate(&req(), &resp(r#"{"name":"a","age":"x"}"#))
.await
.verdict,
Verdict::Fail
);
assert_eq!(g.evaluate(&req(), &resp("[]")).await.verdict, Verdict::Fail);
assert_eq!(
g.evaluate(&req(), &resp("plain text")).await.verdict,
Verdict::Fail
);
}
fn empty_registry() -> ProviderRegistry {
ProviderRegistry::new("http://localhost", "http://localhost")
}
#[test]
fn resolve_skips_unknown_and_keeps_known() {
let gates = resolve_gates(
&[
"non-empty".to_owned(),
"judge-diff".to_owned(),
"json-valid".to_owned(),
],
&[],
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
let ids: Vec<_> = gates.iter().map(|g| g.id()).collect();
assert_eq!(ids, ["non-empty", "json-valid"]);
}
#[test]
fn resolve_builds_configured_subprocess_gate() {
let defs = vec![GateDef {
id: "my-tests".to_owned(),
cmd: vec!["true".to_owned()],
timeout_ms: 1000,
judge: None,
consistency: None,
schema: None,
decision: None,
on_abstain: AbstainPolicy::FailOpen,
}];
let gates = resolve_gates(
&["my-tests".to_owned(), "undefined".to_owned()],
&defs,
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
let ids: Vec<_> = gates.iter().map(|g| g.id()).collect();
assert_eq!(
ids,
["my-tests"],
"configured id resolves; unknown id skipped"
);
}
#[tokio::test]
async fn configured_subprocess_gate_runs_end_to_end() {
let script = r#"c=$(cat); case "$c" in *BAD*) echo '{"verdict":"fail"}';; *) echo '{"verdict":"pass"}';; esac"#;
let defs = vec![GateDef {
id: "no-bad".to_owned(),
cmd: vec!["bash".to_owned(), "-c".to_owned(), script.to_owned()],
timeout_ms: 5000,
judge: None,
consistency: None,
schema: None,
decision: None,
on_abstain: AbstainPolicy::FailOpen,
}];
let gates = resolve_gates(
&["no-bad".to_owned()],
&defs,
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert_eq!(gates.len(), 1);
let good = gates[0].evaluate(&req(), &resp("all good")).await;
assert_eq!(good.verdict, Verdict::Pass);
let bad = gates[0].evaluate(&req(), &resp("this is BAD")).await;
assert_eq!(bad.verdict, Verdict::Fail);
}
#[test]
fn resolve_builds_configured_judge_gate() {
use crate::provider::{MockProvider, Provider};
use std::collections::HashMap;
use std::sync::Arc;
let defs = vec![GateDef {
id: "quality".to_owned(),
cmd: vec![],
timeout_ms: 30_000,
judge: Some(firstpass_core::JudgeDef {
model: "anthropic/judge".to_owned(),
threshold: 0.7,
rubric: None,
}),
consistency: None,
schema: None,
decision: None,
on_abstain: AbstainPolicy::FailOpen,
}];
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", HashMap::new())),
);
let gates = resolve_gates(
&["quality".to_owned()],
&defs,
&ProviderRegistry::from_map(map),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert_eq!(
gates.iter().map(|g| g.id()).collect::<Vec<_>>(),
["quality"]
);
let skipped = resolve_gates(
&["quality".to_owned()],
&defs,
&ProviderRegistry::from_map(HashMap::new()),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert!(
skipped.is_empty(),
"judge with no registered provider is skipped"
);
}
#[test]
fn resolve_builds_configured_decision_gate() {
let def = |api_key_env: &str| GateDef {
id: "verify".to_owned(),
cmd: vec![],
timeout_ms: 30_000,
judge: None,
consistency: None,
schema: None,
decision: Some(firstpass_core::DecisionDef {
provider: "typesafe".to_owned(),
model: "jev-latest".to_owned(),
api_key_env: api_key_env.to_owned(),
base_url: "https://api.typesafe.ai".to_owned(),
timeout_ms: 1_000,
threshold: 0.5,
instructions: None,
}),
on_abstain: AbstainPolicy::FailOpen,
};
let gates = resolve_gates(
&["verify".to_owned()],
&[def("PATH")],
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert_eq!(
gates.iter().map(|g| g.id()).collect::<Vec<_>>(),
["verify"],
"decision gate resolves when its API key env var is set"
);
let gates = resolve_gates(
&["verify".to_owned()],
&[def("FIRSTPASS_TEST_DEFINITELY_UNSET_DECISION_KEY_XYZ")],
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert!(
gates.is_empty(),
"decision gate with an unset API key env var is skipped, not a hard failure"
);
}
#[test]
fn resolve_builds_configured_consistency_gate() {
use crate::provider::{MockProvider, Provider};
use std::collections::HashMap;
use std::sync::Arc;
let defs = vec![GateDef {
id: "uncertainty".to_owned(),
cmd: vec![],
timeout_ms: 30_000,
judge: None,
consistency: Some(firstpass_core::ConsistencyDef {
model: "anthropic/claude-haiku-4-5".to_owned(),
k: 3,
threshold: 0.6,
}),
schema: None,
decision: None,
on_abstain: AbstainPolicy::FailOpen,
}];
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", HashMap::new())),
);
let gates = resolve_gates(
&["uncertainty".to_owned()],
&defs,
&ProviderRegistry::from_map(map),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert_eq!(
gates.iter().map(|g| g.id()).collect::<Vec<_>>(),
["uncertainty"]
);
let skipped = resolve_gates(
&["uncertainty".to_owned()],
&defs,
&ProviderRegistry::from_map(HashMap::new()),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert!(
skipped.is_empty(),
"consistency gate with no registered provider is skipped"
);
}
#[test]
fn aggregate_semantics() {
let pass = GateResult::deterministic("a", Verdict::Pass, 0);
let fail = GateResult::deterministic("b", Verdict::Fail, 0);
let abstain = GateResult::abstain("c", "x", 0);
assert_eq!(aggregate(&[]), Verdict::Pass);
assert_eq!(aggregate(std::slice::from_ref(&pass)), Verdict::Pass);
assert_eq!(aggregate(&[pass.clone(), fail]), Verdict::Fail);
assert_eq!(aggregate(&[pass, abstain]), Verdict::Pass);
}
#[test]
fn error_budget_auto_disables_past_threshold() {
let h = GateHealth::new(10, 0.5); for _ in 0..5 {
h.record("g", true);
}
for _ in 0..5 {
h.record("g", false);
}
assert!(h.is_enabled(), "50% is at, not above, the budget");
for _ in 0..6 {
h.record("g", true);
}
assert!(
!h.is_enabled(),
"gate should auto-disable once error rate exceeds budget"
);
}
#[test]
fn healthy_gate_stays_enabled() {
let h = GateHealth::new(20, 0.5);
for _ in 0..100 {
h.record("g", false);
}
assert!(h.is_enabled());
}
#[test]
fn gate_health_registry_default_tenant_is_a_single_bucket() {
let registry = GateHealthRegistry::new().with_budget("g", 4, 0.5);
for _ in 0..4 {
registry.record("default", "g", true);
}
assert!(!registry.enabled("default", "g"));
}
#[test]
fn gate_health_registry_scopes_budget_per_tenant() {
let registry = GateHealthRegistry::new().with_budget("g", 4, 0.5);
for _ in 0..4 {
registry.record("tenant-a", "g", true);
}
assert!(!registry.enabled("tenant-a", "g"), "A should be disabled");
assert!(registry.enabled("tenant-b", "g"), "B must be unaffected");
}
#[test]
fn gate_health_registry_unregistered_gate_always_enabled() {
let registry = GateHealthRegistry::new();
assert!(registry.enabled("tenant-a", "unknown-gate"));
registry.record("tenant-a", "unknown-gate", true); assert!(registry.enabled("tenant-a", "unknown-gate"));
}
#[tokio::test]
async fn resolve_builds_configured_schema_gate() {
let defs = vec![GateDef {
id: "extract-shape".to_owned(),
cmd: vec![],
timeout_ms: 30_000,
judge: None,
consistency: None,
schema: Some(json!({"type": "object", "required": ["name"]})),
decision: None,
on_abstain: firstpass_core::AbstainPolicy::FailOpen,
}];
let gates = resolve_gates(
&["extract-shape".to_owned()],
&defs,
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert_eq!(
gates.iter().map(|g| g.id()).collect::<Vec<_>>(),
["extract-shape"],
"a [[gate]] def with `schema` resolves to a runnable SchemaGate"
);
let ok = gates[0].evaluate(&req(), &resp(r#"{"name":"a"}"#)).await;
assert_eq!(ok.verdict, Verdict::Pass);
let missing = gates[0].evaluate(&req(), &resp(r#"{}"#)).await;
assert_eq!(missing.verdict, Verdict::Fail);
}
#[test]
fn resolve_wraps_fail_closed_gate() {
let mk = |on_abstain| {
vec![GateDef {
id: "tests".to_owned(),
cmd: vec!["true".to_owned()],
timeout_ms: 1000,
judge: None,
consistency: None,
schema: None,
decision: None,
on_abstain,
}]
};
let open = resolve_gates(
&["tests".to_owned()],
&mk(firstpass_core::AbstainPolicy::FailOpen),
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert!(!open[0].abstain_fails_closed(), "default stays fail-open");
let closed = resolve_gates(
&["tests".to_owned()],
&mk(firstpass_core::AbstainPolicy::FailClosed),
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert!(
closed[0].abstain_fails_closed(),
"on_abstain = fail_closed wraps the gate"
);
assert_eq!(closed[0].id(), "tests", "wrapper preserves the id");
}
#[test]
fn aggregate_with_policy_fail_closed_abstain_blocks() {
use std::collections::HashSet;
let pass = GateResult::deterministic("a", Verdict::Pass, 0);
let abstain = GateResult::abstain("c", "timeout", 0);
assert_eq!(
aggregate_with_policy(&[pass.clone(), abstain.clone()], &HashSet::new()),
Verdict::Pass
);
let closed: HashSet<&str> = ["c"].into_iter().collect();
assert_eq!(
aggregate_with_policy(&[pass, abstain], &closed),
Verdict::Fail
);
}
fn tool_use_resp() -> ModelResponse {
ModelResponse {
model: "anthropic/claude-haiku-4-5".to_owned(),
text: String::new(), in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
raw: serde_json::json!({
"content": [{
"type": "tool_use",
"id": "t1",
"name": "search",
"input": { "q": "hello" }
}]
}),
}
}
fn tool_req_with_schema() -> ModelRequest {
ModelRequest {
model: "anthropic/claude-haiku-4-5".to_owned(),
system: None,
messages: vec![],
max_tokens: 64,
tools: serde_json::json!([{
"name": "search",
"input_schema": {
"type": "object",
"required": ["q"],
"properties": { "q": { "type": "string" } }
}
}]),
raw: serde_json::Value::Null,
cache_prefix: false,
}
}
#[tokio::test]
async fn non_empty_passes_on_tool_use_when_tools_declared() {
assert_eq!(
NonEmptyGate
.evaluate(&tool_req_with_schema(), &tool_use_resp())
.await
.verdict,
Verdict::Pass
);
}
#[tokio::test]
async fn non_empty_fails_on_tool_call_when_no_tools_declared() {
assert_eq!(
NonEmptyGate
.evaluate(&req(), &tool_use_resp())
.await
.verdict,
Verdict::Fail,
"tool call without declared tools must Fail non-empty (undeclared/hallucinated)"
);
}
#[tokio::test]
async fn non_empty_still_fails_on_truly_empty_response() {
assert_eq!(
NonEmptyGate.evaluate(&req(), &resp("")).await.verdict,
Verdict::Fail
);
assert_eq!(
NonEmptyGate.evaluate(&req(), &resp(" ")).await.verdict,
Verdict::Fail
);
}
#[tokio::test]
async fn json_valid_passes_on_tool_use_when_tools_declared() {
assert_eq!(
JsonValidGate
.evaluate(&tool_req_with_schema(), &tool_use_resp())
.await
.verdict,
Verdict::Pass
);
}
#[tokio::test]
async fn json_valid_fails_on_tool_call_when_no_tools_declared() {
assert_eq!(
JsonValidGate
.evaluate(&req(), &tool_use_resp())
.await
.verdict,
Verdict::Fail,
"tool call without declared tools must Fail json-valid"
);
}
#[tokio::test]
async fn json_valid_passes_on_preamble_text_plus_tool_calls() {
let resp_with_preamble = ModelResponse {
text: "I will look that up for you.".to_owned(),
raw: serde_json::json!({
"content": [
{"type": "text", "text": "I will look that up for you."},
{"type": "tool_use", "id": "t1", "name": "search", "input": {"q": "hello"}}
]
}),
..tool_use_resp()
};
assert_eq!(
JsonValidGate
.evaluate(&tool_req_with_schema(), &resp_with_preamble)
.await
.verdict,
Verdict::Pass,
"preamble text + tool calls with declared tools must Pass json-valid"
);
}
#[tokio::test]
async fn json_valid_still_fails_on_non_json_text_response() {
assert_eq!(
JsonValidGate
.evaluate(&req(), &resp("not json"))
.await
.verdict,
Verdict::Fail
);
}
#[tokio::test]
async fn schema_gate_passes_on_tool_use_when_tools_declared() {
let g = SchemaGate::new(
"extract",
serde_json::json!({"type": "object", "required": ["name"]}),
);
assert_eq!(
g.evaluate(&tool_req_with_schema(), &tool_use_resp())
.await
.verdict,
Verdict::Pass
);
}
#[tokio::test]
async fn schema_gate_passes_on_preamble_text_plus_tool_calls() {
let g = SchemaGate::new(
"extract",
serde_json::json!({"type": "object", "required": ["name"]}),
);
let resp_with_preamble = ModelResponse {
text: "Here is the result:".to_owned(),
raw: serde_json::json!({
"content": [
{"type": "text", "text": "Here is the result:"},
{"type": "tool_use", "id": "t1", "name": "search", "input": {"q": "hello"}}
]
}),
..tool_use_resp()
};
assert_eq!(
g.evaluate(&tool_req_with_schema(), &resp_with_preamble)
.await
.verdict,
Verdict::Pass,
"preamble text + tool calls with declared tools must Pass schema-gate"
);
}
#[tokio::test]
async fn tool_args_valid_passes_with_valid_args() {
assert_eq!(
ToolArgsValidGate
.evaluate(&tool_req_with_schema(), &tool_use_resp())
.await
.verdict,
Verdict::Pass
);
}
#[tokio::test]
async fn tool_args_valid_passes_when_no_tool_calls() {
assert_eq!(
ToolArgsValidGate
.evaluate(&req(), &resp("hello"))
.await
.verdict,
Verdict::Pass
);
}
#[tokio::test]
async fn tool_args_valid_fails_on_hallucinated_tool_name() {
let bad_resp = ModelResponse {
text: String::new(),
raw: serde_json::json!({
"content": [{
"type": "tool_use",
"id": "t1",
"name": "unknown_tool",
"input": { "x": 1 }
}]
}),
..tool_use_resp()
};
let r = ToolArgsValidGate
.evaluate(&tool_req_with_schema(), &bad_resp)
.await;
assert_eq!(r.verdict, Verdict::Fail);
assert!(
r.reason
.as_deref()
.is_some_and(|s| s.contains("unknown_tool")),
"reason must name the hallucinated tool"
);
}
#[tokio::test]
async fn tool_args_valid_fails_when_no_tools_declared_but_response_has_calls() {
assert_eq!(
ToolArgsValidGate
.evaluate(&req(), &tool_use_resp()) .await
.verdict,
Verdict::Fail
);
}
#[tokio::test]
async fn tool_args_valid_fails_on_missing_required_field() {
let bad_resp = ModelResponse {
text: String::new(),
raw: serde_json::json!({
"content": [{
"type": "tool_use",
"id": "t1",
"name": "search",
"input": {} }]
}),
..tool_use_resp()
};
let r = ToolArgsValidGate
.evaluate(&tool_req_with_schema(), &bad_resp)
.await;
assert_eq!(r.verdict, Verdict::Fail);
assert!(
r.reason.as_deref().is_some_and(|s| s.contains("search")),
"reason must name the failing tool"
);
}
#[tokio::test]
async fn tool_args_valid_checks_responses_function_call_args() {
let responses_req = serde_json::json!({
"model": "gpt-5.5", "input": "weather?", "stream": true,
"tools": [{ "type": "function", "name": "get_weather", "parameters": {
"type": "object", "required": ["city"],
"properties": { "city": { "type": "string" } } } }],
});
let chat = crate::responses::request_to_chat(&responses_req).to_string();
let req = crate::proxy::parse_openai_request(chat.as_bytes(), true).unwrap();
let resp = |args: &str| ModelResponse {
raw: serde_json::json!({ "choices": [{ "message": { "role": "assistant",
"content": null, "tool_calls": [{ "id": "call_1", "type": "function",
"function": { "name": "get_weather", "arguments": args } }] } }] }),
..tool_use_resp()
};
let bad = ToolArgsValidGate
.evaluate(&req, &resp(r#"{"town":"Paris"}"#))
.await;
assert_eq!(bad.verdict, Verdict::Fail, "{:?}", bad.reason);
let good = ToolArgsValidGate
.evaluate(&req, &resp(r#"{"city":"Paris"}"#))
.await;
assert_eq!(good.verdict, Verdict::Pass, "{:?}", good.reason);
}
#[tokio::test]
async fn tool_args_valid_checks_flat_responses_tool_schema() {
let req = ModelRequest {
tools: serde_json::json!([{ "type": "function", "name": "search", "parameters": {
"type": "object", "required": ["q"], "properties": { "q": { "type": "string" } } } }]),
..tool_req_with_schema()
};
let bad = ModelResponse {
raw: serde_json::json!({ "content": [{ "type": "tool_use", "id": "t1",
"name": "search", "input": {} }] }),
..tool_use_resp()
};
assert_eq!(
ToolArgsValidGate.evaluate(&req, &bad).await.verdict,
Verdict::Fail
);
assert_eq!(
ToolArgsValidGate
.evaluate(&req, &tool_use_resp())
.await
.verdict,
Verdict::Pass
);
}
#[tokio::test]
async fn tool_args_valid_resolves_as_builtin() {
let gates = resolve_gates(
&["tool-args-valid".to_owned()],
&[],
&empty_registry(),
&Auth::default(),
&PriceTable::default(),
&reqwest::Client::new(),
);
assert_eq!(
gates.iter().map(|g| g.id()).collect::<Vec<_>>(),
["tool-args-valid"]
);
}
#[test]
fn schema_violation_nested_required_missing() {
let schema = json!({
"type": "object",
"properties": {
"address": {
"type": "object",
"required": ["street"],
"properties": { "street": {"type": "string"} }
}
}
});
assert!(
schema_violation(&json!({"address": {}}), &schema, "root").is_some(),
"nested required field must produce a violation"
);
assert!(
schema_violation(
&json!({"address": {"street": "1 Main St"}}),
&schema,
"root"
)
.is_none(),
"valid nested object must produce no violation"
);
}
#[test]
fn schema_violation_enum_check() {
let schema = json!({ "enum": ["red", "green", "blue"] });
assert!(
schema_violation(&json!("purple"), &schema, "root").is_some(),
"value outside enum must produce a violation"
);
assert!(
schema_violation(&json!("red"), &schema, "root").is_none(),
"value inside enum must produce no violation"
);
}
#[test]
fn schema_violation_items_type_wrong() {
let schema = json!({
"type": "array",
"items": { "type": "integer" }
});
assert!(
schema_violation(&json!([1, 2, "three"]), &schema, "root").is_some(),
"array item of wrong type must produce a violation"
);
assert!(
schema_violation(&json!([1, 2, 3]), &schema, "root").is_none(),
"array of correct type must produce no violation"
);
}
#[test]
fn schema_violation_additional_properties_subschema() {
let schema = serde_json::json!({
"type": "object",
"properties": {"a": {"type": "string"}},
"additionalProperties": {"type": "integer"}
});
assert!(
schema_violation(&serde_json::json!({"a": "x", "b": 1}), &schema, "args").is_none()
);
assert!(
schema_violation(&serde_json::json!({"a": "x", "b": "no"}), &schema, "args").is_some()
);
}
#[test]
fn anthropic_tool_use_without_input_is_malformed() {
let raw = serde_json::json!({"content": [{"type": "tool_use", "id": "t1", "name": "f"}]});
let calls = extract_response_tool_calls(&raw);
assert_eq!(calls.len(), 1);
assert!(serde_json::from_str::<serde_json::Value>(&calls[0].1).is_err());
}
#[test]
fn openai_tool_call_without_string_arguments_is_malformed() {
let raw = serde_json::json!({"choices": [{"message": {"tool_calls": [
{"id": "c1", "type": "function", "function": {"name": "f"}},
{"id": "c2", "type": "function", "function": {"name": "f", "arguments": {"x": 1}}}
]}}]});
let calls = extract_response_tool_calls(&raw);
assert_eq!(calls.len(), 2);
for (_, args) in &calls {
assert!(
serde_json::from_str::<serde_json::Value>(args).is_err(),
"missing/non-string arguments must not parse as an empty object"
);
}
}
#[test]
fn schema_violation_additional_properties_false() {
let schema = json!({
"type": "object",
"properties": { "name": {"type": "string"} },
"additionalProperties": false
});
assert!(
schema_violation(&json!({"name": "Alice", "age": 30}), &schema, "root").is_some(),
"extra key with additionalProperties:false must produce a violation"
);
assert!(
schema_violation(&json!({"name": "Alice"}), &schema, "root").is_none(),
"object with only declared keys must produce no violation"
);
}
#[test]
fn schema_violation_complex_valid_nested() {
let schema = json!({
"type": "object",
"required": ["id", "tags"],
"additionalProperties": false,
"properties": {
"id": {"type": "integer"},
"tags": {
"type": "array",
"items": {"type": "string"}
}
}
});
assert!(
schema_violation(
&json!({"id": 1, "tags": ["rust", "agent"]}),
&schema,
"root"
)
.is_none(),
"fully valid complex value must produce no violation"
);
}
#[tokio::test]
async fn tool_args_valid_fails_on_openai_tool_call_missing_function_key() {
let malformed_resp = ModelResponse {
text: String::new(),
raw: serde_json::json!({
"choices": [{
"message": {
"role": "assistant",
"tool_calls": [
{"id": "call_1", "type": "function"}
]
}
}]
}),
..tool_use_resp()
};
let r = ToolArgsValidGate
.evaluate(&tool_req_with_schema(), &malformed_resp)
.await;
assert_eq!(
r.verdict,
Verdict::Fail,
"malformed OpenAI tool call (missing function) must Fail, not Pass"
);
}
#[tokio::test]
async fn tool_args_valid_fails_on_anthropic_tool_use_missing_name() {
let malformed_resp = ModelResponse {
text: String::new(),
raw: serde_json::json!({
"content": [{
"type": "tool_use",
"id": "t1",
"input": {"q": "hello"}
}]
}),
..tool_use_resp()
};
let r = ToolArgsValidGate
.evaluate(&tool_req_with_schema(), &malformed_resp)
.await;
assert_eq!(
r.verdict,
Verdict::Fail,
"Anthropic tool_use missing name must Fail, not Pass"
);
}
}