assay-core 3.9.1

High-performance evaluation framework for LLM agents (Core)
Documentation
use assay_core::policy_engine::{evaluate_tool_args, VerdictStatus};

use crate::{CheckInput, Outcome};

pub fn args_valid(params: &serde_json::Value, input: &CheckInput) -> (Outcome, String) {
    let schema = match params.get("schema") {
        Some(s) => s,
        None => return (Outcome::Error, "Config error: schema missing".into()),
    };

    // Wrap simple schema in tool map for policy_engine
    let tool_name = input.tool_name.as_deref().unwrap_or("unknown");
    let policy = serde_json::json!({
         tool_name: schema
    });

    let args = match &input.args {
        Some(a) => a,
        None => return (Outcome::Error, "No args provided".into()),
    };

    let verdict = evaluate_tool_args(&policy, tool_name, args);

    match verdict.status {
        VerdictStatus::Allowed => (Outcome::Pass, "args valid".into()),
        VerdictStatus::Blocked => {
            // Map reason codes to test expectations if needed
            if verdict.reason_code == "E_ARG_SCHEMA" {
                // Extract first violation for parity valid reason check
                // The test expects "percent 50 exceeds maximum 30" etc.
                // The real engine returns structured JSON violations.
                // We need to adapt the message to match the mock test cases OR update test cases.
                // For V1 integration, ensuring Outcome matches is priority #1.
                // The mock test strings are very specific "percent {} exceeds maximum {}".
                // Real engine says: "data.percent: 50.0 is greater than the maximum of 30.0" (JSON schema output)
                // To pass the STRICT parity test provided by the user (which checks `reason == reason`),
                // we just need consistent strings.
                // Since *both* Batch and Streaming call *this* wrapper, they will get identical strings.
                // So we can return a generic string or the detailed one.
                // Let's return the structured details stringified
                (
                    Outcome::Fail,
                    format!("Schema violation: {}", verdict.details),
                )
            } else if verdict.reason_code == "E_POLICY_MISSING_TOOL" {
                (Outcome::Error, "Tool not in policy".into())
            } else {
                (Outcome::Fail, format!("Blocked: {}", verdict.reason_code))
            }
        }
    }
}

pub fn sequence_valid(params: &serde_json::Value, input: &CheckInput) -> (Outcome, String) {
    let trace = match &input.trace {
        Some(t) => t,
        None => return (Outcome::Error, "No trace provided".into()),
    };

    let tool_names: Vec<&str> = trace.iter().map(|t| t.tool_name.as_str()).collect();

    if let Some(rules) = params.get("rules").and_then(|r| r.as_array()) {
        for rule in rules {
            if let Some(rule_type) = rule.get("type").and_then(|t| t.as_str()) {
                match rule_type {
                    "require" => {
                        if let Some(tool) = rule.get("tool").and_then(|t| t.as_str()) {
                            if !tool_names.contains(&tool) {
                                return (
                                    Outcome::Fail,
                                    format!("required tool not called: {}", tool),
                                );
                            }
                        }
                    }
                    "before" => {
                        let first = rule.get("first").and_then(|t| t.as_str());
                        let then = rule.get("then").and_then(|t| t.as_str());

                        if let (Some(first), Some(then)) = (first, then) {
                            let first_idx = tool_names.iter().position(|&t| t == first);
                            let then_idx = tool_names.iter().position(|&t| t == then);

                            match (first_idx, then_idx) {
                                (Some(f), Some(t)) if f >= t => {
                                    return (
                                        Outcome::Fail,
                                        format!("{} must come before {}", first, then),
                                    );
                                }
                                (None, Some(_)) => {
                                    return (
                                        Outcome::Fail,
                                        format!("{} must come before {}", first, then),
                                    );
                                }
                                _ => {}
                            }
                        }
                    }
                    _ => {}
                }
            }
        }
    }

    (Outcome::Pass, "sequence valid".into())
}

pub fn blocklist(params: &serde_json::Value, input: &CheckInput) -> (Outcome, String) {
    let tool = match &input.tool_name {
        Some(t) => t,
        None => return (Outcome::Error, "No tool_name provided".into()),
    };

    if let Some(blocked) = params.get("blocked").and_then(|b| b.as_array()) {
        for b in blocked {
            if let Some(blocked_name) = b.as_str() {
                if tool == blocked_name {
                    return (Outcome::Fail, format!("tool {} is blocked", tool));
                }
            }
        }
    }

    (Outcome::Pass, "tool allowed".into())
}