assay-core 3.9.1

High-performance evaluation framework for LLM agents (Core)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

use serde::{Deserialize, Serialize};
use std::collections::HashMap;

/// A single step in the explained trace
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExplainedStep {
    pub index: usize,
    pub tool: String,

    /// Tool arguments (if available)
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub args: Option<serde_json::Value>,

    /// Verdict for this step
    pub verdict: StepVerdict,

    /// Rules that were evaluated
    pub rules_evaluated: Vec<RuleEvaluation>,

    /// Current state of stateful rules after this step
    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
    pub state_snapshot: HashMap<String, String>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum StepVerdict {
    /// Tool call allowed
    Allowed,
    /// Tool call blocked by a rule
    Blocked,
    /// Tool call allowed but triggered a warning
    Warning,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RuleEvaluation {
    pub rule_id: String,
    pub rule_type: String,
    pub passed: bool,
    pub explanation: String,

    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub context: Option<serde_json::Value>,
}

/// Complete explanation of a trace
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraceExplanation {
    pub policy_name: String,
    pub policy_version: String,
    pub total_steps: usize,
    pub allowed_steps: usize,
    pub blocked_steps: usize,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub first_block_index: Option<usize>,

    /// Detailed step-by-step explanation
    pub steps: Vec<ExplainedStep>,

    /// Summary of rules that blocked
    pub blocking_rules: Vec<String>,
}

/// Tool call input for explanation
#[derive(Debug, Clone, Deserialize)]
pub struct ToolCall {
    /// Tool name
    #[serde(alias = "name", alias = "tool_name")]
    pub tool: String,

    /// Tool arguments
    #[serde(default)]
    pub args: Option<serde_json::Value>,
}