assay-core 2.17.0

High-performance evaluation framework for LLM agents (Core)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(untagged)]
pub enum TraceEntry {
    V1(TraceEntryV1),
    V2(TraceEvent),
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraceEntryV1 {
    pub schema_version: u32,
    #[serde(rename = "type")]
    pub entry_type: String, // "assay.trace"
    pub request_id: String,
    pub prompt: String,
    pub response: String,
    #[serde(default)]
    pub meta: serde_json::Value,
}

// --- V2 Schema (Streamable) ---

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "type")]
pub enum TraceEvent {
    #[serde(rename = "episode_start")]
    EpisodeStart(EpisodeStart),
    #[serde(rename = "step")]
    Step(StepEntry),
    #[serde(rename = "tool_call")]
    ToolCall(ToolCallEntry),
    #[serde(rename = "episode_end")]
    EpisodeEnd(EpisodeEnd),
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct EpisodeStart {
    pub episode_id: String,
    pub timestamp: u64, // ms
    #[serde(default)]
    pub input: serde_json::Value, // { prompt: ... }
    #[serde(default)]
    pub meta: serde_json::Value,
}

// --- Provenance ---

// --- Provenance ---

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct TruncationMeta {
    pub field: String,
    pub original_len: usize,
    pub kept_len: usize,
    pub sha256: String,
    pub strategy: String,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct StepEntry {
    pub episode_id: String,
    pub step_id: String,
    #[serde(default)]
    pub idx: u32,
    pub timestamp: u64,
    pub kind: String,
    pub name: Option<String>,
    #[serde(default)]
    pub content: Option<String>,
    #[serde(default)]
    pub content_sha256: Option<String>, // Hash of full content
    #[serde(default)]
    pub truncations: Vec<TruncationMeta>,
    #[serde(default)]
    pub meta: serde_json::Value,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ToolCallEntry {
    pub episode_id: String,
    pub step_id: String,
    pub timestamp: u64,
    pub tool_name: String,
    pub call_index: Option<u32>, // Added for uniqueness (step_id, call_index)
    #[serde(default)]
    pub args: serde_json::Value,
    #[serde(default)]
    pub args_sha256: Option<String>,
    #[serde(default)]
    pub result: Option<serde_json::Value>,
    #[serde(default)]
    pub result_sha256: Option<String>,
    #[serde(default)]
    pub error: Option<String>,
    #[serde(default)]
    pub truncations: Vec<TruncationMeta>,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct EpisodeEnd {
    pub episode_id: String,
    pub timestamp: u64,
    pub outcome: Option<String>, // "pass", "fail", "error"
    #[serde(default)]
    pub final_output: Option<String>,
}