agent-evaluate-contract 0.5.0

Transport-neutral contracts for Agent Infra evaluation APIs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

//! Objective metrics derived deterministically from one authoritative Trace partition.

use serde::{Deserialize, Serialize};
use serde_json::Value;

/// Identifies one Trace partition and gives the resulting metric set a stable id.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct TraceObjectiveMetricsRequest {
    pub trace_id: String,
    pub conversation_id: String,
    pub run_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub expected_turns: Option<u64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub expected_tool_calls: Option<u64>,
}

/// Transport-neutral event projection consumed by Evaluate's pure calculator.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ObjectiveTraceEvent {
    pub id: String,
    pub event_type: String,
    #[serde(default)]
    pub payload: Value,
    #[serde(default)]
    pub sequence: i64,
    #[serde(default)]
    pub created_at_ms: i64,
}

/// One raw, reproducible observation. No weighted or model-judged score is produced.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ObjectiveMetricObservation {
    pub metric: String,
    pub value: f64,
    pub unit: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub numerator: Option<u64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub denominator: Option<u64>,
}

/// Complete bounded metric view derivable without an LLM judge or repository code.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct TraceObjectiveMetrics {
    pub trace_id: String,
    pub conversation_id: String,
    pub run_id: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub expected_turns: Option<u64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub expected_tool_calls: Option<u64>,
    pub event_count: u64,
    pub recognized_event_count: u64,
    pub unknown_event_count: u64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub started_at_ms: Option<i64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub completed_at_ms: Option<i64>,
    pub wall_duration_ms: u64,
    pub model_duration_ms: u64,
    pub max_model_duration_ms: u64,
    pub tool_duration_ms: u64,
    pub max_tool_duration_ms: u64,
    pub run_started_count: u64,
    pub run_completed_count: u64,
    pub run_failed_count: u64,
    pub model_started_count: u64,
    pub model_completed_count: u64,
    pub model_failed_count: u64,
    pub tool_started_count: u64,
    pub tool_completed_count: u64,
    pub tool_failed_count: u64,
    pub retry_count: u64,
    pub context_compaction_count: u64,
    pub input_tokens: u64,
    pub input_cached_tokens: u64,
    pub input_cache_write_tokens: u64,
    pub output_tokens: u64,
    pub total_tokens: u64,
    pub cost_micros: u64,
    pub error_count: u64,
    pub timestamp_regression_count: u64,
    pub sequence_regression_count: u64,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub recognized_event_rate: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub run_success_rate: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub model_success_rate: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tool_success_rate: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub turn_completion_rate: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub expected_tool_completion_rate: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub average_model_duration_ms: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub average_tool_duration_ms: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub output_tokens_per_second: Option<f64>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub input_cache_hit_rate: Option<f64>,
    pub complete: bool,
    #[serde(default)]
    pub missing_signals: Vec<String>,
    #[serde(default)]
    pub observations: Vec<ObjectiveMetricObservation>,
}