agent-evaluate-v2-contract 0.1.0

Vendored Evaluate v2 transport contract for agent-infra-sdk
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

//! Asynchronous analysis: model-graded judges, plugins and re-analysis.
//!
//! Re-analysis is the payoff of storing a trace reference instead of a
//! verdict: a new analyzer version can rescore an existing case run without
//! executing the agent again.

use serde::{Deserialize, Serialize};

use crate::EvalScope;
use crate::catalog::SnapshotRef;
use crate::evaluator::CaseEvaluator;

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum AnalysisState {
    Queued,
    Running,
    Succeeded,
    Failed,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct AnalysisTask {
    pub id: String,
    pub scope: EvalScope,
    pub run_id: String,
    pub case_run_id: String,
    #[serde(rename = "snapshotRef")]
    pub snapshot_ref: SnapshotRef,
    pub case_id: String,
    pub evaluator: CaseEvaluator,
    /// Report revision this result will land in.
    pub revision: u32,
    pub state: AnalysisState,
    pub attempt: u32,
    pub fencing_token: i64,
    pub created_at_ms: i64,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimAnalysisRequest {
    #[serde(default = "default_lease_ms")]
    pub lease_ms: i64,
    #[serde(default = "default_batch")]
    pub max_tasks: usize,
}

fn default_lease_ms() -> i64 {
    120_000
}

fn default_batch() -> usize {
    1
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimAnalysisResponse {
    pub tasks: Vec<AnalysisTask>,
    pub lease_until_ms: i64,
}

/// One judge's contribution. Kept per judge so a later question — "which
/// judge was dragging the score?" — is answerable from stored data.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct JudgeVote {
    pub model_ref: String,
    /// Median of this judge's own self-consistency samples.
    pub score: f64,
    #[serde(default)]
    pub cost_micros: u64,
    #[serde(default)]
    pub detail: String,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct JudgeEvidence {
    pub votes: Vec<JudgeVote>,
    /// Attestation bound to the isolated executor boundary. Production fails
    /// closed without it; the control plane never runs judge code itself.
    #[serde(default)]
    pub attestation: String,
    /// Set when the executor stopped early on the ensemble cost budget.
    #[serde(default)]
    pub budget_exhausted: bool,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct PluginEvidence {
    pub plugin_ref: String,
    pub passed: bool,
    #[serde(default)]
    pub detail: String,
    #[serde(default)]
    pub cost_micros: u64,
    #[serde(default)]
    pub attestation: String,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum AnalysisOutcome {
    Judge(JudgeEvidence),
    Plugin(PluginEvidence),
    /// The analyzer could not produce a result. Retryable, and deliberately
    /// separate from a quality failure.
    Failed {
        error: String,
    },
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SubmitAnalysisRequest {
    pub task_id: String,
    pub fencing_token: i64,
    pub outcome: AnalysisOutcome,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SubmitAnalysisResponse {
    pub accepted: bool,
}

/// Rescore stored case runs with a new analyzer set. Produces a new report
/// revision; the executed evidence and trace reference are untouched.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RerunAnalysisRequest {
    pub run_id: String,
    /// Empty rescoring means "every asynchronous evaluator of every case".
    #[serde(default)]
    pub case_run_ids: Vec<String>,
    #[serde(default)]
    pub evaluators: Vec<CaseEvaluator>,
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RerunAnalysisResponse {
    pub revision: u32,
    pub scheduled_tasks: usize,
}