agent-evaluate-contract 0.5.0

Transport-neutral contracts for Agent Infra evaluation APIs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154

//! Human review queue and the error-analysis sampling protocol.

use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ReviewState {
    Queued,
    Claimed,
    Approved,
    Rejected,
}

/// The reviewer's judgement of the agent, kept apart from the review's own
/// workflow state: approving a review of a failed case is not a pass.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ReviewOutcome {
    Pass,
    Fail,
}

impl ReviewOutcome {
    pub const fn is_pass(self) -> bool {
        matches!(self, Self::Pass)
    }
}

pub const MAX_REVIEW_LABELS: usize = 16;
pub const MAX_REVIEW_LABEL_CHARS: usize = 64;
pub const MAX_REVIEW_NOTE_CHARS: usize = 4_000;

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct HumanReview {
    pub id: String,
    pub run_id: String,
    pub case_run_id: String,
    pub state: ReviewState,
    pub reason: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub reviewer: Option<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub decision_note: Option<String>,
    /// Free-text observation captured during error analysis, before any
    /// failure taxonomy exists.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub note: Option<String>,
    /// Named failure categories. These become the candidate metric set.
    #[serde(default)]
    pub labels: Vec<String>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub overall: Option<ReviewOutcome>,
    /// True when the review was enqueued by stratified sampling rather than
    /// by a failing result.
    #[serde(default)]
    pub sampled: bool,
    pub fencing_token: i64,
    pub created_at_ms: i64,
    pub updated_at_ms: i64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct EnqueueReviewRequest {
    pub run_id: String,
    pub case_run_id: String,
    pub reason: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct ClaimReviewRequest {
    #[serde(default = "default_review_lease")]
    pub lease_ms: i64,
}

fn default_review_lease() -> i64 {
    300_000
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct CompleteReviewRequest {
    pub review_id: String,
    pub fencing_token: i64,
    pub approved: bool,
    #[serde(default)]
    pub note: String,
    #[serde(default)]
    pub labels: Vec<String>,
    /// Defaults to `approved` when omitted.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub overall: Option<ReviewOutcome>,
}

impl CompleteReviewRequest {
    pub fn effective_outcome(&self) -> ReviewOutcome {
        self.overall.unwrap_or(if self.approved {
            ReviewOutcome::Pass
        } else {
            ReviewOutcome::Fail
        })
    }
}

/// How a review batch is drawn from a finished run.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ReviewSamplingStrategy {
    /// Category x outcome strata, every stratum represented. Passing cases are
    /// included on purpose: failures the evaluators never caught are only
    /// visible there.
    #[default]
    Stratified,
    /// Uniform sample across the whole run.
    Random,
    /// Only cases the evaluators already failed.
    FailedOnly,
}

pub const MAX_REVIEW_SAMPLE_SIZE: usize = 200;
pub const MAX_REVIEW_STRATA: usize = 256;

fn default_sample_size() -> usize {
    30
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct SampleReviewsRequest {
    pub run_id: String,
    #[serde(default = "default_sample_size")]
    pub size: usize,
    #[serde(default)]
    pub strategy: ReviewSamplingStrategy,
    /// Makes the draw reproducible across replicas and retries.
    #[serde(default)]
    pub seed: u64,
    #[serde(default)]
    pub reason: String,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SampleReviewsResponse {
    pub run_id: String,
    pub requested: usize,
    pub sampled: usize,
    /// Reviews already queued for a sampled case are reused, never duplicated.
    pub created: usize,
    pub strata: usize,
    pub reviews: Vec<HumanReview>,
}