agent_evaluate_contract/
review.rs

1//! Human review queue and the error-analysis sampling protocol.
2
3use serde::{Deserialize, Serialize};
4
5#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
6#[serde(rename_all = "snake_case")]
7pub enum ReviewState {
8    Queued,
9    Claimed,
10    Approved,
11    Rejected,
12}
13
14/// The reviewer's judgement of the agent, kept apart from the review's own
15/// workflow state: approving a review of a failed case is not a pass.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
17#[serde(rename_all = "snake_case")]
18pub enum ReviewOutcome {
19    Pass,
20    Fail,
21}
22
23impl ReviewOutcome {
24    pub const fn is_pass(self) -> bool {
25        matches!(self, Self::Pass)
26    }
27}
28
29pub const MAX_REVIEW_LABELS: usize = 16;
30pub const MAX_REVIEW_LABEL_CHARS: usize = 64;
31pub const MAX_REVIEW_NOTE_CHARS: usize = 4_000;
32
33#[derive(Debug, Clone, Serialize, Deserialize)]
34#[serde(rename_all = "camelCase")]
35pub struct HumanReview {
36    pub id: String,
37    pub run_id: String,
38    pub case_run_id: String,
39    pub state: ReviewState,
40    pub reason: String,
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub reviewer: Option<String>,
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub decision_note: Option<String>,
45    /// Free-text observation captured during error analysis, before any
46    /// failure taxonomy exists.
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub note: Option<String>,
49    /// Named failure categories. These become the candidate metric set.
50    #[serde(default)]
51    pub labels: Vec<String>,
52    #[serde(default, skip_serializing_if = "Option::is_none")]
53    pub overall: Option<ReviewOutcome>,
54    /// True when the review was enqueued by stratified sampling rather than
55    /// by a failing result.
56    #[serde(default)]
57    pub sampled: bool,
58    pub fencing_token: i64,
59    pub created_at_ms: i64,
60    pub updated_at_ms: i64,
61}
62
63#[derive(Debug, Clone, Serialize, Deserialize)]
64#[serde(rename_all = "camelCase", deny_unknown_fields)]
65pub struct EnqueueReviewRequest {
66    pub run_id: String,
67    pub case_run_id: String,
68    pub reason: String,
69}
70
71#[derive(Debug, Clone, Serialize, Deserialize)]
72#[serde(rename_all = "camelCase", deny_unknown_fields)]
73pub struct ClaimReviewRequest {
74    #[serde(default = "default_review_lease")]
75    pub lease_ms: i64,
76}
77
78fn default_review_lease() -> i64 {
79    300_000
80}
81
82#[derive(Debug, Clone, Serialize, Deserialize)]
83#[serde(rename_all = "camelCase", deny_unknown_fields)]
84pub struct CompleteReviewRequest {
85    pub review_id: String,
86    pub fencing_token: i64,
87    pub approved: bool,
88    #[serde(default)]
89    pub note: String,
90    #[serde(default)]
91    pub labels: Vec<String>,
92    /// Defaults to `approved` when omitted.
93    #[serde(default, skip_serializing_if = "Option::is_none")]
94    pub overall: Option<ReviewOutcome>,
95}
96
97impl CompleteReviewRequest {
98    pub fn effective_outcome(&self) -> ReviewOutcome {
99        self.overall.unwrap_or(if self.approved {
100            ReviewOutcome::Pass
101        } else {
102            ReviewOutcome::Fail
103        })
104    }
105}
106
107/// How a review batch is drawn from a finished run.
108#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
109#[serde(rename_all = "snake_case")]
110pub enum ReviewSamplingStrategy {
111    /// Category x outcome strata, every stratum represented. Passing cases are
112    /// included on purpose: failures the evaluators never caught are only
113    /// visible there.
114    #[default]
115    Stratified,
116    /// Uniform sample across the whole run.
117    Random,
118    /// Only cases the evaluators already failed.
119    FailedOnly,
120}
121
122pub const MAX_REVIEW_SAMPLE_SIZE: usize = 200;
123pub const MAX_REVIEW_STRATA: usize = 256;
124
125fn default_sample_size() -> usize {
126    30
127}
128
129#[derive(Debug, Clone, Serialize, Deserialize)]
130#[serde(rename_all = "camelCase", deny_unknown_fields)]
131pub struct SampleReviewsRequest {
132    pub run_id: String,
133    #[serde(default = "default_sample_size")]
134    pub size: usize,
135    #[serde(default)]
136    pub strategy: ReviewSamplingStrategy,
137    /// Makes the draw reproducible across replicas and retries.
138    #[serde(default)]
139    pub seed: u64,
140    #[serde(default)]
141    pub reason: String,
142}
143
144#[derive(Debug, Clone, Serialize, Deserialize)]
145#[serde(rename_all = "camelCase")]
146pub struct SampleReviewsResponse {
147    pub run_id: String,
148    pub requested: usize,
149    pub sampled: usize,
150    /// Reviews already queued for a sampled case are reused, never duplicated.
151    pub created: usize,
152    pub strata: usize,
153    pub reviews: Vec<HumanReview>,
154}
agent_evaluate_contract/review.rs

agent_evaluate_contract/
review.rs
