agent_evaluate_contract/
lib.rs

1//! HTTP API contract for the Evaluate service.
2//!
3//! This service provides a programmatic interface for running agent evaluation
4//! cases, collecting metrics, and producing reports. The eval harness replays
5//! predefined conversation turns against an agent runtime instance and judges
6//! outcomes against oracles.
7//!
8//! # Layering
9//!
10//! ```text
11//! agent-evaluate-contract   paths + DTOs        (this crate)
12//! agent-evaluate-runtime    eval runner + judge
13//! agent-evaluate-server     router + auth + bin
14//! ```
15
16use serde::{Deserialize, Serialize};
17use serde_json::Value;
18
19pub const SERVICE_NAME: &str = "agent-evaluate";
20pub const DEFAULT_PORT: u16 = 5150;
21pub const AUDIENCE: &str = "agent-evaluate";
22pub const SERVICE_TOKEN_ENV: &str = "AGENT_EVALUATE_SERVICE_TOKEN";
23pub const MAX_RESOURCE_ID_CHARS: usize = 191;
24
25pub const HEALTH_PATH: &str = "/health";
26pub const RUN_EVAL_PATH: &str = "/internal/v1/evaluate/run";
27pub const LIST_CASES_PATH: &str = "/internal/v1/evaluate/cases";
28pub const GET_CASE_PATH: &str = "/internal/v1/evaluate/cases/get";
29pub const GET_REPORT_PATH: &str = "/internal/v1/evaluate/reports";
30pub const LOAD_CASES_PATH: &str = "/internal/v1/evaluate/cases/load";
31pub const SUITES_PATH: &str = "/internal/v1/evaluate/suites";
32pub const RUNS_PATH: &str = "/internal/v1/evaluate/runs";
33pub const OPERATIONS_PATH: &str = "/internal/v1/evaluate/operations";
34pub const RUN_CASES_PATH: &str = "/internal/v1/evaluate/runs/cases";
35pub const CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/runs/cancel";
36pub const DURABLE_REPORT_PATH: &str = "/internal/v1/evaluate/reports/get";
37pub const COMPARE_REPORTS_PATH: &str = "/internal/v1/evaluate/reports/compare";
38pub const REVIEW_QUEUE_PATH: &str = "/internal/v1/evaluate/reviews";
39pub const CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/claim";
40pub const COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/complete";
41pub const ENVIRONMENT_PROFILES_PATH: &str = "/internal/v1/evaluate/environment-profiles";
42pub const EVALUATOR_VERSIONS_PATH: &str = "/internal/v1/evaluate/evaluator-versions";
43pub const DRIVER_CLAIM_PATH: &str = "/internal/v1/evaluate/driver/cases/claim";
44pub const DRIVER_RENEW_PATH: &str = "/internal/v1/evaluate/driver/cases/renew";
45pub const DRIVER_SUBMIT_PATH: &str = "/internal/v1/evaluate/driver/cases/submit";
46pub const ADMIN_OVERVIEW_PATH: &str = "/internal/v1/evaluate/admin/overview";
47pub const ADMIN_SUITES_PATH: &str = "/internal/v1/evaluate/admin/suites";
48pub const ADMIN_RUNS_PATH: &str = "/internal/v1/evaluate/admin/runs";
49pub const ADMIN_RUN_CASES_PATH: &str = "/internal/v1/evaluate/admin/runs/cases";
50pub const ADMIN_CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/admin/runs/cancel";
51pub const ADMIN_REPORT_PATH: &str = "/internal/v1/evaluate/admin/reports/get";
52pub const ADMIN_REVIEWS_PATH: &str = "/internal/v1/evaluate/admin/reviews";
53pub const ADMIN_CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/claim";
54pub const ADMIN_COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/complete";
55
56// ---------------------------------------------------------------------------
57// Eval case types
58// ---------------------------------------------------------------------------
59
60#[derive(Debug, Clone, Serialize, Deserialize)]
61#[serde(rename_all = "camelCase")]
62pub struct EvalCaseFile {
63    pub schema_version: i32,
64    pub cases: Vec<EvalCase>,
65}
66
67#[derive(Debug, Clone, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase")]
69pub struct EvalCase {
70    pub id: String,
71    pub category: String,
72    #[serde(default)]
73    pub notes: String,
74    pub turns: Vec<EvalTurn>,
75    #[serde(default, skip_serializing_if = "Option::is_none")]
76    pub workspace_files: Option<Value>,
77    #[serde(default, skip_serializing_if = "Option::is_none")]
78    pub oracle: Option<EvalOracle>,
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub verify_commands: Option<Vec<VerifyCommand>>,
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub trace_assertions: Option<Value>,
83    #[serde(default, skip_serializing_if = "Option::is_none")]
84    pub outcome_assertions: Option<Value>,
85    #[serde(default)]
86    pub expects_artifact: bool,
87    #[serde(default, skip_serializing_if = "Option::is_none")]
88    pub generated_workspace_files: Option<Value>,
89    #[serde(default, skip_serializing_if = "Option::is_none")]
90    pub workspace_copies: Option<Value>,
91    #[serde(default, skip_serializing_if = "Option::is_none")]
92    pub workspace_patches: Option<Value>,
93}
94
95#[derive(Debug, Clone, Serialize, Deserialize)]
96#[serde(rename_all = "camelCase")]
97pub struct EvalTurn {
98    pub user: String,
99}
100
101#[derive(Debug, Clone, Serialize, Deserialize)]
102#[serde(rename_all = "camelCase")]
103pub struct EvalOracle {
104    pub kind: String,
105    #[serde(default)]
106    pub expected_values: Vec<Value>,
107    #[serde(default)]
108    pub notes: String,
109}
110
111#[derive(Debug, Clone, Serialize, Deserialize)]
112#[serde(rename_all = "camelCase")]
113pub struct VerifyCommand {
114    pub name: String,
115    pub command: String,
116    #[serde(default)]
117    pub expected_exit_code: i32,
118    #[serde(default, skip_serializing_if = "Option::is_none")]
119    pub output_contains: Option<Vec<String>>,
120}
121
122// ---------------------------------------------------------------------------
123// API request/response types
124// ---------------------------------------------------------------------------
125
126#[derive(Debug, Clone, Serialize, Deserialize)]
127#[serde(rename_all = "camelCase")]
128pub struct RunEvalRequest {
129    #[serde(default)]
130    pub case_ids: Vec<String>,
131    #[serde(default)]
132    pub categories: Vec<String>,
133    #[serde(default)]
134    pub runtime_types: Vec<String>,
135    #[serde(default)]
136    pub max_turns: usize,
137    #[serde(default)]
138    pub turn_timeout_ms: u64,
139}
140
141#[derive(Debug, Clone, Serialize, Deserialize)]
142#[serde(rename_all = "camelCase")]
143pub struct EvalReport {
144    pub id: String,
145    pub total_cases: usize,
146    pub passed_cases: usize,
147    pub failed_cases: usize,
148    pub infrastructure_failure_cases: usize,
149    /// Cases whose immutable schema/fixture/evaluator configuration could not
150    /// produce a meaningful quality verdict. Kept separate from provider or
151    /// runtime infrastructure failures.
152    #[serde(default)]
153    pub invalid_case_count: usize,
154    pub non_infrastructure_pass_rate: f64,
155    pub case_results: Vec<EvalCaseResult>,
156    pub created_at: String,
157    pub duration_ms: u64,
158    #[serde(default)]
159    pub total_cost_micros: u64,
160    #[serde(default)]
161    pub p95_case_duration_ms: u64,
162}
163
164#[derive(Debug, Clone, Serialize, Deserialize)]
165#[serde(rename_all = "camelCase")]
166pub struct EvalCaseResult {
167    pub case_id: String,
168    pub category: String,
169    pub passed: bool,
170    pub infrastructure_failure: bool,
171    /// Structured terminal classification for non-quality failures. The
172    /// legacy `infrastructure_failure` boolean remains for wire compatibility.
173    #[serde(default, skip_serializing_if = "Option::is_none")]
174    pub failure_kind: Option<CaseFailureKind>,
175    pub turns_completed: usize,
176    pub total_turns: usize,
177    #[serde(default)]
178    pub oracle_results: Vec<OracleVerdict>,
179    #[serde(default)]
180    pub verify_results: Vec<VerifyCommandResult>,
181    #[serde(default)]
182    pub trace_assertion_results: Vec<AssertionVerdict>,
183    #[serde(default)]
184    pub outcome_assertion_results: Vec<AssertionVerdict>,
185    /// Versioned evaluator outputs. Legacy inline assertions continue to use
186    /// outcome_assertion_results during migration.
187    #[serde(default)]
188    pub evaluator_verdicts: Vec<EvaluatorVerdict>,
189    #[serde(default, skip_serializing_if = "Option::is_none")]
190    pub error: Option<String>,
191    pub duration_ms: u64,
192    #[serde(default)]
193    pub cost_micros: u64,
194    /// Durable experiment identity. Legacy synchronous runs leave these at
195    /// their defaults; durable reports retain them so repeated samples from
196    /// different variants are never conflated as flakiness.
197    #[serde(default)]
198    pub sample_index: u32,
199    #[serde(default, skip_serializing_if = "Option::is_none")]
200    pub variant_id: Option<String>,
201}
202
203#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
204#[serde(rename_all = "snake_case")]
205pub enum CaseFailureKind {
206    InfraFailed,
207    InvalidCase,
208}
209
210#[derive(Debug, Clone, Serialize, Deserialize)]
211#[serde(rename_all = "camelCase")]
212pub struct OracleVerdict {
213    pub expected: String,
214    pub actual: String,
215    pub matched: bool,
216}
217
218#[derive(Debug, Clone, Serialize, Deserialize)]
219#[serde(rename_all = "camelCase")]
220pub struct VerifyCommandResult {
221    pub name: String,
222    pub exit_code: i32,
223    pub expected_exit_code: i32,
224    pub passed: bool,
225    #[serde(default, skip_serializing_if = "Option::is_none")]
226    pub output: Option<String>,
227}
228
229#[derive(Debug, Clone, Serialize, Deserialize)]
230#[serde(rename_all = "camelCase")]
231pub struct AssertionVerdict {
232    pub rule: String,
233    pub passed: bool,
234    #[serde(default, skip_serializing_if = "Option::is_none")]
235    pub detail: Option<String>,
236}
237
238/// Strictly parsed compatibility schema for `outcomeAssertions`.
239///
240/// `EvalCase` retains the original JSON field so v1 wire JSON is unchanged;
241/// evaluators must deserialize it into this type and fail closed on malformed
242/// or unknown rules.
243#[derive(Debug, Clone, Default, Serialize, Deserialize)]
244#[serde(rename_all = "camelCase", deny_unknown_fields)]
245pub struct OutcomeAssertionSpec {
246    #[serde(default)]
247    pub turns: Vec<TurnAssertionSpec>,
248    #[serde(default)]
249    pub workspace_files: Vec<WorkspaceFileAssertionSpec>,
250    #[serde(default)]
251    pub required_changed_paths: Vec<String>,
252    #[serde(default)]
253    pub forbid_workspace_changes: bool,
254}
255
256#[derive(Debug, Clone, Default, Serialize, Deserialize)]
257#[serde(rename_all = "camelCase", deny_unknown_fields)]
258pub struct TurnAssertionSpec {
259    pub index: usize,
260    #[serde(default)]
261    pub contains: Vec<String>,
262    #[serde(default)]
263    pub contains_any: Vec<Vec<String>>,
264}
265
266#[derive(Debug, Clone, Default, Serialize, Deserialize)]
267#[serde(rename_all = "camelCase", deny_unknown_fields)]
268pub struct WorkspaceFileAssertionSpec {
269    pub path: String,
270    #[serde(default)]
271    pub contains: Vec<String>,
272    #[serde(default)]
273    pub contains_any: Vec<Vec<String>>,
274    #[serde(default, skip_serializing_if = "Option::is_none")]
275    pub max_chars: Option<usize>,
276    #[serde(default, skip_serializing_if = "Option::is_none")]
277    pub max_lines: Option<usize>,
278}
279
280/// Versioned typed assertion schema used by durable suites. The legacy
281/// `Value` fields on [`EvalCase`] remain only for v1 wire compatibility.
282#[derive(Debug, Clone, Serialize, Deserialize)]
283#[serde(
284    tag = "kind",
285    rename_all = "snake_case",
286    rename_all_fields = "camelCase",
287    deny_unknown_fields
288)]
289pub enum TypedAssertion {
290    OutputContains {
291        turn: usize,
292        values: Vec<String>,
293        #[serde(default)]
294        any: bool,
295    },
296    WorkspaceFile {
297        path: String,
298        #[serde(default)]
299        contains: Vec<String>,
300        #[serde(default, skip_serializing_if = "Option::is_none")]
301        max_bytes: Option<usize>,
302    },
303    CommandExit {
304        name: String,
305        command_ref: String,
306        expected_exit_code: i32,
307    },
308    TraceEvent {
309        event_type: String,
310        #[serde(default)]
311        minimum_count: usize,
312    },
313    ArtifactExists {
314        artifact_kind: String,
315    },
316    /// A custom evaluator that is executed by the isolated executor. The
317    /// control plane accepts only an attested result for the exact immutable
318    /// plugin reference; it never loads plugin code into the service process.
319    Plugin {
320        plugin_ref: String,
321        #[serde(default)]
322        config: Value,
323    },
324}
325
326/// Immutable reference to a catalog resource. Evaluate snapshots the resolved
327/// version into a CaseAttempt; mutable aliases are deliberately unsupported.
328#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
329#[serde(rename_all = "camelCase", deny_unknown_fields)]
330pub struct VersionedResourceRef {
331    pub id: String,
332    pub version: String,
333}
334
335/// Agent is an optimistic precondition; Runtime ID remains the execution
336/// selector and Runtime is the authoritative owner of their binding.
337#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "camelCase", deny_unknown_fields)]
339pub struct EvaluationSubject {
340    pub agent_id: String,
341    pub runtime_id: String,
342}
343
344#[derive(Debug, Clone, Serialize, Deserialize)]
345#[serde(rename_all = "camelCase", deny_unknown_fields)]
346pub struct RegisterEnvironmentProfileRequest {
347    pub profile_id: String,
348    pub version: String,
349    /// Existing Workspace Environment Space selected for the Task. Provisioned
350    /// environments can be added later behind the same immutable profile.
351    #[serde(default, skip_serializing_if = "Option::is_none")]
352    pub space_id: Option<String>,
353    #[serde(default)]
354    pub metadata: std::collections::BTreeMap<String, String>,
355}
356
357#[derive(Debug, Clone, Serialize, Deserialize)]
358#[serde(rename_all = "camelCase", deny_unknown_fields)]
359pub struct EnvironmentProfileVersion {
360    pub tenant_id: String,
361    pub project_id: String,
362    pub profile_id: String,
363    pub version: String,
364    pub digest: String,
365    #[serde(default, skip_serializing_if = "Option::is_none")]
366    pub space_id: Option<String>,
367    #[serde(default)]
368    pub metadata: std::collections::BTreeMap<String, String>,
369    pub created_at_ms: i64,
370}
371
372#[derive(Debug, Clone, Serialize, Deserialize)]
373#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
374pub enum EvaluatorImplementation {
375    /// Pure, deterministic assertions evaluated from a bounded Evidence view.
376    BuiltinAssertions { assertions: Vec<TypedAssertion> },
377}
378
379#[derive(Debug, Clone, Serialize, Deserialize)]
380#[serde(rename_all = "camelCase", deny_unknown_fields)]
381pub struct RegisterEvaluatorVersionRequest {
382    pub evaluator_id: String,
383    pub version: String,
384    pub implementation: EvaluatorImplementation,
385}
386
387#[derive(Debug, Clone, Serialize, Deserialize)]
388#[serde(rename_all = "camelCase", deny_unknown_fields)]
389pub struct EvaluatorVersion {
390    pub tenant_id: String,
391    pub project_id: String,
392    pub evaluator_id: String,
393    pub version: String,
394    pub digest: String,
395    pub implementation: EvaluatorImplementation,
396    pub created_at_ms: i64,
397}
398
399#[derive(Debug, Clone, Serialize, Deserialize)]
400#[serde(rename_all = "camelCase")]
401pub struct EvaluatorVerdict {
402    pub evaluator_id: String,
403    pub evaluator_version: String,
404    pub passed: bool,
405    #[serde(default)]
406    pub assertions: Vec<AssertionVerdict>,
407}
408
409#[derive(Debug, Clone, Serialize, Deserialize)]
410#[serde(rename_all = "camelCase", deny_unknown_fields)]
411pub struct VersionedEvalCase {
412    pub id: String,
413    pub category: String,
414    #[serde(default)]
415    pub turns: Vec<EvalTurn>,
416    /// Small, immutable text fixtures materialised into the selected Space
417    /// before Runtime executes the case. Large fixtures remain external and
418    /// are referenced through `fixtureRef`.
419    #[serde(default)]
420    pub workspace_files: std::collections::BTreeMap<String, String>,
421    #[serde(default)]
422    pub assertions: Vec<TypedAssertion>,
423    /// Versioned environment selection. Omitted only for legacy suites.
424    #[serde(default, skip_serializing_if = "Option::is_none")]
425    pub environment_profile: Option<VersionedResourceRef>,
426    /// Domain-owned evaluator policies executed after evidence collection.
427    #[serde(default)]
428    pub evaluator_versions: Vec<VersionedResourceRef>,
429    #[serde(default)]
430    pub verify_commands: Vec<VerifyCommand>,
431    #[serde(default, skip_serializing_if = "Option::is_none")]
432    pub fixture_ref: Option<String>,
433    #[serde(default)]
434    pub budget: EvalBudget,
435}
436
437#[derive(Debug, Clone, Serialize, Deserialize)]
438#[serde(rename_all = "camelCase", deny_unknown_fields)]
439pub struct EvalBudget {
440    pub timeout_ms: u64,
441    pub max_output_bytes: usize,
442    pub max_cost_micros: u64,
443}
444
445impl Default for EvalBudget {
446    fn default() -> Self {
447        Self {
448            timeout_ms: 300_000,
449            max_output_bytes: 8 * 1024 * 1024,
450            max_cost_micros: 0,
451        }
452    }
453}
454
455#[derive(Debug, Clone, Serialize, Deserialize)]
456#[serde(rename_all = "camelCase", deny_unknown_fields)]
457pub struct EvalSuiteVersion {
458    pub tenant_id: String,
459    pub project_id: String,
460    pub suite_id: String,
461    pub version: String,
462    pub manifest_digest: String,
463    pub schema_version: u32,
464    pub cases: Vec<VersionedEvalCase>,
465    pub created_at_ms: i64,
466}
467
468#[derive(Debug, Clone, Serialize, Deserialize)]
469#[serde(rename_all = "camelCase", deny_unknown_fields)]
470pub struct RegisterSuiteRequest {
471    pub suite_id: String,
472    pub version: String,
473    pub manifest_digest: String,
474    pub schema_version: u32,
475    pub cases: Vec<VersionedEvalCase>,
476}
477
478#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
479#[serde(rename_all = "snake_case")]
480pub enum OperationState {
481    Queued,
482    Running,
483    Succeeded,
484    Failed,
485    Canceling,
486    Canceled,
487}
488
489#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
490#[serde(rename_all = "snake_case")]
491pub enum CaseRunState {
492    Queued,
493    Running,
494    Passed,
495    QualityFailed,
496    InfraFailed,
497    InvalidCase,
498    Canceled,
499}
500
501#[derive(Debug, Clone, Serialize, Deserialize)]
502#[serde(rename_all = "camelCase", deny_unknown_fields)]
503pub struct CreateEvalRunRequest {
504    pub suite_id: String,
505    pub suite_version: String,
506    pub subject_version: String,
507    pub executor_profile: String,
508    /// Concrete execution target. New Runtime Task based runs require it;
509    /// subject_version remains as a compatibility/report label.
510    #[serde(default, skip_serializing_if = "Option::is_none")]
511    pub subject: Option<EvaluationSubject>,
512    #[serde(default)]
513    pub case_ids: Vec<String>,
514    #[serde(default)]
515    pub seed: u64,
516    #[serde(default = "default_concurrency")]
517    pub concurrency: usize,
518    #[serde(default)]
519    pub idempotency_key: String,
520    /// Independent deterministic samples per `(case, variant)`.
521    #[serde(default = "default_samples")]
522    pub samples: u32,
523    /// Empty preserves the v1 single-subject run. A non-empty matrix expands
524    /// the run without requiring consumers to coordinate many operations.
525    #[serde(default)]
526    pub experiment_matrix: Vec<ExperimentVariant>,
527}
528
529fn default_concurrency() -> usize {
530    8
531}
532
533fn default_samples() -> u32 {
534    1
535}
536
537#[derive(Debug, Clone, Serialize, Deserialize)]
538#[serde(rename_all = "camelCase", deny_unknown_fields)]
539pub struct ExperimentVariant {
540    pub id: String,
541    pub subject_version: String,
542    pub executor_profile: String,
543    #[serde(default)]
544    pub parameters: Value,
545}
546
547#[derive(Debug, Clone, Serialize, Deserialize)]
548#[serde(rename_all = "camelCase")]
549pub struct EvalOperation {
550    pub id: String,
551    pub run_id: String,
552    pub state: OperationState,
553    pub phase: String,
554    pub attempt: u32,
555    pub completed_cases: usize,
556    pub total_cases: usize,
557    #[serde(default, skip_serializing_if = "Option::is_none")]
558    pub report_id: Option<String>,
559    pub created_at_ms: i64,
560    pub updated_at_ms: i64,
561    pub version: i64,
562    #[serde(default, skip_serializing_if = "Option::is_none")]
563    pub next_poll_after_ms: Option<u64>,
564    #[serde(default, skip_serializing_if = "Option::is_none")]
565    pub error_code: Option<String>,
566}
567
568#[derive(Debug, Clone, Serialize, Deserialize)]
569#[serde(rename_all = "camelCase", deny_unknown_fields)]
570pub struct GetOperationRequest {
571    pub operation_id: String,
572}
573
574#[derive(Debug, Clone, Serialize, Deserialize)]
575#[serde(rename_all = "camelCase", deny_unknown_fields)]
576pub struct CancelRunRequest {
577    pub run_id: String,
578}
579
580#[derive(Debug, Clone, Serialize, Deserialize)]
581#[serde(rename_all = "camelCase", deny_unknown_fields)]
582pub struct ListCaseRunsRequest {
583    pub run_id: String,
584    #[serde(default, skip_serializing_if = "Option::is_none")]
585    pub cursor: Option<String>,
586    #[serde(default, skip_serializing_if = "Option::is_none")]
587    pub state: Option<CaseRunState>,
588    #[serde(default = "default_case_page")]
589    pub limit: usize,
590}
591
592fn default_case_page() -> usize {
593    50
594}
595
596#[derive(Debug, Clone, Serialize, Deserialize)]
597#[serde(rename_all = "camelCase")]
598pub struct CaseRunSummary {
599    pub id: String,
600    pub run_id: String,
601    pub case_id: String,
602    pub position: usize,
603    pub state: CaseRunState,
604    pub attempt: u32,
605    pub duration_ms: u64,
606    #[serde(default, skip_serializing_if = "Option::is_none")]
607    pub error: Option<String>,
608    #[serde(default)]
609    pub evidence_refs: Vec<EvidenceRef>,
610    #[serde(default)]
611    pub sample_index: u32,
612    #[serde(default, skip_serializing_if = "Option::is_none")]
613    pub variant_id: Option<String>,
614}
615
616#[derive(Debug, Clone, Serialize, Deserialize)]
617#[serde(rename_all = "camelCase")]
618pub struct CaseRunPage {
619    pub items: Vec<CaseRunSummary>,
620    #[serde(default, skip_serializing_if = "Option::is_none")]
621    pub next_cursor: Option<String>,
622}
623
624#[derive(Debug, Clone, Serialize, Deserialize)]
625#[serde(rename_all = "camelCase", deny_unknown_fields)]
626pub struct EvalAccessScope {
627    pub tenant_id: String,
628    pub project_id: String,
629    pub subject: String,
630}
631
632#[derive(Debug, Clone, Serialize, Deserialize)]
633#[serde(rename_all = "camelCase", deny_unknown_fields)]
634pub struct SandboxExecutionRequest {
635    /// Tenant/project ownership is passed explicitly so the isolated executor
636    /// can only finalize evidence in the namespace later validated by the
637    /// control plane.
638    pub scope: EvalAccessScope,
639    pub operation_id: String,
640    pub run_id: String,
641    pub case_run_id: String,
642    pub subject_version: String,
643    pub executor_profile: String,
644    #[serde(default, skip_serializing_if = "Option::is_none")]
645    pub subject: Option<EvaluationSubject>,
646    #[serde(default, skip_serializing_if = "Option::is_none")]
647    pub environment_profile: Option<EnvironmentProfileVersion>,
648    #[serde(default)]
649    pub evaluator_versions: Vec<EvaluatorVersion>,
650    pub seed: u64,
651    pub case: VersionedEvalCase,
652    pub fencing_token: i64,
653    #[serde(default)]
654    pub sample_index: u32,
655    #[serde(default, skip_serializing_if = "Option::is_none")]
656    pub variant: Option<ExperimentVariant>,
657}
658
659#[derive(Debug, Clone, Serialize, Deserialize)]
660#[serde(rename_all = "camelCase", deny_unknown_fields)]
661pub struct PluginEvaluationEvidence {
662    pub plugin_ref: String,
663    pub passed: bool,
664    #[serde(default)]
665    pub detail: String,
666    /// Attestation produced by the configured sandbox boundary. Production
667    /// evaluation fails closed when it is absent.
668    pub attestation: String,
669}
670
671/// Immutable evidence object owned by the scoped Evaluate namespace.
672#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
673#[serde(rename_all = "camelCase", deny_unknown_fields)]
674pub struct EvidenceRef {
675    pub reference: String,
676    pub tenant_id: String,
677    pub project_id: String,
678    pub run_id: String,
679    pub case_run_id: String,
680    /// Lower-case `sha256:<64 hex>` digest of the finalized bytes.
681    pub digest: String,
682    pub size_bytes: u64,
683    /// Attestation bound to the isolated executor response boundary.
684    pub attestation: String,
685}
686
687/// Bounded evidence returned by the isolated executor. Verdict calculation
688/// remains owned by Evaluate and is deterministic from this typed structure.
689#[derive(Debug, Clone, Default, Serialize, Deserialize)]
690#[serde(rename_all = "camelCase", deny_unknown_fields)]
691pub struct SandboxExecutionEvidence {
692    #[serde(default)]
693    pub responses: Vec<String>,
694    #[serde(default)]
695    pub workspace_files: std::collections::BTreeMap<String, String>,
696    #[serde(default)]
697    pub command_exit_codes: std::collections::BTreeMap<String, i32>,
698    #[serde(default)]
699    pub trace_event_counts: std::collections::BTreeMap<String, usize>,
700    #[serde(default)]
701    pub artifact_kinds: Vec<String>,
702    /// Immutable blob/object references owned by Evaluate's evidence namespace.
703    #[serde(default)]
704    pub evidence_refs: Vec<EvidenceRef>,
705    #[serde(default)]
706    pub plugin_evaluations: Vec<PluginEvaluationEvidence>,
707    #[serde(default, skip_serializing_if = "Option::is_none")]
708    pub infrastructure_error: Option<String>,
709    /// A sandbox may explicitly reject an immutable case as unsupported or
710    /// invalid. This is deliberately typed so provider text is never parsed as
711    /// a terminal-state code.
712    #[serde(default, skip_serializing_if = "Option::is_none")]
713    pub invalid_case_error: Option<String>,
714    #[serde(default)]
715    pub duration_ms: u64,
716    /// Executor-measured cost in millionths of the configured billing unit.
717    #[serde(default)]
718    pub cost_micros: u64,
719}
720
721/// Runtime-facing pull protocol. `evaluate_task_id` is the durable
722/// [`EvalOperation::id`] returned when the SDK/CLI configures a run. Infra
723/// never needs a callback URL for the Runtime.
724#[derive(Debug, Clone, Serialize, Deserialize)]
725#[serde(rename_all = "camelCase", deny_unknown_fields)]
726pub struct ClaimEvaluationCaseRequest {
727    pub evaluate_task_id: String,
728    #[serde(default = "default_driver_lease_ms")]
729    pub lease_ms: i64,
730}
731
732fn default_driver_lease_ms() -> i64 {
733    60_000
734}
735
736#[derive(Debug, Clone, Serialize, Deserialize)]
737#[serde(rename_all = "camelCase", deny_unknown_fields)]
738pub struct EvaluationCaseLease {
739    pub execution: SandboxExecutionRequest,
740    pub lease_until_ms: i64,
741}
742
743#[derive(Debug, Clone, Serialize, Deserialize)]
744#[serde(rename_all = "camelCase", deny_unknown_fields)]
745pub struct ClaimEvaluationCaseResponse {
746    pub operation: EvalOperation,
747    #[serde(default, skip_serializing_if = "Option::is_none")]
748    pub case: Option<EvaluationCaseLease>,
749}
750
751#[derive(Debug, Clone, Serialize, Deserialize)]
752#[serde(rename_all = "camelCase", deny_unknown_fields)]
753pub struct RenewEvaluationCaseRequest {
754    pub evaluate_task_id: String,
755    pub case_run_id: String,
756    pub fencing_token: i64,
757    #[serde(default = "default_driver_lease_ms")]
758    pub lease_ms: i64,
759}
760
761#[derive(Debug, Clone, Serialize, Deserialize)]
762#[serde(rename_all = "camelCase", deny_unknown_fields)]
763pub struct RenewEvaluationCaseResponse {
764    pub renewed: bool,
765    pub lease_until_ms: i64,
766}
767
768#[derive(Debug, Clone, Serialize, Deserialize)]
769#[serde(rename_all = "camelCase", deny_unknown_fields)]
770pub struct SubmitEvaluationEvidenceRequest {
771    pub evaluate_task_id: String,
772    pub case_run_id: String,
773    pub fencing_token: i64,
774    pub evidence: SandboxExecutionEvidence,
775}
776
777#[derive(Debug, Clone, Serialize, Deserialize)]
778#[serde(rename_all = "camelCase", deny_unknown_fields)]
779pub struct SubmitEvaluationEvidenceResponse {
780    pub accepted: bool,
781    pub operation: EvalOperation,
782}
783
784#[derive(Debug, Clone, Serialize, Deserialize)]
785#[serde(rename_all = "camelCase")]
786pub struct ListCasesRequest {
787    #[serde(default, skip_serializing_if = "Option::is_none")]
788    pub category: Option<String>,
789}
790
791#[derive(Debug, Clone, Serialize, Deserialize)]
792#[serde(rename_all = "camelCase")]
793pub struct ListCasesResponse {
794    pub cases: Vec<EvalCaseSummary>,
795}
796
797#[derive(Debug, Clone, Serialize, Deserialize)]
798#[serde(rename_all = "camelCase")]
799pub struct EvalCaseSummary {
800    pub id: String,
801    pub category: String,
802    pub notes: String,
803    pub turns: usize,
804    #[serde(default)]
805    pub has_oracle: bool,
806    #[serde(default)]
807    pub has_verify_commands: bool,
808    #[serde(default)]
809    pub expects_artifact: bool,
810}
811
812#[derive(Debug, Clone, Serialize, Deserialize)]
813#[serde(rename_all = "camelCase")]
814pub struct GetCaseRequest {
815    pub case_id: String,
816}
817
818#[derive(Debug, Clone, Serialize, Deserialize)]
819#[serde(rename_all = "camelCase")]
820pub struct LoadCasesRequest {
821    pub file_path: String,
822}
823
824#[derive(Debug, Clone, Serialize, Deserialize)]
825#[serde(rename_all = "camelCase")]
826pub struct GetReportRequest {
827    pub report_id: String,
828}
829
830#[derive(Debug, Clone, Serialize, Deserialize)]
831#[serde(rename_all = "camelCase", deny_unknown_fields)]
832pub struct CompareReportsRequest {
833    pub report_id: String,
834    pub baseline_report_id: String,
835    #[serde(default)]
836    pub minimum_pass_rate: f64,
837    #[serde(default = "default_max_infra_rate")]
838    pub maximum_infra_failure_rate: f64,
839    #[serde(default = "default_significance")]
840    pub significance_level: f64,
841    #[serde(default)]
842    pub minimum_effect: f64,
843}
844
845fn default_max_infra_rate() -> f64 {
846    1.0
847}
848
849fn default_significance() -> f64 {
850    0.05
851}
852
853#[derive(Debug, Clone, Serialize, Deserialize)]
854#[serde(rename_all = "camelCase")]
855pub struct CaseOutcomeDiff {
856    pub case_id: String,
857    #[serde(default, skip_serializing_if = "Option::is_none")]
858    pub variant_id: Option<String>,
859    pub baseline_passed: bool,
860    pub current_passed: bool,
861    pub flaky_signal: bool,
862    #[serde(default)]
863    pub historical_flake_rate: f64,
864    #[serde(default)]
865    pub historical_samples: u64,
866}
867
868#[derive(Debug, Clone, Serialize, Deserialize)]
869#[serde(rename_all = "camelCase")]
870pub struct ReportComparison {
871    pub report_id: String,
872    pub baseline_report_id: String,
873    pub pass_rate_delta: f64,
874    pub infra_failure_rate_delta: f64,
875    pub pass_rate_confidence_low: f64,
876    pub pass_rate_confidence_high: f64,
877    pub effect_confidence_low: f64,
878    pub effect_confidence_high: f64,
879    pub p_value: f64,
880    pub statistically_significant: bool,
881    pub changed_cases: Vec<CaseOutcomeDiff>,
882    pub gate_passed: bool,
883}
884
885#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
886#[serde(rename_all = "snake_case")]
887pub enum ReviewState {
888    Queued,
889    Claimed,
890    Approved,
891    Rejected,
892}
893
894#[derive(Debug, Clone, Serialize, Deserialize)]
895#[serde(rename_all = "camelCase")]
896pub struct HumanReview {
897    pub id: String,
898    pub run_id: String,
899    pub case_run_id: String,
900    pub state: ReviewState,
901    pub reason: String,
902    #[serde(default, skip_serializing_if = "Option::is_none")]
903    pub reviewer: Option<String>,
904    #[serde(default, skip_serializing_if = "Option::is_none")]
905    pub decision_note: Option<String>,
906    pub fencing_token: i64,
907    pub created_at_ms: i64,
908    pub updated_at_ms: i64,
909}
910
911#[derive(Debug, Clone, Serialize, Deserialize)]
912#[serde(rename_all = "camelCase", deny_unknown_fields)]
913pub struct EnqueueReviewRequest {
914    pub run_id: String,
915    pub case_run_id: String,
916    pub reason: String,
917}
918
919#[derive(Debug, Clone, Serialize, Deserialize)]
920#[serde(rename_all = "camelCase", deny_unknown_fields)]
921pub struct ClaimReviewRequest {
922    #[serde(default = "default_review_lease")]
923    pub lease_ms: i64,
924}
925
926fn default_review_lease() -> i64 {
927    300_000
928}
929
930#[derive(Debug, Clone, Serialize, Deserialize)]
931#[serde(rename_all = "camelCase", deny_unknown_fields)]
932pub struct CompleteReviewRequest {
933    pub review_id: String,
934    pub fencing_token: i64,
935    pub approved: bool,
936    #[serde(default)]
937    pub note: String,
938}
939
940// ---------------------------------------------------------------------------
941// Evaluate administration query model
942// ---------------------------------------------------------------------------
943
944pub const DEFAULT_ADMIN_PAGE_SIZE: usize = 50;
945pub const MAX_ADMIN_PAGE_SIZE: usize = 200;
946
947fn default_admin_page() -> usize {
948    DEFAULT_ADMIN_PAGE_SIZE
949}
950
951#[derive(Debug, Clone, Serialize, Deserialize)]
952#[serde(rename_all = "camelCase", deny_unknown_fields)]
953pub struct ListAdminSuitesRequest {
954    #[serde(default, skip_serializing_if = "Option::is_none")]
955    pub cursor: Option<String>,
956    #[serde(default = "default_admin_page")]
957    pub limit: usize,
958}
959
960impl Default for ListAdminSuitesRequest {
961    fn default() -> Self {
962        Self {
963            cursor: None,
964            limit: DEFAULT_ADMIN_PAGE_SIZE,
965        }
966    }
967}
968
969#[derive(Debug, Clone, Serialize, Deserialize)]
970#[serde(rename_all = "camelCase", deny_unknown_fields)]
971pub struct ListAdminRunsRequest {
972    #[serde(default, skip_serializing_if = "Option::is_none")]
973    pub cursor: Option<String>,
974    #[serde(default, skip_serializing_if = "Option::is_none")]
975    pub state: Option<OperationState>,
976    #[serde(default = "default_admin_page")]
977    pub limit: usize,
978}
979
980impl Default for ListAdminRunsRequest {
981    fn default() -> Self {
982        Self {
983            cursor: None,
984            state: None,
985            limit: DEFAULT_ADMIN_PAGE_SIZE,
986        }
987    }
988}
989
990#[derive(Debug, Clone, Serialize, Deserialize)]
991#[serde(rename_all = "camelCase", deny_unknown_fields)]
992pub struct ListAdminReviewsRequest {
993    #[serde(default, skip_serializing_if = "Option::is_none")]
994    pub cursor: Option<String>,
995    #[serde(default, skip_serializing_if = "Option::is_none")]
996    pub state: Option<ReviewState>,
997    #[serde(default = "default_admin_page")]
998    pub limit: usize,
999}
1000
1001impl Default for ListAdminReviewsRequest {
1002    fn default() -> Self {
1003        Self {
1004            cursor: None,
1005            state: None,
1006            limit: DEFAULT_ADMIN_PAGE_SIZE,
1007        }
1008    }
1009}
1010
1011#[derive(Debug, Clone, Serialize, Deserialize)]
1012#[serde(rename_all = "camelCase")]
1013pub struct EvaluateAdminOverview {
1014    pub suite_versions: u64,
1015    pub run_states: std::collections::BTreeMap<String, u64>,
1016    pub case_states: std::collections::BTreeMap<String, u64>,
1017    pub review_states: std::collections::BTreeMap<String, u64>,
1018}
1019
1020#[derive(Debug, Clone, Serialize, Deserialize)]
1021#[serde(rename_all = "camelCase")]
1022pub struct AdminSuiteSummary {
1023    pub suite_id: String,
1024    pub version: String,
1025    pub manifest_digest: String,
1026    pub schema_version: u32,
1027    pub case_count: usize,
1028    pub created_at_ms: i64,
1029}
1030
1031#[derive(Debug, Clone, Serialize, Deserialize)]
1032#[serde(rename_all = "camelCase")]
1033pub struct AdminSuitePage {
1034    pub items: Vec<AdminSuiteSummary>,
1035    #[serde(default, skip_serializing_if = "Option::is_none")]
1036    pub next_cursor: Option<String>,
1037}
1038
1039#[derive(Debug, Clone, Serialize, Deserialize)]
1040#[serde(rename_all = "camelCase")]
1041pub struct AdminRunSummary {
1042    pub run_id: String,
1043    pub operation_id: String,
1044    pub suite_id: String,
1045    pub suite_version: String,
1046    pub subject_version: String,
1047    pub executor_profile: String,
1048    #[serde(default, skip_serializing_if = "Option::is_none")]
1049    pub subject: Option<EvaluationSubject>,
1050    pub state: OperationState,
1051    pub phase: String,
1052    pub completed_cases: usize,
1053    pub total_cases: usize,
1054    #[serde(default, skip_serializing_if = "Option::is_none")]
1055    pub report_id: Option<String>,
1056    pub created_at_ms: i64,
1057    pub updated_at_ms: i64,
1058    pub version: i64,
1059}
1060
1061#[derive(Debug, Clone, Serialize, Deserialize)]
1062#[serde(rename_all = "camelCase")]
1063pub struct AdminRunPage {
1064    pub items: Vec<AdminRunSummary>,
1065    #[serde(default, skip_serializing_if = "Option::is_none")]
1066    pub next_cursor: Option<String>,
1067}
1068
1069#[derive(Debug, Clone, Serialize, Deserialize)]
1070#[serde(rename_all = "camelCase")]
1071pub struct AdminReviewPage {
1072    pub items: Vec<HumanReview>,
1073    #[serde(default, skip_serializing_if = "Option::is_none")]
1074    pub next_cursor: Option<String>,
1075}
1076
1077/// Error body shared by every non-2xx response.
1078#[derive(Debug, Clone, Serialize, Deserialize)]
1079pub struct ApiError {
1080    pub status: u16,
1081    pub message: String,
1082}
1083
1084#[cfg(test)]
1085mod tests {
1086    use super::{
1087        DEFAULT_ADMIN_PAGE_SIZE, ListAdminReviewsRequest, ListAdminRunsRequest,
1088        ListAdminSuitesRequest, TypedAssertion,
1089    };
1090    use serde_json::json;
1091
1092    #[test]
1093    fn typed_assertion_uses_snake_case_tags_and_camel_case_fields() {
1094        let assertion = TypedAssertion::TraceEvent {
1095            event_type: "task.completed".into(),
1096            minimum_count: 1,
1097        };
1098        let value = serde_json::to_value(&assertion).unwrap();
1099        assert_eq!(
1100            value,
1101            json!({
1102                "kind": "trace_event",
1103                "eventType": "task.completed",
1104                "minimumCount": 1
1105            })
1106        );
1107        assert!(serde_json::from_value::<TypedAssertion>(value).is_ok());
1108    }
1109
1110    #[test]
1111    fn admin_queries_use_the_same_bounded_default_in_rust_and_serde() {
1112        assert_eq!(
1113            ListAdminSuitesRequest::default().limit,
1114            DEFAULT_ADMIN_PAGE_SIZE
1115        );
1116        assert_eq!(
1117            ListAdminRunsRequest::default().limit,
1118            DEFAULT_ADMIN_PAGE_SIZE
1119        );
1120        assert_eq!(
1121            ListAdminReviewsRequest::default().limit,
1122            DEFAULT_ADMIN_PAGE_SIZE
1123        );
1124        let decoded: ListAdminRunsRequest = serde_json::from_value(json!({})).unwrap();
1125        assert_eq!(decoded.limit, DEFAULT_ADMIN_PAGE_SIZE);
1126    }
1127}
agent_evaluate_contract/lib.rs

agent_evaluate_contract/
lib.rs
