agent_evaluate_contract/
lib.rs

1//! HTTP API contract for the Evaluate service.
2//!
3//! This service provides a programmatic interface for running agent evaluation
4//! cases, collecting metrics, and producing reports. The eval harness replays
5//! predefined conversation turns against an agent runtime instance and judges
6//! outcomes against oracles.
7//!
8//! # Layering
9//!
10//! ```text
11//! agent-evaluate-contract   paths + DTOs        (this crate)
12//! agent-evaluate-runtime    eval runner + judge
13//! agent-evaluate-server     router + auth + bin
14//! ```
15
16use serde::{Deserialize, Serialize};
17use serde_json::Value;
18
19pub const SERVICE_NAME: &str = "agent-evaluate";
20pub const DEFAULT_PORT: u16 = 5150;
21pub const AUDIENCE: &str = "agent-evaluate";
22pub const SERVICE_TOKEN_ENV: &str = "AGENT_EVALUATE_SERVICE_TOKEN";
23pub const MAX_RESOURCE_ID_CHARS: usize = 191;
24
25pub const HEALTH_PATH: &str = "/health";
26pub const RUN_EVAL_PATH: &str = "/internal/v1/evaluate/run";
27pub const LIST_CASES_PATH: &str = "/internal/v1/evaluate/cases";
28pub const GET_CASE_PATH: &str = "/internal/v1/evaluate/cases/get";
29pub const GET_REPORT_PATH: &str = "/internal/v1/evaluate/reports";
30pub const LOAD_CASES_PATH: &str = "/internal/v1/evaluate/cases/load";
31pub const SUITES_PATH: &str = "/internal/v1/evaluate/suites";
32pub const RUNS_PATH: &str = "/internal/v1/evaluate/runs";
33pub const OPERATIONS_PATH: &str = "/internal/v1/evaluate/operations";
34pub const RUN_CASES_PATH: &str = "/internal/v1/evaluate/runs/cases";
35pub const CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/runs/cancel";
36pub const DURABLE_REPORT_PATH: &str = "/internal/v1/evaluate/reports/get";
37pub const COMPARE_REPORTS_PATH: &str = "/internal/v1/evaluate/reports/compare";
38pub const REVIEW_QUEUE_PATH: &str = "/internal/v1/evaluate/reviews";
39pub const CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/claim";
40pub const COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/complete";
41pub const ENVIRONMENT_PROFILES_PATH: &str = "/internal/v1/evaluate/environment-profiles";
42pub const EVALUATOR_VERSIONS_PATH: &str = "/internal/v1/evaluate/evaluator-versions";
43pub const DRIVER_CLAIM_PATH: &str = "/internal/v1/evaluate/driver/cases/claim";
44pub const DRIVER_RENEW_PATH: &str = "/internal/v1/evaluate/driver/cases/renew";
45pub const DRIVER_SUBMIT_PATH: &str = "/internal/v1/evaluate/driver/cases/submit";
46
47// ---------------------------------------------------------------------------
48// Eval case types
49// ---------------------------------------------------------------------------
50
51#[derive(Debug, Clone, Serialize, Deserialize)]
52#[serde(rename_all = "camelCase")]
53pub struct EvalCaseFile {
54    pub schema_version: i32,
55    pub cases: Vec<EvalCase>,
56}
57
58#[derive(Debug, Clone, Serialize, Deserialize)]
59#[serde(rename_all = "camelCase")]
60pub struct EvalCase {
61    pub id: String,
62    pub category: String,
63    #[serde(default)]
64    pub notes: String,
65    pub turns: Vec<EvalTurn>,
66    #[serde(default, skip_serializing_if = "Option::is_none")]
67    pub workspace_files: Option<Value>,
68    #[serde(default, skip_serializing_if = "Option::is_none")]
69    pub oracle: Option<EvalOracle>,
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub verify_commands: Option<Vec<VerifyCommand>>,
72    #[serde(default, skip_serializing_if = "Option::is_none")]
73    pub trace_assertions: Option<Value>,
74    #[serde(default, skip_serializing_if = "Option::is_none")]
75    pub outcome_assertions: Option<Value>,
76    #[serde(default)]
77    pub expects_artifact: bool,
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub generated_workspace_files: Option<Value>,
80    #[serde(default, skip_serializing_if = "Option::is_none")]
81    pub workspace_copies: Option<Value>,
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub workspace_patches: Option<Value>,
84}
85
86#[derive(Debug, Clone, Serialize, Deserialize)]
87#[serde(rename_all = "camelCase")]
88pub struct EvalTurn {
89    pub user: String,
90}
91
92#[derive(Debug, Clone, Serialize, Deserialize)]
93#[serde(rename_all = "camelCase")]
94pub struct EvalOracle {
95    pub kind: String,
96    #[serde(default)]
97    pub expected_values: Vec<Value>,
98    #[serde(default)]
99    pub notes: String,
100}
101
102#[derive(Debug, Clone, Serialize, Deserialize)]
103#[serde(rename_all = "camelCase")]
104pub struct VerifyCommand {
105    pub name: String,
106    pub command: String,
107    #[serde(default)]
108    pub expected_exit_code: i32,
109    #[serde(default, skip_serializing_if = "Option::is_none")]
110    pub output_contains: Option<Vec<String>>,
111}
112
113// ---------------------------------------------------------------------------
114// API request/response types
115// ---------------------------------------------------------------------------
116
117#[derive(Debug, Clone, Serialize, Deserialize)]
118#[serde(rename_all = "camelCase")]
119pub struct RunEvalRequest {
120    #[serde(default)]
121    pub case_ids: Vec<String>,
122    #[serde(default)]
123    pub categories: Vec<String>,
124    #[serde(default)]
125    pub runtime_types: Vec<String>,
126    #[serde(default)]
127    pub max_turns: usize,
128    #[serde(default)]
129    pub turn_timeout_ms: u64,
130}
131
132#[derive(Debug, Clone, Serialize, Deserialize)]
133#[serde(rename_all = "camelCase")]
134pub struct EvalReport {
135    pub id: String,
136    pub total_cases: usize,
137    pub passed_cases: usize,
138    pub failed_cases: usize,
139    pub infrastructure_failure_cases: usize,
140    /// Cases whose immutable schema/fixture/evaluator configuration could not
141    /// produce a meaningful quality verdict. Kept separate from provider or
142    /// runtime infrastructure failures.
143    #[serde(default)]
144    pub invalid_case_count: usize,
145    pub non_infrastructure_pass_rate: f64,
146    pub case_results: Vec<EvalCaseResult>,
147    pub created_at: String,
148    pub duration_ms: u64,
149    #[serde(default)]
150    pub total_cost_micros: u64,
151    #[serde(default)]
152    pub p95_case_duration_ms: u64,
153}
154
155#[derive(Debug, Clone, Serialize, Deserialize)]
156#[serde(rename_all = "camelCase")]
157pub struct EvalCaseResult {
158    pub case_id: String,
159    pub category: String,
160    pub passed: bool,
161    pub infrastructure_failure: bool,
162    /// Structured terminal classification for non-quality failures. The
163    /// legacy `infrastructure_failure` boolean remains for wire compatibility.
164    #[serde(default, skip_serializing_if = "Option::is_none")]
165    pub failure_kind: Option<CaseFailureKind>,
166    pub turns_completed: usize,
167    pub total_turns: usize,
168    #[serde(default)]
169    pub oracle_results: Vec<OracleVerdict>,
170    #[serde(default)]
171    pub verify_results: Vec<VerifyCommandResult>,
172    #[serde(default)]
173    pub trace_assertion_results: Vec<AssertionVerdict>,
174    #[serde(default)]
175    pub outcome_assertion_results: Vec<AssertionVerdict>,
176    /// Versioned evaluator outputs. Legacy inline assertions continue to use
177    /// outcome_assertion_results during migration.
178    #[serde(default)]
179    pub evaluator_verdicts: Vec<EvaluatorVerdict>,
180    #[serde(default, skip_serializing_if = "Option::is_none")]
181    pub error: Option<String>,
182    pub duration_ms: u64,
183    #[serde(default)]
184    pub cost_micros: u64,
185    /// Durable experiment identity. Legacy synchronous runs leave these at
186    /// their defaults; durable reports retain them so repeated samples from
187    /// different variants are never conflated as flakiness.
188    #[serde(default)]
189    pub sample_index: u32,
190    #[serde(default, skip_serializing_if = "Option::is_none")]
191    pub variant_id: Option<String>,
192}
193
194#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
195#[serde(rename_all = "snake_case")]
196pub enum CaseFailureKind {
197    InfraFailed,
198    InvalidCase,
199}
200
201#[derive(Debug, Clone, Serialize, Deserialize)]
202#[serde(rename_all = "camelCase")]
203pub struct OracleVerdict {
204    pub expected: String,
205    pub actual: String,
206    pub matched: bool,
207}
208
209#[derive(Debug, Clone, Serialize, Deserialize)]
210#[serde(rename_all = "camelCase")]
211pub struct VerifyCommandResult {
212    pub name: String,
213    pub exit_code: i32,
214    pub expected_exit_code: i32,
215    pub passed: bool,
216    #[serde(default, skip_serializing_if = "Option::is_none")]
217    pub output: Option<String>,
218}
219
220#[derive(Debug, Clone, Serialize, Deserialize)]
221#[serde(rename_all = "camelCase")]
222pub struct AssertionVerdict {
223    pub rule: String,
224    pub passed: bool,
225    #[serde(default, skip_serializing_if = "Option::is_none")]
226    pub detail: Option<String>,
227}
228
229/// Strictly parsed compatibility schema for `outcomeAssertions`.
230///
231/// `EvalCase` retains the original JSON field so v1 wire JSON is unchanged;
232/// evaluators must deserialize it into this type and fail closed on malformed
233/// or unknown rules.
234#[derive(Debug, Clone, Default, Serialize, Deserialize)]
235#[serde(rename_all = "camelCase", deny_unknown_fields)]
236pub struct OutcomeAssertionSpec {
237    #[serde(default)]
238    pub turns: Vec<TurnAssertionSpec>,
239    #[serde(default)]
240    pub workspace_files: Vec<WorkspaceFileAssertionSpec>,
241    #[serde(default)]
242    pub required_changed_paths: Vec<String>,
243    #[serde(default)]
244    pub forbid_workspace_changes: bool,
245}
246
247#[derive(Debug, Clone, Default, Serialize, Deserialize)]
248#[serde(rename_all = "camelCase", deny_unknown_fields)]
249pub struct TurnAssertionSpec {
250    pub index: usize,
251    #[serde(default)]
252    pub contains: Vec<String>,
253    #[serde(default)]
254    pub contains_any: Vec<Vec<String>>,
255}
256
257#[derive(Debug, Clone, Default, Serialize, Deserialize)]
258#[serde(rename_all = "camelCase", deny_unknown_fields)]
259pub struct WorkspaceFileAssertionSpec {
260    pub path: String,
261    #[serde(default)]
262    pub contains: Vec<String>,
263    #[serde(default)]
264    pub contains_any: Vec<Vec<String>>,
265    #[serde(default, skip_serializing_if = "Option::is_none")]
266    pub max_chars: Option<usize>,
267    #[serde(default, skip_serializing_if = "Option::is_none")]
268    pub max_lines: Option<usize>,
269}
270
271/// Versioned typed assertion schema used by durable suites. The legacy
272/// `Value` fields on [`EvalCase`] remain only for v1 wire compatibility.
273#[derive(Debug, Clone, Serialize, Deserialize)]
274#[serde(
275    tag = "kind",
276    rename_all = "snake_case",
277    rename_all_fields = "camelCase",
278    deny_unknown_fields
279)]
280pub enum TypedAssertion {
281    OutputContains {
282        turn: usize,
283        values: Vec<String>,
284        #[serde(default)]
285        any: bool,
286    },
287    WorkspaceFile {
288        path: String,
289        #[serde(default)]
290        contains: Vec<String>,
291        #[serde(default, skip_serializing_if = "Option::is_none")]
292        max_bytes: Option<usize>,
293    },
294    CommandExit {
295        name: String,
296        command_ref: String,
297        expected_exit_code: i32,
298    },
299    TraceEvent {
300        event_type: String,
301        #[serde(default)]
302        minimum_count: usize,
303    },
304    ArtifactExists {
305        artifact_kind: String,
306    },
307    /// A custom evaluator that is executed by the isolated executor. The
308    /// control plane accepts only an attested result for the exact immutable
309    /// plugin reference; it never loads plugin code into the service process.
310    Plugin {
311        plugin_ref: String,
312        #[serde(default)]
313        config: Value,
314    },
315}
316
317/// Immutable reference to a catalog resource. Evaluate snapshots the resolved
318/// version into a CaseAttempt; mutable aliases are deliberately unsupported.
319#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
320#[serde(rename_all = "camelCase", deny_unknown_fields)]
321pub struct VersionedResourceRef {
322    pub id: String,
323    pub version: String,
324}
325
326/// Agent is an optimistic precondition; Runtime ID remains the execution
327/// selector and Runtime is the authoritative owner of their binding.
328#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
329#[serde(rename_all = "camelCase", deny_unknown_fields)]
330pub struct EvaluationSubject {
331    pub agent_id: String,
332    pub runtime_id: String,
333}
334
335#[derive(Debug, Clone, Serialize, Deserialize)]
336#[serde(rename_all = "camelCase", deny_unknown_fields)]
337pub struct RegisterEnvironmentProfileRequest {
338    pub profile_id: String,
339    pub version: String,
340    /// Existing Workspace Environment Space selected for the Task. Provisioned
341    /// environments can be added later behind the same immutable profile.
342    #[serde(default, skip_serializing_if = "Option::is_none")]
343    pub space_id: Option<String>,
344    #[serde(default)]
345    pub metadata: std::collections::BTreeMap<String, String>,
346}
347
348#[derive(Debug, Clone, Serialize, Deserialize)]
349#[serde(rename_all = "camelCase", deny_unknown_fields)]
350pub struct EnvironmentProfileVersion {
351    pub tenant_id: String,
352    pub project_id: String,
353    pub profile_id: String,
354    pub version: String,
355    pub digest: String,
356    #[serde(default, skip_serializing_if = "Option::is_none")]
357    pub space_id: Option<String>,
358    #[serde(default)]
359    pub metadata: std::collections::BTreeMap<String, String>,
360    pub created_at_ms: i64,
361}
362
363#[derive(Debug, Clone, Serialize, Deserialize)]
364#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
365pub enum EvaluatorImplementation {
366    /// Pure, deterministic assertions evaluated from a bounded Evidence view.
367    BuiltinAssertions { assertions: Vec<TypedAssertion> },
368}
369
370#[derive(Debug, Clone, Serialize, Deserialize)]
371#[serde(rename_all = "camelCase", deny_unknown_fields)]
372pub struct RegisterEvaluatorVersionRequest {
373    pub evaluator_id: String,
374    pub version: String,
375    pub implementation: EvaluatorImplementation,
376}
377
378#[derive(Debug, Clone, Serialize, Deserialize)]
379#[serde(rename_all = "camelCase", deny_unknown_fields)]
380pub struct EvaluatorVersion {
381    pub tenant_id: String,
382    pub project_id: String,
383    pub evaluator_id: String,
384    pub version: String,
385    pub digest: String,
386    pub implementation: EvaluatorImplementation,
387    pub created_at_ms: i64,
388}
389
390#[derive(Debug, Clone, Serialize, Deserialize)]
391#[serde(rename_all = "camelCase")]
392pub struct EvaluatorVerdict {
393    pub evaluator_id: String,
394    pub evaluator_version: String,
395    pub passed: bool,
396    #[serde(default)]
397    pub assertions: Vec<AssertionVerdict>,
398}
399
400#[derive(Debug, Clone, Serialize, Deserialize)]
401#[serde(rename_all = "camelCase", deny_unknown_fields)]
402pub struct VersionedEvalCase {
403    pub id: String,
404    pub category: String,
405    #[serde(default)]
406    pub turns: Vec<EvalTurn>,
407    /// Small, immutable text fixtures materialised into the selected Space
408    /// before Runtime executes the case. Large fixtures remain external and
409    /// are referenced through `fixtureRef`.
410    #[serde(default)]
411    pub workspace_files: std::collections::BTreeMap<String, String>,
412    #[serde(default)]
413    pub assertions: Vec<TypedAssertion>,
414    /// Versioned environment selection. Omitted only for legacy suites.
415    #[serde(default, skip_serializing_if = "Option::is_none")]
416    pub environment_profile: Option<VersionedResourceRef>,
417    /// Domain-owned evaluator policies executed after evidence collection.
418    #[serde(default)]
419    pub evaluator_versions: Vec<VersionedResourceRef>,
420    #[serde(default)]
421    pub verify_commands: Vec<VerifyCommand>,
422    #[serde(default, skip_serializing_if = "Option::is_none")]
423    pub fixture_ref: Option<String>,
424    #[serde(default)]
425    pub budget: EvalBudget,
426}
427
428#[derive(Debug, Clone, Serialize, Deserialize)]
429#[serde(rename_all = "camelCase", deny_unknown_fields)]
430pub struct EvalBudget {
431    pub timeout_ms: u64,
432    pub max_output_bytes: usize,
433    pub max_cost_micros: u64,
434}
435
436impl Default for EvalBudget {
437    fn default() -> Self {
438        Self {
439            timeout_ms: 300_000,
440            max_output_bytes: 8 * 1024 * 1024,
441            max_cost_micros: 0,
442        }
443    }
444}
445
446#[derive(Debug, Clone, Serialize, Deserialize)]
447#[serde(rename_all = "camelCase", deny_unknown_fields)]
448pub struct EvalSuiteVersion {
449    pub tenant_id: String,
450    pub project_id: String,
451    pub suite_id: String,
452    pub version: String,
453    pub manifest_digest: String,
454    pub schema_version: u32,
455    pub cases: Vec<VersionedEvalCase>,
456    pub created_at_ms: i64,
457}
458
459#[derive(Debug, Clone, Serialize, Deserialize)]
460#[serde(rename_all = "camelCase", deny_unknown_fields)]
461pub struct RegisterSuiteRequest {
462    pub suite_id: String,
463    pub version: String,
464    pub manifest_digest: String,
465    pub schema_version: u32,
466    pub cases: Vec<VersionedEvalCase>,
467}
468
469#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
470#[serde(rename_all = "snake_case")]
471pub enum OperationState {
472    Queued,
473    Running,
474    Succeeded,
475    Failed,
476    Canceling,
477    Canceled,
478}
479
480#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
481#[serde(rename_all = "snake_case")]
482pub enum CaseRunState {
483    Queued,
484    Running,
485    Passed,
486    QualityFailed,
487    InfraFailed,
488    InvalidCase,
489    Canceled,
490}
491
492#[derive(Debug, Clone, Serialize, Deserialize)]
493#[serde(rename_all = "camelCase", deny_unknown_fields)]
494pub struct CreateEvalRunRequest {
495    pub suite_id: String,
496    pub suite_version: String,
497    pub subject_version: String,
498    pub executor_profile: String,
499    /// Concrete execution target. New Runtime Task based runs require it;
500    /// subject_version remains as a compatibility/report label.
501    #[serde(default, skip_serializing_if = "Option::is_none")]
502    pub subject: Option<EvaluationSubject>,
503    #[serde(default)]
504    pub case_ids: Vec<String>,
505    #[serde(default)]
506    pub seed: u64,
507    #[serde(default = "default_concurrency")]
508    pub concurrency: usize,
509    #[serde(default)]
510    pub idempotency_key: String,
511    /// Independent deterministic samples per `(case, variant)`.
512    #[serde(default = "default_samples")]
513    pub samples: u32,
514    /// Empty preserves the v1 single-subject run. A non-empty matrix expands
515    /// the run without requiring consumers to coordinate many operations.
516    #[serde(default)]
517    pub experiment_matrix: Vec<ExperimentVariant>,
518}
519
520fn default_concurrency() -> usize {
521    8
522}
523
524fn default_samples() -> u32 {
525    1
526}
527
528#[derive(Debug, Clone, Serialize, Deserialize)]
529#[serde(rename_all = "camelCase", deny_unknown_fields)]
530pub struct ExperimentVariant {
531    pub id: String,
532    pub subject_version: String,
533    pub executor_profile: String,
534    #[serde(default)]
535    pub parameters: Value,
536}
537
538#[derive(Debug, Clone, Serialize, Deserialize)]
539#[serde(rename_all = "camelCase")]
540pub struct EvalOperation {
541    pub id: String,
542    pub run_id: String,
543    pub state: OperationState,
544    pub phase: String,
545    pub attempt: u32,
546    pub completed_cases: usize,
547    pub total_cases: usize,
548    #[serde(default, skip_serializing_if = "Option::is_none")]
549    pub report_id: Option<String>,
550    pub created_at_ms: i64,
551    pub updated_at_ms: i64,
552    pub version: i64,
553    #[serde(default, skip_serializing_if = "Option::is_none")]
554    pub next_poll_after_ms: Option<u64>,
555    #[serde(default, skip_serializing_if = "Option::is_none")]
556    pub error_code: Option<String>,
557}
558
559#[derive(Debug, Clone, Serialize, Deserialize)]
560#[serde(rename_all = "camelCase", deny_unknown_fields)]
561pub struct GetOperationRequest {
562    pub operation_id: String,
563}
564
565#[derive(Debug, Clone, Serialize, Deserialize)]
566#[serde(rename_all = "camelCase", deny_unknown_fields)]
567pub struct CancelRunRequest {
568    pub run_id: String,
569}
570
571#[derive(Debug, Clone, Serialize, Deserialize)]
572#[serde(rename_all = "camelCase", deny_unknown_fields)]
573pub struct ListCaseRunsRequest {
574    pub run_id: String,
575    #[serde(default, skip_serializing_if = "Option::is_none")]
576    pub cursor: Option<String>,
577    #[serde(default, skip_serializing_if = "Option::is_none")]
578    pub state: Option<CaseRunState>,
579    #[serde(default = "default_case_page")]
580    pub limit: usize,
581}
582
583fn default_case_page() -> usize {
584    50
585}
586
587#[derive(Debug, Clone, Serialize, Deserialize)]
588#[serde(rename_all = "camelCase")]
589pub struct CaseRunSummary {
590    pub id: String,
591    pub run_id: String,
592    pub case_id: String,
593    pub position: usize,
594    pub state: CaseRunState,
595    pub attempt: u32,
596    pub duration_ms: u64,
597    #[serde(default, skip_serializing_if = "Option::is_none")]
598    pub error: Option<String>,
599    #[serde(default)]
600    pub evidence_refs: Vec<EvidenceRef>,
601    #[serde(default)]
602    pub sample_index: u32,
603    #[serde(default, skip_serializing_if = "Option::is_none")]
604    pub variant_id: Option<String>,
605}
606
607#[derive(Debug, Clone, Serialize, Deserialize)]
608#[serde(rename_all = "camelCase")]
609pub struct CaseRunPage {
610    pub items: Vec<CaseRunSummary>,
611    #[serde(default, skip_serializing_if = "Option::is_none")]
612    pub next_cursor: Option<String>,
613}
614
615#[derive(Debug, Clone, Serialize, Deserialize)]
616#[serde(rename_all = "camelCase", deny_unknown_fields)]
617pub struct EvalAccessScope {
618    pub tenant_id: String,
619    pub project_id: String,
620    pub subject: String,
621}
622
623#[derive(Debug, Clone, Serialize, Deserialize)]
624#[serde(rename_all = "camelCase", deny_unknown_fields)]
625pub struct SandboxExecutionRequest {
626    /// Tenant/project ownership is passed explicitly so the isolated executor
627    /// can only finalize evidence in the namespace later validated by the
628    /// control plane.
629    pub scope: EvalAccessScope,
630    pub operation_id: String,
631    pub run_id: String,
632    pub case_run_id: String,
633    pub subject_version: String,
634    pub executor_profile: String,
635    #[serde(default, skip_serializing_if = "Option::is_none")]
636    pub subject: Option<EvaluationSubject>,
637    #[serde(default, skip_serializing_if = "Option::is_none")]
638    pub environment_profile: Option<EnvironmentProfileVersion>,
639    #[serde(default)]
640    pub evaluator_versions: Vec<EvaluatorVersion>,
641    pub seed: u64,
642    pub case: VersionedEvalCase,
643    pub fencing_token: i64,
644    #[serde(default)]
645    pub sample_index: u32,
646    #[serde(default, skip_serializing_if = "Option::is_none")]
647    pub variant: Option<ExperimentVariant>,
648}
649
650#[derive(Debug, Clone, Serialize, Deserialize)]
651#[serde(rename_all = "camelCase", deny_unknown_fields)]
652pub struct PluginEvaluationEvidence {
653    pub plugin_ref: String,
654    pub passed: bool,
655    #[serde(default)]
656    pub detail: String,
657    /// Attestation produced by the configured sandbox boundary. Production
658    /// evaluation fails closed when it is absent.
659    pub attestation: String,
660}
661
662/// Immutable evidence object owned by the scoped Evaluate namespace.
663#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
664#[serde(rename_all = "camelCase", deny_unknown_fields)]
665pub struct EvidenceRef {
666    pub reference: String,
667    pub tenant_id: String,
668    pub project_id: String,
669    pub run_id: String,
670    pub case_run_id: String,
671    /// Lower-case `sha256:<64 hex>` digest of the finalized bytes.
672    pub digest: String,
673    pub size_bytes: u64,
674    /// Attestation bound to the isolated executor response boundary.
675    pub attestation: String,
676}
677
678/// Bounded evidence returned by the isolated executor. Verdict calculation
679/// remains owned by Evaluate and is deterministic from this typed structure.
680#[derive(Debug, Clone, Default, Serialize, Deserialize)]
681#[serde(rename_all = "camelCase", deny_unknown_fields)]
682pub struct SandboxExecutionEvidence {
683    #[serde(default)]
684    pub responses: Vec<String>,
685    #[serde(default)]
686    pub workspace_files: std::collections::BTreeMap<String, String>,
687    #[serde(default)]
688    pub command_exit_codes: std::collections::BTreeMap<String, i32>,
689    #[serde(default)]
690    pub trace_event_counts: std::collections::BTreeMap<String, usize>,
691    #[serde(default)]
692    pub artifact_kinds: Vec<String>,
693    /// Immutable blob/object references owned by Evaluate's evidence namespace.
694    #[serde(default)]
695    pub evidence_refs: Vec<EvidenceRef>,
696    #[serde(default)]
697    pub plugin_evaluations: Vec<PluginEvaluationEvidence>,
698    #[serde(default, skip_serializing_if = "Option::is_none")]
699    pub infrastructure_error: Option<String>,
700    /// A sandbox may explicitly reject an immutable case as unsupported or
701    /// invalid. This is deliberately typed so provider text is never parsed as
702    /// a terminal-state code.
703    #[serde(default, skip_serializing_if = "Option::is_none")]
704    pub invalid_case_error: Option<String>,
705    #[serde(default)]
706    pub duration_ms: u64,
707    /// Executor-measured cost in millionths of the configured billing unit.
708    #[serde(default)]
709    pub cost_micros: u64,
710}
711
712/// Runtime-facing pull protocol. `evaluate_task_id` is the durable
713/// [`EvalOperation::id`] returned when the SDK/CLI configures a run. Infra
714/// never needs a callback URL for the Runtime.
715#[derive(Debug, Clone, Serialize, Deserialize)]
716#[serde(rename_all = "camelCase", deny_unknown_fields)]
717pub struct ClaimEvaluationCaseRequest {
718    pub evaluate_task_id: String,
719    #[serde(default = "default_driver_lease_ms")]
720    pub lease_ms: i64,
721}
722
723fn default_driver_lease_ms() -> i64 {
724    60_000
725}
726
727#[derive(Debug, Clone, Serialize, Deserialize)]
728#[serde(rename_all = "camelCase", deny_unknown_fields)]
729pub struct EvaluationCaseLease {
730    pub execution: SandboxExecutionRequest,
731    pub lease_until_ms: i64,
732}
733
734#[derive(Debug, Clone, Serialize, Deserialize)]
735#[serde(rename_all = "camelCase", deny_unknown_fields)]
736pub struct ClaimEvaluationCaseResponse {
737    pub operation: EvalOperation,
738    #[serde(default, skip_serializing_if = "Option::is_none")]
739    pub case: Option<EvaluationCaseLease>,
740}
741
742#[derive(Debug, Clone, Serialize, Deserialize)]
743#[serde(rename_all = "camelCase", deny_unknown_fields)]
744pub struct RenewEvaluationCaseRequest {
745    pub evaluate_task_id: String,
746    pub case_run_id: String,
747    pub fencing_token: i64,
748    #[serde(default = "default_driver_lease_ms")]
749    pub lease_ms: i64,
750}
751
752#[derive(Debug, Clone, Serialize, Deserialize)]
753#[serde(rename_all = "camelCase", deny_unknown_fields)]
754pub struct RenewEvaluationCaseResponse {
755    pub renewed: bool,
756    pub lease_until_ms: i64,
757}
758
759#[derive(Debug, Clone, Serialize, Deserialize)]
760#[serde(rename_all = "camelCase", deny_unknown_fields)]
761pub struct SubmitEvaluationEvidenceRequest {
762    pub evaluate_task_id: String,
763    pub case_run_id: String,
764    pub fencing_token: i64,
765    pub evidence: SandboxExecutionEvidence,
766}
767
768#[derive(Debug, Clone, Serialize, Deserialize)]
769#[serde(rename_all = "camelCase", deny_unknown_fields)]
770pub struct SubmitEvaluationEvidenceResponse {
771    pub accepted: bool,
772    pub operation: EvalOperation,
773}
774
775#[derive(Debug, Clone, Serialize, Deserialize)]
776#[serde(rename_all = "camelCase")]
777pub struct ListCasesRequest {
778    #[serde(default, skip_serializing_if = "Option::is_none")]
779    pub category: Option<String>,
780}
781
782#[derive(Debug, Clone, Serialize, Deserialize)]
783#[serde(rename_all = "camelCase")]
784pub struct ListCasesResponse {
785    pub cases: Vec<EvalCaseSummary>,
786}
787
788#[derive(Debug, Clone, Serialize, Deserialize)]
789#[serde(rename_all = "camelCase")]
790pub struct EvalCaseSummary {
791    pub id: String,
792    pub category: String,
793    pub notes: String,
794    pub turns: usize,
795    #[serde(default)]
796    pub has_oracle: bool,
797    #[serde(default)]
798    pub has_verify_commands: bool,
799    #[serde(default)]
800    pub expects_artifact: bool,
801}
802
803#[derive(Debug, Clone, Serialize, Deserialize)]
804#[serde(rename_all = "camelCase")]
805pub struct GetCaseRequest {
806    pub case_id: String,
807}
808
809#[derive(Debug, Clone, Serialize, Deserialize)]
810#[serde(rename_all = "camelCase")]
811pub struct LoadCasesRequest {
812    pub file_path: String,
813}
814
815#[derive(Debug, Clone, Serialize, Deserialize)]
816#[serde(rename_all = "camelCase")]
817pub struct GetReportRequest {
818    pub report_id: String,
819}
820
821#[derive(Debug, Clone, Serialize, Deserialize)]
822#[serde(rename_all = "camelCase", deny_unknown_fields)]
823pub struct CompareReportsRequest {
824    pub report_id: String,
825    pub baseline_report_id: String,
826    #[serde(default)]
827    pub minimum_pass_rate: f64,
828    #[serde(default = "default_max_infra_rate")]
829    pub maximum_infra_failure_rate: f64,
830    #[serde(default = "default_significance")]
831    pub significance_level: f64,
832    #[serde(default)]
833    pub minimum_effect: f64,
834}
835
836fn default_max_infra_rate() -> f64 {
837    1.0
838}
839
840fn default_significance() -> f64 {
841    0.05
842}
843
844#[derive(Debug, Clone, Serialize, Deserialize)]
845#[serde(rename_all = "camelCase")]
846pub struct CaseOutcomeDiff {
847    pub case_id: String,
848    #[serde(default, skip_serializing_if = "Option::is_none")]
849    pub variant_id: Option<String>,
850    pub baseline_passed: bool,
851    pub current_passed: bool,
852    pub flaky_signal: bool,
853    #[serde(default)]
854    pub historical_flake_rate: f64,
855    #[serde(default)]
856    pub historical_samples: u64,
857}
858
859#[derive(Debug, Clone, Serialize, Deserialize)]
860#[serde(rename_all = "camelCase")]
861pub struct ReportComparison {
862    pub report_id: String,
863    pub baseline_report_id: String,
864    pub pass_rate_delta: f64,
865    pub infra_failure_rate_delta: f64,
866    pub pass_rate_confidence_low: f64,
867    pub pass_rate_confidence_high: f64,
868    pub effect_confidence_low: f64,
869    pub effect_confidence_high: f64,
870    pub p_value: f64,
871    pub statistically_significant: bool,
872    pub changed_cases: Vec<CaseOutcomeDiff>,
873    pub gate_passed: bool,
874}
875
876#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
877#[serde(rename_all = "snake_case")]
878pub enum ReviewState {
879    Queued,
880    Claimed,
881    Approved,
882    Rejected,
883}
884
885#[derive(Debug, Clone, Serialize, Deserialize)]
886#[serde(rename_all = "camelCase")]
887pub struct HumanReview {
888    pub id: String,
889    pub run_id: String,
890    pub case_run_id: String,
891    pub state: ReviewState,
892    pub reason: String,
893    #[serde(default, skip_serializing_if = "Option::is_none")]
894    pub reviewer: Option<String>,
895    #[serde(default, skip_serializing_if = "Option::is_none")]
896    pub decision_note: Option<String>,
897    pub fencing_token: i64,
898    pub created_at_ms: i64,
899    pub updated_at_ms: i64,
900}
901
902#[derive(Debug, Clone, Serialize, Deserialize)]
903#[serde(rename_all = "camelCase", deny_unknown_fields)]
904pub struct EnqueueReviewRequest {
905    pub run_id: String,
906    pub case_run_id: String,
907    pub reason: String,
908}
909
910#[derive(Debug, Clone, Serialize, Deserialize)]
911#[serde(rename_all = "camelCase", deny_unknown_fields)]
912pub struct ClaimReviewRequest {
913    #[serde(default = "default_review_lease")]
914    pub lease_ms: i64,
915}
916
917fn default_review_lease() -> i64 {
918    300_000
919}
920
921#[derive(Debug, Clone, Serialize, Deserialize)]
922#[serde(rename_all = "camelCase", deny_unknown_fields)]
923pub struct CompleteReviewRequest {
924    pub review_id: String,
925    pub fencing_token: i64,
926    pub approved: bool,
927    #[serde(default)]
928    pub note: String,
929}
930
931/// Error body shared by every non-2xx response.
932#[derive(Debug, Clone, Serialize, Deserialize)]
933pub struct ApiError {
934    pub status: u16,
935    pub message: String,
936}
937
938#[cfg(test)]
939mod tests {
940    use super::TypedAssertion;
941    use serde_json::json;
942
943    #[test]
944    fn typed_assertion_uses_snake_case_tags_and_camel_case_fields() {
945        let assertion = TypedAssertion::TraceEvent {
946            event_type: "task.completed".into(),
947            minimum_count: 1,
948        };
949        let value = serde_json::to_value(&assertion).unwrap();
950        assert_eq!(
951            value,
952            json!({
953                "kind": "trace_event",
954                "eventType": "task.completed",
955                "minimumCount": 1
956            })
957        );
958        assert!(serde_json::from_value::<TypedAssertion>(value).is_ok());
959    }
960}
agent_evaluate_contract/lib.rs

agent_evaluate_contract/
lib.rs
