agent_evaluate_contract/
lib.rs

1//! HTTP API contract for the Evaluate service.
2//!
3//! Evaluate is a control plane over four immutable catalog resources —
4//! Case, Environment Profile, Evaluator and Suite — plus the durable Task
5//! that a Runtime pulls from. Case is the node that owns the dependency
6//! edges, so it carries its own id, version and digest like every other
7//! catalog resource; a Suite is only an ordered selection of Case versions.
8//!
9//! # Layering
10//!
11//! ```text
12//! agent-evaluate-contract   paths + DTOs        (this crate)
13//! agent-evaluate-runtime    control plane + verdicts
14//! agent-evaluate-server     router + auth + bin
15//! ```
16
17pub mod gate;
18pub mod metric;
19pub mod review;
20
21pub use gate::*;
22pub use metric::*;
23pub use review::*;
24
25use serde::{Deserialize, Serialize};
26use serde_json::Value;
27
28pub const SERVICE_NAME: &str = "agent-evaluate";
29pub const DEFAULT_PORT: u16 = 5150;
30pub const AUDIENCE: &str = "agent-evaluate";
31pub const MAX_RESOURCE_ID_CHARS: usize = 191;
32
33pub const HEALTH_PATH: &str = "/health";
34pub const CASE_VERSIONS_PATH: &str = "/internal/v1/evaluate/case-versions";
35pub const SUITES_PATH: &str = "/internal/v1/evaluate/suites";
36pub const RUNS_PATH: &str = "/internal/v1/evaluate/runs";
37pub const OPERATIONS_PATH: &str = "/internal/v1/evaluate/operations";
38pub const RUN_CASES_PATH: &str = "/internal/v1/evaluate/runs/cases";
39pub const CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/runs/cancel";
40pub const DURABLE_REPORT_PATH: &str = "/internal/v1/evaluate/reports/get";
41pub const COMPARE_REPORTS_PATH: &str = "/internal/v1/evaluate/reports/compare";
42pub const REVIEW_QUEUE_PATH: &str = "/internal/v1/evaluate/reviews";
43pub const CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/claim";
44pub const COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/complete";
45pub const SAMPLE_REVIEWS_PATH: &str = "/internal/v1/evaluate/reviews/sample";
46pub const METRIC_HEALTH_PATH: &str = "/internal/v1/evaluate/metrics/health";
47pub const METRIC_CANDIDATES_PATH: &str = "/internal/v1/evaluate/metrics/candidates";
48pub const ENVIRONMENT_PROFILES_PATH: &str = "/internal/v1/evaluate/environment-profiles";
49pub const EVALUATOR_VERSIONS_PATH: &str = "/internal/v1/evaluate/evaluator-versions";
50pub const DRIVER_CLAIM_PATH: &str = "/internal/v1/evaluate/driver/cases/claim";
51pub const DRIVER_RENEW_PATH: &str = "/internal/v1/evaluate/driver/cases/renew";
52pub const DRIVER_SUBMIT_PATH: &str = "/internal/v1/evaluate/driver/cases/submit";
53pub const ADMIN_OVERVIEW_PATH: &str = "/internal/v1/evaluate/admin/overview";
54pub const ADMIN_CASES_PATH: &str = "/internal/v1/evaluate/admin/cases";
55pub const ADMIN_SUITES_PATH: &str = "/internal/v1/evaluate/admin/suites";
56pub const ADMIN_RUNS_PATH: &str = "/internal/v1/evaluate/admin/runs";
57pub const ADMIN_RUN_CASES_PATH: &str = "/internal/v1/evaluate/admin/runs/cases";
58pub const ADMIN_CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/admin/runs/cancel";
59pub const ADMIN_REPORT_PATH: &str = "/internal/v1/evaluate/admin/reports/get";
60pub const ADMIN_REVIEWS_PATH: &str = "/internal/v1/evaluate/admin/reviews";
61pub const ADMIN_CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/claim";
62pub const ADMIN_COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/complete";
63pub const ADMIN_METRIC_HEALTH_PATH: &str = "/internal/v1/evaluate/admin/metrics/health";
64
65// ---------------------------------------------------------------------------
66// Eval case types
67// ---------------------------------------------------------------------------
68
69#[derive(Debug, Clone, Serialize, Deserialize)]
70#[serde(rename_all = "camelCase", deny_unknown_fields)]
71pub struct EvalTurn {
72    pub user: String,
73}
74
75#[derive(Debug, Clone, Serialize, Deserialize)]
76#[serde(rename_all = "camelCase", deny_unknown_fields)]
77pub struct VerifyCommand {
78    pub name: String,
79    pub command: String,
80    #[serde(default)]
81    pub expected_exit_code: i32,
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub output_contains: Option<Vec<String>>,
84}
85
86// ---------------------------------------------------------------------------
87// API request/response types
88// ---------------------------------------------------------------------------
89
90#[derive(Debug, Clone, Serialize, Deserialize)]
91#[serde(rename_all = "camelCase")]
92pub struct EvalReport {
93    pub id: String,
94    pub total_cases: usize,
95    pub passed_cases: usize,
96    pub failed_cases: usize,
97    pub infrastructure_failure_cases: usize,
98    /// Cases whose immutable schema/fixture/evaluator configuration could not
99    /// produce a meaningful quality verdict. Kept separate from provider or
100    /// runtime infrastructure failures.
101    #[serde(default)]
102    pub invalid_case_count: usize,
103    pub non_infrastructure_pass_rate: f64,
104    /// Per-evaluator slices. The aggregate pass rate above cannot say which
105    /// dimension moved; these can.
106    #[serde(default)]
107    pub metrics: Vec<MetricSummary>,
108    pub case_results: Vec<EvalCaseResult>,
109    pub created_at: String,
110    pub duration_ms: u64,
111    #[serde(default)]
112    pub total_cost_micros: u64,
113    /// Judge cost, reported apart from the subject's execution cost.
114    #[serde(default)]
115    pub judge_cost_micros: u64,
116    #[serde(default)]
117    pub p95_case_duration_ms: u64,
118}
119
120#[derive(Debug, Clone, Serialize, Deserialize)]
121#[serde(rename_all = "camelCase")]
122pub struct EvalCaseResult {
123    pub case_id: String,
124    /// Immutable Case version this result was produced from. Trend, flake and
125    /// regression statistics key on the pair, never on the bare id.
126    pub case_version: String,
127    pub category: String,
128    pub passed: bool,
129    pub infrastructure_failure: bool,
130    /// Structured terminal classification for non-quality failures. The
131    /// legacy `infrastructure_failure` boolean remains for wire compatibility.
132    #[serde(default, skip_serializing_if = "Option::is_none")]
133    pub failure_kind: Option<CaseFailureKind>,
134    pub turns_completed: usize,
135    pub total_turns: usize,
136    /// Verdicts of the Case's own inline assertions.
137    #[serde(default)]
138    pub assertion_results: Vec<AssertionVerdict>,
139    /// Versioned evaluator outputs.
140    #[serde(default)]
141    pub evaluator_verdicts: Vec<EvaluatorVerdict>,
142    #[serde(default, skip_serializing_if = "Option::is_none")]
143    pub error: Option<String>,
144    pub duration_ms: u64,
145    #[serde(default)]
146    pub cost_micros: u64,
147    /// Durable experiment identity, retained so repeated samples from
148    /// different variants are never conflated as flakiness.
149    #[serde(default)]
150    pub sample_index: u32,
151    #[serde(default, skip_serializing_if = "Option::is_none")]
152    pub variant_id: Option<String>,
153}
154
155#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
156#[serde(rename_all = "snake_case")]
157pub enum CaseFailureKind {
158    InfraFailed,
159    InvalidCase,
160}
161
162#[derive(Debug, Clone, Serialize, Deserialize)]
163#[serde(rename_all = "camelCase")]
164pub struct AssertionVerdict {
165    pub rule: String,
166    pub passed: bool,
167    #[serde(default, skip_serializing_if = "Option::is_none")]
168    pub detail: Option<String>,
169}
170
171/// Versioned typed assertion schema used by immutable Case versions.
172#[derive(Debug, Clone, Serialize, Deserialize)]
173#[serde(
174    tag = "kind",
175    rename_all = "snake_case",
176    rename_all_fields = "camelCase",
177    deny_unknown_fields
178)]
179pub enum TypedAssertion {
180    OutputContains {
181        turn: usize,
182        values: Vec<String>,
183        #[serde(default)]
184        any: bool,
185    },
186    WorkspaceFile {
187        path: String,
188        #[serde(default)]
189        contains: Vec<String>,
190        #[serde(default, skip_serializing_if = "Option::is_none")]
191        max_bytes: Option<usize>,
192    },
193    CommandExit {
194        name: String,
195        command_ref: String,
196        expected_exit_code: i32,
197    },
198    TraceEvent {
199        event_type: String,
200        #[serde(default)]
201        minimum_count: usize,
202    },
203    ArtifactExists {
204        artifact_kind: String,
205    },
206    /// A custom evaluator that is executed by the isolated executor. The
207    /// control plane accepts only an attested result for the exact immutable
208    /// plugin reference; it never loads plugin code into the service process.
209    Plugin {
210        plugin_ref: String,
211        #[serde(default)]
212        config: Value,
213    },
214}
215
216/// Immutable reference to a catalog resource. Evaluate snapshots the resolved
217/// version into a CaseAttempt; mutable aliases are deliberately unsupported.
218#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
219#[serde(rename_all = "camelCase", deny_unknown_fields)]
220pub struct VersionedResourceRef {
221    pub id: String,
222    pub version: String,
223}
224
225/// Agent is an optimistic precondition; Runtime ID remains the execution
226/// selector and Runtime is the authoritative owner of their binding.
227#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
228#[serde(rename_all = "camelCase", deny_unknown_fields)]
229pub struct EvaluationSubject {
230    pub agent_id: String,
231    pub runtime_id: String,
232}
233
234#[derive(Debug, Clone, Serialize, Deserialize)]
235#[serde(rename_all = "camelCase", deny_unknown_fields)]
236pub struct RegisterEnvironmentProfileRequest {
237    pub profile_id: String,
238    pub version: String,
239    /// Existing Workspace Environment Space selected for the Task. Provisioned
240    /// environments can be added later behind the same immutable profile.
241    #[serde(default, skip_serializing_if = "Option::is_none")]
242    pub space_id: Option<String>,
243    #[serde(default)]
244    pub metadata: std::collections::BTreeMap<String, String>,
245}
246
247#[derive(Debug, Clone, Serialize, Deserialize)]
248#[serde(rename_all = "camelCase", deny_unknown_fields)]
249pub struct EnvironmentProfileVersion {
250    pub tenant_id: String,
251    pub project_id: String,
252    pub profile_id: String,
253    pub version: String,
254    pub digest: String,
255    #[serde(default, skip_serializing_if = "Option::is_none")]
256    pub space_id: Option<String>,
257    #[serde(default)]
258    pub metadata: std::collections::BTreeMap<String, String>,
259    pub created_at_ms: i64,
260}
261
262#[derive(Debug, Clone, Serialize, Deserialize)]
263#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
264pub enum EvaluatorImplementation {
265    /// Pure, deterministic assertions evaluated from a bounded Evidence view.
266    BuiltinAssertions { assertions: Vec<TypedAssertion> },
267    /// Model-graded evaluator executed by the isolated executor. The control
268    /// plane only maps its attested evidence onto a verdict.
269    LlmJudge {
270        /// Pinned judge model. Changing it requires a new evaluator version so
271        /// judge drift can never be mistaken for subject regression.
272        model_ref: String,
273        /// Digest of the rubric prompt handed to the judge.
274        prompt_digest: String,
275        #[serde(default)]
276        max_cost_micros: u64,
277    },
278}
279
280impl EvaluatorImplementation {
281    pub const fn is_model_graded(&self) -> bool {
282        matches!(self, Self::LlmJudge { .. })
283    }
284}
285
286/// Human-label agreement accumulated for one evaluator id.
287#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
288#[serde(rename_all = "camelCase")]
289pub struct JudgeCalibration {
290    pub labeled_samples: u64,
291    pub agreed_samples: u64,
292}
293
294impl JudgeCalibration {
295    pub fn agreement(&self) -> f64 {
296        if self.labeled_samples == 0 {
297            0.0
298        } else {
299            self.agreed_samples as f64 / self.labeled_samples as f64
300        }
301    }
302}
303
304/// Minimum human-label evidence before a model-graded evaluator may gate.
305pub const JUDGE_PROMOTION_MIN_LABELS: u64 = 50;
306pub const JUDGE_PROMOTION_MIN_AGREEMENT: f64 = 0.9;
307
308#[derive(Debug, Clone, Serialize, Deserialize)]
309#[serde(rename_all = "camelCase", deny_unknown_fields)]
310pub struct RegisterEvaluatorVersionRequest {
311    pub evaluator_id: String,
312    pub version: String,
313    pub implementation: EvaluatorImplementation,
314    #[serde(default)]
315    pub metric: MetricSpec,
316}
317
318#[derive(Debug, Clone, Serialize, Deserialize)]
319#[serde(rename_all = "camelCase", deny_unknown_fields)]
320pub struct EvaluatorVersion {
321    pub tenant_id: String,
322    pub project_id: String,
323    pub evaluator_id: String,
324    pub version: String,
325    pub digest: String,
326    pub implementation: EvaluatorImplementation,
327    #[serde(default)]
328    pub metric: MetricSpec,
329    pub created_at_ms: i64,
330}
331
332#[derive(Debug, Clone, Serialize, Deserialize)]
333#[serde(rename_all = "camelCase")]
334pub struct EvaluatorVerdict {
335    pub evaluator_id: String,
336    pub evaluator_version: String,
337    pub passed: bool,
338    #[serde(default)]
339    pub class: MetricClass,
340    #[serde(default)]
341    pub decision: MetricDecision,
342    /// Judge-side cost, kept apart from the subject's execution cost.
343    #[serde(default)]
344    pub cost_micros: u64,
345    #[serde(default)]
346    pub assertions: Vec<AssertionVerdict>,
347}
348
349/// Immutable Case content. A Case is the node that owns the evaluation
350/// dependency edges, so it is registered and versioned in its own right and a
351/// Suite only references it.
352#[derive(Debug, Clone, Serialize, Deserialize)]
353#[serde(rename_all = "camelCase", deny_unknown_fields)]
354pub struct RegisterEvalCaseRequest {
355    pub case_id: String,
356    pub version: String,
357    pub category: String,
358    #[serde(default)]
359    pub turns: Vec<EvalTurn>,
360    /// Small, immutable text fixtures materialised into the selected Space
361    /// before Runtime executes the case. Large fixtures remain external and
362    /// are referenced through `fixtureRef`.
363    #[serde(default)]
364    pub workspace_files: std::collections::BTreeMap<String, String>,
365    #[serde(default)]
366    pub assertions: Vec<TypedAssertion>,
367    /// Versioned environment selection.
368    #[serde(default, skip_serializing_if = "Option::is_none")]
369    pub environment_profile: Option<VersionedResourceRef>,
370    /// Domain-owned evaluator policies executed after evidence collection.
371    #[serde(default)]
372    pub evaluator_versions: Vec<VersionedResourceRef>,
373    #[serde(default)]
374    pub verify_commands: Vec<VerifyCommand>,
375    #[serde(default, skip_serializing_if = "Option::is_none")]
376    pub fixture_ref: Option<String>,
377    #[serde(default)]
378    pub budget: EvalBudget,
379}
380
381#[derive(Debug, Clone, Serialize, Deserialize)]
382#[serde(rename_all = "camelCase", deny_unknown_fields)]
383pub struct EvalCaseVersion {
384    pub tenant_id: String,
385    pub project_id: String,
386    pub case_id: String,
387    pub version: String,
388    pub digest: String,
389    pub category: String,
390    #[serde(default)]
391    pub turns: Vec<EvalTurn>,
392    #[serde(default)]
393    pub workspace_files: std::collections::BTreeMap<String, String>,
394    #[serde(default)]
395    pub assertions: Vec<TypedAssertion>,
396    #[serde(default, skip_serializing_if = "Option::is_none")]
397    pub environment_profile: Option<VersionedResourceRef>,
398    #[serde(default)]
399    pub evaluator_versions: Vec<VersionedResourceRef>,
400    #[serde(default)]
401    pub verify_commands: Vec<VerifyCommand>,
402    #[serde(default, skip_serializing_if = "Option::is_none")]
403    pub fixture_ref: Option<String>,
404    #[serde(default)]
405    pub budget: EvalBudget,
406    pub created_at_ms: i64,
407}
408
409impl EvalCaseVersion {
410    pub fn reference(&self) -> VersionedResourceRef {
411        VersionedResourceRef {
412            id: self.case_id.clone(),
413            version: self.version.clone(),
414        }
415    }
416}
417
418#[derive(Debug, Clone, Serialize, Deserialize)]
419#[serde(rename_all = "camelCase", deny_unknown_fields)]
420pub struct EvalBudget {
421    pub timeout_ms: u64,
422    pub max_output_bytes: usize,
423    pub max_cost_micros: u64,
424}
425
426impl Default for EvalBudget {
427    fn default() -> Self {
428        Self {
429            timeout_ms: 300_000,
430            max_output_bytes: 8 * 1024 * 1024,
431            max_cost_micros: 0,
432        }
433    }
434}
435
436/// Ordered selection of immutable Case versions. A Suite adds no content of
437/// its own, so changing one Case never rewrites the others.
438#[derive(Debug, Clone, Serialize, Deserialize)]
439#[serde(rename_all = "camelCase", deny_unknown_fields)]
440pub struct EvalSuiteVersion {
441    pub tenant_id: String,
442    pub project_id: String,
443    pub suite_id: String,
444    pub version: String,
445    pub manifest_digest: String,
446    pub schema_version: u32,
447    pub cases: Vec<VersionedResourceRef>,
448    pub created_at_ms: i64,
449}
450
451#[derive(Debug, Clone, Serialize, Deserialize)]
452#[serde(rename_all = "camelCase", deny_unknown_fields)]
453pub struct RegisterSuiteRequest {
454    pub suite_id: String,
455    pub version: String,
456    pub manifest_digest: String,
457    pub schema_version: u32,
458    pub cases: Vec<VersionedResourceRef>,
459}
460
461#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
462#[serde(rename_all = "snake_case")]
463pub enum OperationState {
464    Queued,
465    Running,
466    Succeeded,
467    Failed,
468    Canceling,
469    Canceled,
470}
471
472#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
473#[serde(rename_all = "snake_case")]
474pub enum CaseRunState {
475    Queued,
476    Running,
477    Passed,
478    QualityFailed,
479    InfraFailed,
480    InvalidCase,
481    Canceled,
482}
483
484#[derive(Debug, Clone, Serialize, Deserialize)]
485#[serde(rename_all = "camelCase", deny_unknown_fields)]
486pub struct CreateEvalRunRequest {
487    /// Suite version to expand. Mutually exclusive with `cases`.
488    #[serde(default, skip_serializing_if = "Option::is_none")]
489    pub suite: Option<VersionedResourceRef>,
490    /// Ad-hoc Case selection. Because Case is a catalog resource, a run can be
491    /// composed without first inventing a Suite for it.
492    #[serde(default)]
493    pub cases: Vec<VersionedResourceRef>,
494    pub subject_version: String,
495    pub executor_profile: String,
496    /// Concrete execution target. New Runtime Task based runs require it;
497    /// subject_version remains as a compatibility/report label.
498    #[serde(default, skip_serializing_if = "Option::is_none")]
499    pub subject: Option<EvaluationSubject>,
500    #[serde(default)]
501    pub seed: u64,
502    #[serde(default = "default_concurrency")]
503    pub concurrency: usize,
504    #[serde(default)]
505    pub idempotency_key: String,
506    /// Independent deterministic samples per `(case, variant)`.
507    #[serde(default = "default_samples")]
508    pub samples: u32,
509    /// Empty preserves the single-subject run. A non-empty matrix expands
510    /// the run without requiring consumers to coordinate many operations.
511    #[serde(default)]
512    pub experiment_matrix: Vec<ExperimentVariant>,
513}
514
515fn default_concurrency() -> usize {
516    8
517}
518
519fn default_samples() -> u32 {
520    1
521}
522
523#[derive(Debug, Clone, Serialize, Deserialize)]
524#[serde(rename_all = "camelCase", deny_unknown_fields)]
525pub struct ExperimentVariant {
526    pub id: String,
527    pub subject_version: String,
528    pub executor_profile: String,
529    #[serde(default)]
530    pub parameters: Value,
531}
532
533#[derive(Debug, Clone, Serialize, Deserialize)]
534#[serde(rename_all = "camelCase")]
535pub struct EvalOperation {
536    pub id: String,
537    pub run_id: String,
538    pub state: OperationState,
539    pub phase: String,
540    pub attempt: u32,
541    pub completed_cases: usize,
542    pub total_cases: usize,
543    #[serde(default, skip_serializing_if = "Option::is_none")]
544    pub report_id: Option<String>,
545    pub created_at_ms: i64,
546    pub updated_at_ms: i64,
547    pub version: i64,
548    #[serde(default, skip_serializing_if = "Option::is_none")]
549    pub next_poll_after_ms: Option<u64>,
550    #[serde(default, skip_serializing_if = "Option::is_none")]
551    pub error_code: Option<String>,
552}
553
554#[derive(Debug, Clone, Serialize, Deserialize)]
555#[serde(rename_all = "camelCase", deny_unknown_fields)]
556pub struct GetOperationRequest {
557    pub operation_id: String,
558}
559
560#[derive(Debug, Clone, Serialize, Deserialize)]
561#[serde(rename_all = "camelCase", deny_unknown_fields)]
562pub struct CancelRunRequest {
563    pub run_id: String,
564}
565
566#[derive(Debug, Clone, Serialize, Deserialize)]
567#[serde(rename_all = "camelCase", deny_unknown_fields)]
568pub struct ListCaseRunsRequest {
569    pub run_id: String,
570    #[serde(default, skip_serializing_if = "Option::is_none")]
571    pub cursor: Option<String>,
572    #[serde(default, skip_serializing_if = "Option::is_none")]
573    pub state: Option<CaseRunState>,
574    #[serde(default = "default_case_page")]
575    pub limit: usize,
576}
577
578fn default_case_page() -> usize {
579    50
580}
581
582#[derive(Debug, Clone, Serialize, Deserialize)]
583#[serde(rename_all = "camelCase")]
584pub struct CaseRunSummary {
585    pub id: String,
586    pub run_id: String,
587    pub case_id: String,
588    pub case_version: String,
589    pub position: usize,
590    pub state: CaseRunState,
591    pub attempt: u32,
592    pub duration_ms: u64,
593    #[serde(default, skip_serializing_if = "Option::is_none")]
594    pub error: Option<String>,
595    #[serde(default)]
596    pub evidence_refs: Vec<EvidenceRef>,
597    #[serde(default)]
598    pub sample_index: u32,
599    #[serde(default, skip_serializing_if = "Option::is_none")]
600    pub variant_id: Option<String>,
601}
602
603#[derive(Debug, Clone, Serialize, Deserialize)]
604#[serde(rename_all = "camelCase")]
605pub struct CaseRunPage {
606    pub items: Vec<CaseRunSummary>,
607    #[serde(default, skip_serializing_if = "Option::is_none")]
608    pub next_cursor: Option<String>,
609}
610
611#[derive(Debug, Clone, Serialize, Deserialize)]
612#[serde(rename_all = "camelCase", deny_unknown_fields)]
613pub struct EvalAccessScope {
614    pub tenant_id: String,
615    pub project_id: String,
616    pub subject: String,
617}
618
619#[derive(Debug, Clone, Serialize, Deserialize)]
620#[serde(rename_all = "camelCase", deny_unknown_fields)]
621pub struct SandboxExecutionRequest {
622    /// Tenant/project ownership is passed explicitly so the isolated executor
623    /// can only finalize evidence in the namespace later validated by the
624    /// control plane.
625    pub scope: EvalAccessScope,
626    pub operation_id: String,
627    pub run_id: String,
628    pub case_run_id: String,
629    pub subject_version: String,
630    pub executor_profile: String,
631    #[serde(default, skip_serializing_if = "Option::is_none")]
632    pub subject: Option<EvaluationSubject>,
633    #[serde(default, skip_serializing_if = "Option::is_none")]
634    pub environment_profile: Option<EnvironmentProfileVersion>,
635    #[serde(default)]
636    pub evaluator_versions: Vec<EvaluatorVersion>,
637    pub seed: u64,
638    pub case: EvalCaseVersion,
639    pub fencing_token: i64,
640    #[serde(default)]
641    pub sample_index: u32,
642    #[serde(default, skip_serializing_if = "Option::is_none")]
643    pub variant: Option<ExperimentVariant>,
644}
645
646#[derive(Debug, Clone, Serialize, Deserialize)]
647#[serde(rename_all = "camelCase", deny_unknown_fields)]
648pub struct PluginEvaluationEvidence {
649    pub plugin_ref: String,
650    pub passed: bool,
651    #[serde(default)]
652    pub detail: String,
653    /// Attestation produced by the configured sandbox boundary. Production
654    /// evaluation fails closed when it is absent.
655    pub attestation: String,
656}
657
658/// Attested outcome of a model-graded evaluator. Produced by the isolated
659/// executor; the control plane never calls a judge model itself.
660#[derive(Debug, Clone, Serialize, Deserialize)]
661#[serde(rename_all = "camelCase", deny_unknown_fields)]
662pub struct JudgeEvaluationEvidence {
663    pub evaluator_id: String,
664    pub evaluator_version: String,
665    pub passed: bool,
666    /// Judge model actually used, checked against the pinned reference.
667    pub model_ref: String,
668    #[serde(default)]
669    pub detail: String,
670    #[serde(default)]
671    pub cost_micros: u64,
672    pub attestation: String,
673}
674
675/// Immutable evidence object owned by the scoped Evaluate namespace.
676#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
677#[serde(rename_all = "camelCase", deny_unknown_fields)]
678pub struct EvidenceRef {
679    pub reference: String,
680    pub tenant_id: String,
681    pub project_id: String,
682    pub run_id: String,
683    pub case_run_id: String,
684    /// Lower-case `sha256:<64 hex>` digest of the finalized bytes.
685    pub digest: String,
686    pub size_bytes: u64,
687    /// Attestation bound to the isolated executor response boundary.
688    pub attestation: String,
689}
690
691/// Bounded evidence returned by the isolated executor. Verdict calculation
692/// remains owned by Evaluate and is deterministic from this typed structure.
693#[derive(Debug, Clone, Default, Serialize, Deserialize)]
694#[serde(rename_all = "camelCase", deny_unknown_fields)]
695pub struct SandboxExecutionEvidence {
696    #[serde(default)]
697    pub responses: Vec<String>,
698    #[serde(default)]
699    pub workspace_files: std::collections::BTreeMap<String, String>,
700    #[serde(default)]
701    pub command_exit_codes: std::collections::BTreeMap<String, i32>,
702    #[serde(default)]
703    pub trace_event_counts: std::collections::BTreeMap<String, usize>,
704    #[serde(default)]
705    pub artifact_kinds: Vec<String>,
706    /// Immutable blob/object references owned by Evaluate's evidence namespace.
707    #[serde(default)]
708    pub evidence_refs: Vec<EvidenceRef>,
709    #[serde(default)]
710    pub plugin_evaluations: Vec<PluginEvaluationEvidence>,
711    #[serde(default)]
712    pub judge_evaluations: Vec<JudgeEvaluationEvidence>,
713    #[serde(default, skip_serializing_if = "Option::is_none")]
714    pub infrastructure_error: Option<String>,
715    /// A sandbox may explicitly reject an immutable case as unsupported or
716    /// invalid. This is deliberately typed so provider text is never parsed as
717    /// a terminal-state code.
718    #[serde(default, skip_serializing_if = "Option::is_none")]
719    pub invalid_case_error: Option<String>,
720    #[serde(default)]
721    pub duration_ms: u64,
722    /// Executor-measured cost in millionths of the configured billing unit.
723    #[serde(default)]
724    pub cost_micros: u64,
725}
726
727/// Runtime-facing pull protocol. `evaluate_task_id` is the durable
728/// [`EvalOperation::id`] returned when the SDK/CLI configures a run. Infra
729/// never needs a callback URL for the Runtime.
730#[derive(Debug, Clone, Serialize, Deserialize)]
731#[serde(rename_all = "camelCase", deny_unknown_fields)]
732pub struct ClaimEvaluationCaseRequest {
733    pub evaluate_task_id: String,
734    #[serde(default = "default_driver_lease_ms")]
735    pub lease_ms: i64,
736}
737
738fn default_driver_lease_ms() -> i64 {
739    60_000
740}
741
742#[derive(Debug, Clone, Serialize, Deserialize)]
743#[serde(rename_all = "camelCase", deny_unknown_fields)]
744pub struct EvaluationCaseLease {
745    pub execution: SandboxExecutionRequest,
746    pub lease_until_ms: i64,
747}
748
749#[derive(Debug, Clone, Serialize, Deserialize)]
750#[serde(rename_all = "camelCase", deny_unknown_fields)]
751pub struct ClaimEvaluationCaseResponse {
752    pub operation: EvalOperation,
753    #[serde(default, skip_serializing_if = "Option::is_none")]
754    pub case: Option<EvaluationCaseLease>,
755}
756
757#[derive(Debug, Clone, Serialize, Deserialize)]
758#[serde(rename_all = "camelCase", deny_unknown_fields)]
759pub struct RenewEvaluationCaseRequest {
760    pub evaluate_task_id: String,
761    pub case_run_id: String,
762    pub fencing_token: i64,
763    #[serde(default = "default_driver_lease_ms")]
764    pub lease_ms: i64,
765}
766
767#[derive(Debug, Clone, Serialize, Deserialize)]
768#[serde(rename_all = "camelCase", deny_unknown_fields)]
769pub struct RenewEvaluationCaseResponse {
770    pub renewed: bool,
771    pub lease_until_ms: i64,
772}
773
774#[derive(Debug, Clone, Serialize, Deserialize)]
775#[serde(rename_all = "camelCase", deny_unknown_fields)]
776pub struct SubmitEvaluationEvidenceRequest {
777    pub evaluate_task_id: String,
778    pub case_run_id: String,
779    pub fencing_token: i64,
780    pub evidence: SandboxExecutionEvidence,
781}
782
783#[derive(Debug, Clone, Serialize, Deserialize)]
784#[serde(rename_all = "camelCase", deny_unknown_fields)]
785pub struct SubmitEvaluationEvidenceResponse {
786    pub accepted: bool,
787    pub operation: EvalOperation,
788}
789
790#[derive(Debug, Clone, Serialize, Deserialize)]
791#[serde(rename_all = "camelCase", deny_unknown_fields)]
792pub struct GetReportRequest {
793    pub report_id: String,
794}
795
796// ---------------------------------------------------------------------------
797// Evaluate administration query model
798// ---------------------------------------------------------------------------
799
800pub const DEFAULT_ADMIN_PAGE_SIZE: usize = 50;
801pub const MAX_ADMIN_PAGE_SIZE: usize = 200;
802
803fn default_admin_page() -> usize {
804    DEFAULT_ADMIN_PAGE_SIZE
805}
806
807#[derive(Debug, Clone, Serialize, Deserialize)]
808#[serde(rename_all = "camelCase", deny_unknown_fields)]
809pub struct ListAdminCasesRequest {
810    #[serde(default, skip_serializing_if = "Option::is_none")]
811    pub cursor: Option<String>,
812    #[serde(default = "default_admin_page")]
813    pub limit: usize,
814}
815
816impl Default for ListAdminCasesRequest {
817    fn default() -> Self {
818        Self {
819            cursor: None,
820            limit: DEFAULT_ADMIN_PAGE_SIZE,
821        }
822    }
823}
824
825#[derive(Debug, Clone, Serialize, Deserialize)]
826#[serde(rename_all = "camelCase", deny_unknown_fields)]
827pub struct ListAdminSuitesRequest {
828    #[serde(default, skip_serializing_if = "Option::is_none")]
829    pub cursor: Option<String>,
830    #[serde(default = "default_admin_page")]
831    pub limit: usize,
832}
833
834impl Default for ListAdminSuitesRequest {
835    fn default() -> Self {
836        Self {
837            cursor: None,
838            limit: DEFAULT_ADMIN_PAGE_SIZE,
839        }
840    }
841}
842
843#[derive(Debug, Clone, Serialize, Deserialize)]
844#[serde(rename_all = "camelCase", deny_unknown_fields)]
845pub struct ListAdminRunsRequest {
846    #[serde(default, skip_serializing_if = "Option::is_none")]
847    pub cursor: Option<String>,
848    #[serde(default, skip_serializing_if = "Option::is_none")]
849    pub state: Option<OperationState>,
850    #[serde(default = "default_admin_page")]
851    pub limit: usize,
852}
853
854impl Default for ListAdminRunsRequest {
855    fn default() -> Self {
856        Self {
857            cursor: None,
858            state: None,
859            limit: DEFAULT_ADMIN_PAGE_SIZE,
860        }
861    }
862}
863
864#[derive(Debug, Clone, Serialize, Deserialize)]
865#[serde(rename_all = "camelCase", deny_unknown_fields)]
866pub struct ListAdminReviewsRequest {
867    #[serde(default, skip_serializing_if = "Option::is_none")]
868    pub cursor: Option<String>,
869    #[serde(default, skip_serializing_if = "Option::is_none")]
870    pub state: Option<ReviewState>,
871    #[serde(default = "default_admin_page")]
872    pub limit: usize,
873}
874
875impl Default for ListAdminReviewsRequest {
876    fn default() -> Self {
877        Self {
878            cursor: None,
879            state: None,
880            limit: DEFAULT_ADMIN_PAGE_SIZE,
881        }
882    }
883}
884
885#[derive(Debug, Clone, Serialize, Deserialize)]
886#[serde(rename_all = "camelCase")]
887pub struct EvaluateAdminOverview {
888    pub case_versions: u64,
889    pub suite_versions: u64,
890    pub run_states: std::collections::BTreeMap<String, u64>,
891    pub case_states: std::collections::BTreeMap<String, u64>,
892    pub review_states: std::collections::BTreeMap<String, u64>,
893}
894
895#[derive(Debug, Clone, Serialize, Deserialize)]
896#[serde(rename_all = "camelCase")]
897pub struct AdminCaseSummary {
898    pub case_id: String,
899    pub version: String,
900    pub digest: String,
901    pub category: String,
902    pub turns: usize,
903    pub evaluator_versions: Vec<VersionedResourceRef>,
904    #[serde(default, skip_serializing_if = "Option::is_none")]
905    pub environment_profile: Option<VersionedResourceRef>,
906    pub created_at_ms: i64,
907}
908
909#[derive(Debug, Clone, Serialize, Deserialize)]
910#[serde(rename_all = "camelCase")]
911pub struct AdminCasePage {
912    pub items: Vec<AdminCaseSummary>,
913    #[serde(default, skip_serializing_if = "Option::is_none")]
914    pub next_cursor: Option<String>,
915}
916
917#[derive(Debug, Clone, Serialize, Deserialize)]
918#[serde(rename_all = "camelCase")]
919pub struct AdminSuiteSummary {
920    pub suite_id: String,
921    pub version: String,
922    pub manifest_digest: String,
923    pub schema_version: u32,
924    pub case_count: usize,
925    pub created_at_ms: i64,
926}
927
928#[derive(Debug, Clone, Serialize, Deserialize)]
929#[serde(rename_all = "camelCase")]
930pub struct AdminSuitePage {
931    pub items: Vec<AdminSuiteSummary>,
932    #[serde(default, skip_serializing_if = "Option::is_none")]
933    pub next_cursor: Option<String>,
934}
935
936#[derive(Debug, Clone, Serialize, Deserialize)]
937#[serde(rename_all = "camelCase")]
938pub struct AdminRunSummary {
939    pub run_id: String,
940    pub operation_id: String,
941    /// Present only when the run was expanded from a Suite version.
942    #[serde(default, skip_serializing_if = "Option::is_none")]
943    pub suite: Option<VersionedResourceRef>,
944    pub subject_version: String,
945    pub executor_profile: String,
946    #[serde(default, skip_serializing_if = "Option::is_none")]
947    pub subject: Option<EvaluationSubject>,
948    pub state: OperationState,
949    pub phase: String,
950    pub completed_cases: usize,
951    pub total_cases: usize,
952    #[serde(default, skip_serializing_if = "Option::is_none")]
953    pub report_id: Option<String>,
954    pub created_at_ms: i64,
955    pub updated_at_ms: i64,
956    pub version: i64,
957}
958
959#[derive(Debug, Clone, Serialize, Deserialize)]
960#[serde(rename_all = "camelCase")]
961pub struct AdminRunPage {
962    pub items: Vec<AdminRunSummary>,
963    #[serde(default, skip_serializing_if = "Option::is_none")]
964    pub next_cursor: Option<String>,
965}
966
967#[derive(Debug, Clone, Serialize, Deserialize)]
968#[serde(rename_all = "camelCase")]
969pub struct AdminReviewPage {
970    pub items: Vec<HumanReview>,
971    #[serde(default, skip_serializing_if = "Option::is_none")]
972    pub next_cursor: Option<String>,
973}
974
975/// Error body shared by every non-2xx response.
976#[derive(Debug, Clone, Serialize, Deserialize)]
977pub struct ApiError {
978    pub status: u16,
979    pub message: String,
980}
981
982#[cfg(test)]
983mod tests {
984    use super::{
985        DEFAULT_ADMIN_PAGE_SIZE, ListAdminCasesRequest, ListAdminReviewsRequest,
986        ListAdminRunsRequest, ListAdminSuitesRequest, TypedAssertion,
987    };
988    use serde_json::json;
989
990    #[test]
991    fn typed_assertion_uses_snake_case_tags_and_camel_case_fields() {
992        let assertion = TypedAssertion::TraceEvent {
993            event_type: "task.completed".into(),
994            minimum_count: 1,
995        };
996        let value = serde_json::to_value(&assertion).unwrap();
997        assert_eq!(
998            value,
999            json!({
1000                "kind": "trace_event",
1001                "eventType": "task.completed",
1002                "minimumCount": 1
1003            })
1004        );
1005        assert!(serde_json::from_value::<TypedAssertion>(value).is_ok());
1006    }
1007
1008    #[test]
1009    fn admin_queries_use_the_same_bounded_default_in_rust_and_serde() {
1010        assert_eq!(
1011            ListAdminCasesRequest::default().limit,
1012            DEFAULT_ADMIN_PAGE_SIZE
1013        );
1014        assert_eq!(
1015            ListAdminSuitesRequest::default().limit,
1016            DEFAULT_ADMIN_PAGE_SIZE
1017        );
1018        assert_eq!(
1019            ListAdminRunsRequest::default().limit,
1020            DEFAULT_ADMIN_PAGE_SIZE
1021        );
1022        assert_eq!(
1023            ListAdminReviewsRequest::default().limit,
1024            DEFAULT_ADMIN_PAGE_SIZE
1025        );
1026        let decoded: ListAdminRunsRequest = serde_json::from_value(json!({})).unwrap();
1027        assert_eq!(decoded.limit, DEFAULT_ADMIN_PAGE_SIZE);
1028    }
1029}
agent_evaluate_contract/lib.rs

agent_evaluate_contract/
lib.rs
