agent_evaluate_contract/
lib.rs

1//! HTTP API contract for the Evaluate service.
2//!
3//! Evaluate is a control plane over four immutable catalog resources —
4//! Case, Environment Profile, Evaluator and Suite — plus the durable Task
5//! that a Runtime pulls from. Case is the node that owns the dependency
6//! edges, so it carries its own id, version and digest like every other
7//! catalog resource; a Suite is only an ordered selection of Case versions.
8//!
9//! # Layering
10//!
11//! ```text
12//! agent-evaluate-contract   paths + DTOs        (this crate)
13//! agent-evaluate-runtime    control plane + verdicts
14//! agent-evaluate-server     router + auth + bin
15//! ```
16
17pub mod gate;
18pub mod metric;
19pub mod objective;
20pub mod review;
21
22pub use gate::*;
23pub use metric::*;
24pub use objective::*;
25pub use review::*;
26
27use serde::{Deserialize, Serialize};
28use serde_json::Value;
29
30pub const SERVICE_NAME: &str = "agent-evaluate";
31pub const DEFAULT_PORT: u16 = 5150;
32pub const AUDIENCE: &str = "agent-evaluate";
33pub const MAX_RESOURCE_ID_CHARS: usize = 191;
34
35pub const HEALTH_PATH: &str = "/health";
36pub const CASE_VERSIONS_PATH: &str = "/internal/v1/evaluate/case-versions";
37pub const SUITES_PATH: &str = "/internal/v1/evaluate/suites";
38pub const RUNS_PATH: &str = "/internal/v1/evaluate/runs";
39pub const OPERATIONS_PATH: &str = "/internal/v1/evaluate/operations";
40pub const RUN_CASES_PATH: &str = "/internal/v1/evaluate/runs/cases";
41pub const CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/runs/cancel";
42pub const DURABLE_REPORT_PATH: &str = "/internal/v1/evaluate/reports/get";
43pub const COMPARE_REPORTS_PATH: &str = "/internal/v1/evaluate/reports/compare";
44pub const REVIEW_QUEUE_PATH: &str = "/internal/v1/evaluate/reviews";
45pub const CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/claim";
46pub const COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/reviews/complete";
47pub const SAMPLE_REVIEWS_PATH: &str = "/internal/v1/evaluate/reviews/sample";
48pub const METRIC_HEALTH_PATH: &str = "/internal/v1/evaluate/metrics/health";
49pub const METRIC_CANDIDATES_PATH: &str = "/internal/v1/evaluate/metrics/candidates";
50pub const TRACE_OBJECTIVE_METRICS_PATH: &str = "/internal/v1/evaluate/metrics/objective-from-trace";
51pub const ENVIRONMENT_PROFILES_PATH: &str = "/internal/v1/evaluate/environment-profiles";
52pub const EVALUATOR_VERSIONS_PATH: &str = "/internal/v1/evaluate/evaluator-versions";
53pub const DRIVER_CLAIM_PATH: &str = "/internal/v1/evaluate/driver/cases/claim";
54pub const DRIVER_RENEW_PATH: &str = "/internal/v1/evaluate/driver/cases/renew";
55pub const DRIVER_SUBMIT_PATH: &str = "/internal/v1/evaluate/driver/cases/submit";
56pub const ADMIN_OVERVIEW_PATH: &str = "/internal/v1/evaluate/admin/overview";
57pub const ADMIN_CASES_PATH: &str = "/internal/v1/evaluate/admin/cases";
58pub const ADMIN_SUITES_PATH: &str = "/internal/v1/evaluate/admin/suites";
59pub const ADMIN_RUNS_PATH: &str = "/internal/v1/evaluate/admin/runs";
60pub const ADMIN_RUN_CASES_PATH: &str = "/internal/v1/evaluate/admin/runs/cases";
61pub const ADMIN_CANCEL_RUN_PATH: &str = "/internal/v1/evaluate/admin/runs/cancel";
62pub const ADMIN_REPORT_PATH: &str = "/internal/v1/evaluate/admin/reports/get";
63pub const ADMIN_REVIEWS_PATH: &str = "/internal/v1/evaluate/admin/reviews";
64pub const ADMIN_CLAIM_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/claim";
65pub const ADMIN_COMPLETE_REVIEW_PATH: &str = "/internal/v1/evaluate/admin/reviews/complete";
66pub const ADMIN_METRIC_HEALTH_PATH: &str = "/internal/v1/evaluate/admin/metrics/health";
67
68// ---------------------------------------------------------------------------
69// Eval case types
70// ---------------------------------------------------------------------------
71
72#[derive(Debug, Clone, Serialize, Deserialize)]
73#[serde(rename_all = "camelCase", deny_unknown_fields)]
74pub struct EvalTurn {
75    pub user: String,
76}
77
78#[derive(Debug, Clone, Serialize, Deserialize)]
79#[serde(rename_all = "camelCase", deny_unknown_fields)]
80pub struct VerifyCommand {
81    pub name: String,
82    pub command: String,
83    #[serde(default)]
84    pub expected_exit_code: i32,
85    #[serde(default, skip_serializing_if = "Option::is_none")]
86    pub output_contains: Option<Vec<String>>,
87}
88
89// ---------------------------------------------------------------------------
90// API request/response types
91// ---------------------------------------------------------------------------
92
93#[derive(Debug, Clone, Serialize, Deserialize)]
94#[serde(rename_all = "camelCase")]
95pub struct EvalReport {
96    pub id: String,
97    pub total_cases: usize,
98    pub passed_cases: usize,
99    pub failed_cases: usize,
100    pub infrastructure_failure_cases: usize,
101    /// Cases whose immutable schema/fixture/evaluator configuration could not
102    /// produce a meaningful quality verdict. Kept separate from provider or
103    /// runtime infrastructure failures.
104    #[serde(default)]
105    pub invalid_case_count: usize,
106    pub non_infrastructure_pass_rate: f64,
107    /// Per-evaluator slices. The aggregate pass rate above cannot say which
108    /// dimension moved; these can.
109    #[serde(default)]
110    pub metrics: Vec<MetricSummary>,
111    pub case_results: Vec<EvalCaseResult>,
112    pub created_at: String,
113    pub duration_ms: u64,
114    #[serde(default)]
115    pub total_cost_micros: u64,
116    /// Judge cost, reported apart from the subject's execution cost.
117    #[serde(default)]
118    pub judge_cost_micros: u64,
119    #[serde(default)]
120    pub p95_case_duration_ms: u64,
121}
122
123#[derive(Debug, Clone, Serialize, Deserialize)]
124#[serde(rename_all = "camelCase")]
125pub struct EvalCaseResult {
126    pub case_id: String,
127    /// Immutable Case version this result was produced from. Trend, flake and
128    /// regression statistics key on the pair, never on the bare id.
129    pub case_version: String,
130    pub category: String,
131    pub passed: bool,
132    pub infrastructure_failure: bool,
133    /// Structured terminal classification for non-quality failures. The
134    /// legacy `infrastructure_failure` boolean remains for wire compatibility.
135    #[serde(default, skip_serializing_if = "Option::is_none")]
136    pub failure_kind: Option<CaseFailureKind>,
137    pub turns_completed: usize,
138    pub total_turns: usize,
139    /// Verdicts of the Case's own inline assertions.
140    #[serde(default)]
141    pub assertion_results: Vec<AssertionVerdict>,
142    /// Versioned evaluator outputs.
143    #[serde(default)]
144    pub evaluator_verdicts: Vec<EvaluatorVerdict>,
145    #[serde(default, skip_serializing_if = "Option::is_none")]
146    pub error: Option<String>,
147    pub duration_ms: u64,
148    #[serde(default)]
149    pub cost_micros: u64,
150    /// Durable experiment identity, retained so repeated samples from
151    /// different variants are never conflated as flakiness.
152    #[serde(default)]
153    pub sample_index: u32,
154    #[serde(default, skip_serializing_if = "Option::is_none")]
155    pub variant_id: Option<String>,
156}
157
158#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
159#[serde(rename_all = "snake_case")]
160pub enum CaseFailureKind {
161    InfraFailed,
162    InvalidCase,
163}
164
165#[derive(Debug, Clone, Serialize, Deserialize)]
166#[serde(rename_all = "camelCase")]
167pub struct AssertionVerdict {
168    pub rule: String,
169    pub passed: bool,
170    #[serde(default, skip_serializing_if = "Option::is_none")]
171    pub detail: Option<String>,
172}
173
174/// Versioned typed assertion schema used by immutable Case versions.
175#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
176#[serde(
177    tag = "kind",
178    rename_all = "snake_case",
179    rename_all_fields = "camelCase",
180    deny_unknown_fields
181)]
182pub enum TypedAssertion {
183    OutputContains {
184        turn: usize,
185        values: Vec<String>,
186        #[serde(default)]
187        any: bool,
188    },
189    WorkspaceFile {
190        path: String,
191        #[serde(default)]
192        contains: Vec<String>,
193        #[serde(default, skip_serializing_if = "Option::is_none")]
194        max_bytes: Option<usize>,
195    },
196    CommandExit {
197        name: String,
198        command_ref: String,
199        expected_exit_code: i32,
200    },
201    TraceEvent {
202        event_type: String,
203        #[serde(default)]
204        minimum_count: usize,
205    },
206    ArtifactExists {
207        artifact_kind: String,
208    },
209    /// Repository-owned evaluator executed by the Agent Runtime against the
210    /// concrete trace and Workspace produced for this Case.
211    AgentEvaluator {
212        evaluator_ref: String,
213        #[serde(default)]
214        config: Value,
215    },
216    /// A custom evaluator that is executed by the isolated executor. The
217    /// control plane accepts only an attested result for the exact immutable
218    /// plugin reference; it never loads plugin code into the service process.
219    Plugin {
220        plugin_ref: String,
221        #[serde(default)]
222        config: Value,
223    },
224}
225
226/// Immutable reference to a catalog resource. Evaluate snapshots the resolved
227/// version into a CaseAttempt; mutable aliases are deliberately unsupported.
228#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
229#[serde(rename_all = "camelCase", deny_unknown_fields)]
230pub struct VersionedResourceRef {
231    pub id: String,
232    pub version: String,
233}
234
235/// Agent is an optimistic precondition; Runtime ID remains the execution
236/// selector and Runtime is the authoritative owner of their binding.
237#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
238#[serde(rename_all = "camelCase", deny_unknown_fields)]
239pub struct EvaluationSubject {
240    pub agent_id: String,
241    pub runtime_id: String,
242}
243
244#[derive(Debug, Clone, Serialize, Deserialize)]
245#[serde(rename_all = "camelCase", deny_unknown_fields)]
246pub struct RegisterEnvironmentProfileRequest {
247    pub profile_id: String,
248    pub version: String,
249    /// Existing Workspace Environment Space selected for the Task. Provisioned
250    /// environments can be added later behind the same immutable profile.
251    #[serde(default, skip_serializing_if = "Option::is_none")]
252    pub space_id: Option<String>,
253    /// Immutable Workspace image used to materialize this evaluation target.
254    #[serde(default, skip_serializing_if = "Option::is_none")]
255    pub snapshot_id: Option<String>,
256    /// Concrete Computer paired with `space_id` for this evaluation run.
257    #[serde(default, skip_serializing_if = "Option::is_none")]
258    pub computer_id: Option<String>,
259    #[serde(default)]
260    pub metadata: std::collections::BTreeMap<String, String>,
261}
262
263#[derive(Debug, Clone, Serialize, Deserialize)]
264#[serde(rename_all = "camelCase", deny_unknown_fields)]
265pub struct EnvironmentProfileVersion {
266    pub tenant_id: String,
267    pub project_id: String,
268    pub profile_id: String,
269    pub version: String,
270    pub digest: String,
271    #[serde(default, skip_serializing_if = "Option::is_none")]
272    pub space_id: Option<String>,
273    #[serde(default, skip_serializing_if = "Option::is_none")]
274    pub snapshot_id: Option<String>,
275    #[serde(default, skip_serializing_if = "Option::is_none")]
276    pub computer_id: Option<String>,
277    #[serde(default)]
278    pub metadata: std::collections::BTreeMap<String, String>,
279    pub created_at_ms: i64,
280}
281
282#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
283#[serde(
284    tag = "kind",
285    rename_all = "snake_case",
286    rename_all_fields = "camelCase",
287    deny_unknown_fields
288)]
289pub enum EvaluatorImplementation {
290    /// Pure, deterministic assertions evaluated from a bounded Evidence view.
291    BuiltinAssertions { assertions: Vec<TypedAssertion> },
292    /// Model-graded evaluator executed by the isolated executor. The control
293    /// plane only maps its attested evidence onto a verdict.
294    LlmJudge {
295        /// Pinned judge model. Changing it requires a new evaluator version so
296        /// judge drift can never be mistaken for subject regression.
297        #[serde(rename = "modelRef", alias = "model_ref")]
298        model_ref: String,
299        /// Digest of the rubric prompt handed to the judge.
300        #[serde(rename = "promptDigest", alias = "prompt_digest")]
301        prompt_digest: String,
302        #[serde(rename = "maxCostMicros", alias = "max_cost_micros", default)]
303        max_cost_micros: u64,
304    },
305}
306
307#[cfg(test)]
308mod evaluator_implementation_tests {
309    use super::*;
310    use serde_json::json;
311
312    #[test]
313    fn llm_judge_json_uses_camel_case_and_accepts_legacy_snake_case() {
314        let canonical = json!({
315            "kind": "llm_judge",
316            "modelRef": "model-a",
317            "promptDigest": "sha256:prompt",
318            "maxCostMicros": 42
319        });
320        let canonical_value: EvaluatorImplementation =
321            serde_json::from_value(canonical.clone()).expect("camelCase contract");
322        assert_eq!(
323            serde_json::to_value(&canonical_value).expect("serialize canonical"),
324            canonical
325        );
326
327        let legacy: EvaluatorImplementation = serde_json::from_value(json!({
328            "kind": "llm_judge",
329            "model_ref": "model-a",
330            "prompt_digest": "sha256:prompt",
331            "max_cost_micros": 42
332        }))
333        .expect("legacy snakeCase contract");
334        assert_eq!(canonical_value, legacy);
335    }
336}
337
338impl EvaluatorImplementation {
339    pub const fn is_model_graded(&self) -> bool {
340        matches!(self, Self::LlmJudge { .. })
341    }
342}
343
344/// Human-label agreement accumulated for one evaluator id.
345#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
346#[serde(rename_all = "camelCase")]
347pub struct JudgeCalibration {
348    pub labeled_samples: u64,
349    pub agreed_samples: u64,
350}
351
352impl JudgeCalibration {
353    pub fn agreement(&self) -> f64 {
354        if self.labeled_samples == 0 {
355            0.0
356        } else {
357            self.agreed_samples as f64 / self.labeled_samples as f64
358        }
359    }
360}
361
362/// Minimum human-label evidence before a model-graded evaluator may gate.
363pub const JUDGE_PROMOTION_MIN_LABELS: u64 = 50;
364pub const JUDGE_PROMOTION_MIN_AGREEMENT: f64 = 0.9;
365
366#[derive(Debug, Clone, Serialize, Deserialize)]
367#[serde(rename_all = "camelCase", deny_unknown_fields)]
368pub struct RegisterEvaluatorVersionRequest {
369    pub evaluator_id: String,
370    pub version: String,
371    pub implementation: EvaluatorImplementation,
372    #[serde(default)]
373    pub metric: MetricSpec,
374}
375
376#[derive(Debug, Clone, Serialize, Deserialize)]
377#[serde(rename_all = "camelCase", deny_unknown_fields)]
378pub struct EvaluatorVersion {
379    pub tenant_id: String,
380    pub project_id: String,
381    pub evaluator_id: String,
382    pub version: String,
383    pub digest: String,
384    pub implementation: EvaluatorImplementation,
385    #[serde(default)]
386    pub metric: MetricSpec,
387    pub created_at_ms: i64,
388}
389
390#[derive(Debug, Clone, Serialize, Deserialize)]
391#[serde(rename_all = "camelCase")]
392pub struct EvaluatorVerdict {
393    pub evaluator_id: String,
394    pub evaluator_version: String,
395    pub passed: bool,
396    #[serde(default)]
397    pub class: MetricClass,
398    #[serde(default)]
399    pub decision: MetricDecision,
400    /// Judge-side cost, kept apart from the subject's execution cost.
401    #[serde(default)]
402    pub cost_micros: u64,
403    #[serde(default)]
404    pub assertions: Vec<AssertionVerdict>,
405}
406
407/// Immutable Case content. A Case is the node that owns the evaluation
408/// dependency edges, so it is registered and versioned in its own right and a
409/// Suite only references it.
410#[derive(Debug, Clone, Serialize, Deserialize)]
411#[serde(rename_all = "camelCase", deny_unknown_fields)]
412pub struct RegisterEvalCaseRequest {
413    pub case_id: String,
414    pub version: String,
415    pub category: String,
416    #[serde(default)]
417    pub turns: Vec<EvalTurn>,
418    /// Small, immutable text fixtures materialised into the selected Space
419    /// before Runtime executes the case. Large fixtures remain external and
420    /// are referenced through `fixtureRef`.
421    #[serde(default)]
422    pub workspace_files: std::collections::BTreeMap<String, String>,
423    #[serde(default)]
424    pub assertions: Vec<TypedAssertion>,
425    /// Versioned environment selection.
426    #[serde(default, skip_serializing_if = "Option::is_none")]
427    pub environment_profile: Option<VersionedResourceRef>,
428    /// Domain-owned evaluator policies executed after evidence collection.
429    #[serde(default)]
430    pub evaluator_versions: Vec<VersionedResourceRef>,
431    #[serde(default)]
432    pub verify_commands: Vec<VerifyCommand>,
433    #[serde(default, skip_serializing_if = "Option::is_none")]
434    pub fixture_ref: Option<String>,
435    #[serde(default)]
436    pub budget: EvalBudget,
437}
438
439#[derive(Debug, Clone, Serialize, Deserialize)]
440#[serde(rename_all = "camelCase", deny_unknown_fields)]
441pub struct EvalCaseVersion {
442    pub tenant_id: String,
443    pub project_id: String,
444    pub case_id: String,
445    pub version: String,
446    pub digest: String,
447    pub category: String,
448    #[serde(default)]
449    pub turns: Vec<EvalTurn>,
450    #[serde(default)]
451    pub workspace_files: std::collections::BTreeMap<String, String>,
452    #[serde(default)]
453    pub assertions: Vec<TypedAssertion>,
454    #[serde(default, skip_serializing_if = "Option::is_none")]
455    pub environment_profile: Option<VersionedResourceRef>,
456    #[serde(default)]
457    pub evaluator_versions: Vec<VersionedResourceRef>,
458    #[serde(default)]
459    pub verify_commands: Vec<VerifyCommand>,
460    #[serde(default, skip_serializing_if = "Option::is_none")]
461    pub fixture_ref: Option<String>,
462    #[serde(default)]
463    pub budget: EvalBudget,
464    pub created_at_ms: i64,
465}
466
467impl EvalCaseVersion {
468    pub fn reference(&self) -> VersionedResourceRef {
469        VersionedResourceRef {
470            id: self.case_id.clone(),
471            version: self.version.clone(),
472        }
473    }
474}
475
476#[derive(Debug, Clone, Serialize, Deserialize)]
477#[serde(rename_all = "camelCase", deny_unknown_fields)]
478pub struct EvalBudget {
479    pub timeout_ms: u64,
480    pub max_output_bytes: usize,
481    pub max_cost_micros: u64,
482}
483
484impl Default for EvalBudget {
485    fn default() -> Self {
486        Self {
487            timeout_ms: 300_000,
488            max_output_bytes: 8 * 1024 * 1024,
489            max_cost_micros: 0,
490        }
491    }
492}
493
494/// Ordered selection of immutable Case versions. A Suite adds no content of
495/// its own, so changing one Case never rewrites the others.
496#[derive(Debug, Clone, Serialize, Deserialize)]
497#[serde(rename_all = "camelCase", deny_unknown_fields)]
498pub struct EvalSuiteVersion {
499    pub tenant_id: String,
500    pub project_id: String,
501    pub suite_id: String,
502    pub version: String,
503    pub manifest_digest: String,
504    pub schema_version: u32,
505    pub cases: Vec<VersionedResourceRef>,
506    pub created_at_ms: i64,
507}
508
509#[derive(Debug, Clone, Serialize, Deserialize)]
510#[serde(rename_all = "camelCase", deny_unknown_fields)]
511pub struct RegisterSuiteRequest {
512    pub suite_id: String,
513    pub version: String,
514    pub manifest_digest: String,
515    pub schema_version: u32,
516    pub cases: Vec<VersionedResourceRef>,
517}
518
519#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
520#[serde(rename_all = "snake_case")]
521pub enum OperationState {
522    Queued,
523    Running,
524    Succeeded,
525    Failed,
526    Canceling,
527    Canceled,
528}
529
530#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
531#[serde(rename_all = "snake_case")]
532pub enum CaseRunState {
533    Queued,
534    Running,
535    Passed,
536    QualityFailed,
537    InfraFailed,
538    InvalidCase,
539    Canceled,
540}
541
542#[derive(Debug, Clone, Serialize, Deserialize)]
543#[serde(rename_all = "camelCase", deny_unknown_fields)]
544pub struct CreateEvalRunRequest {
545    /// Suite version to expand. Mutually exclusive with `cases`.
546    #[serde(default, skip_serializing_if = "Option::is_none")]
547    pub suite: Option<VersionedResourceRef>,
548    /// Ad-hoc Case selection. Because Case is a catalog resource, a run can be
549    /// composed without first inventing a Suite for it.
550    #[serde(default)]
551    pub cases: Vec<VersionedResourceRef>,
552    pub subject_version: String,
553    pub executor_profile: String,
554    /// Concrete execution target. New Runtime Task based runs require it;
555    /// subject_version remains as a compatibility/report label.
556    #[serde(default, skip_serializing_if = "Option::is_none")]
557    pub subject: Option<EvaluationSubject>,
558    #[serde(default)]
559    pub seed: u64,
560    #[serde(default = "default_concurrency")]
561    pub concurrency: usize,
562    #[serde(default)]
563    pub idempotency_key: String,
564    /// Independent deterministic samples per `(case, variant)`.
565    #[serde(default = "default_samples")]
566    pub samples: u32,
567    /// Empty preserves the single-subject run. A non-empty matrix expands
568    /// the run without requiring consumers to coordinate many operations.
569    #[serde(default)]
570    pub experiment_matrix: Vec<ExperimentVariant>,
571}
572
573fn default_concurrency() -> usize {
574    8
575}
576
577fn default_samples() -> u32 {
578    1
579}
580
581#[derive(Debug, Clone, Serialize, Deserialize)]
582#[serde(rename_all = "camelCase", deny_unknown_fields)]
583pub struct ExperimentVariant {
584    pub id: String,
585    pub subject_version: String,
586    pub executor_profile: String,
587    #[serde(default)]
588    pub parameters: Value,
589}
590
591#[derive(Debug, Clone, Serialize, Deserialize)]
592#[serde(rename_all = "camelCase")]
593pub struct EvalOperation {
594    pub id: String,
595    pub run_id: String,
596    pub state: OperationState,
597    pub phase: String,
598    pub attempt: u32,
599    pub completed_cases: usize,
600    pub total_cases: usize,
601    #[serde(default, skip_serializing_if = "Option::is_none")]
602    pub report_id: Option<String>,
603    pub created_at_ms: i64,
604    pub updated_at_ms: i64,
605    pub version: i64,
606    #[serde(default, skip_serializing_if = "Option::is_none")]
607    pub next_poll_after_ms: Option<u64>,
608    #[serde(default, skip_serializing_if = "Option::is_none")]
609    pub error_code: Option<String>,
610}
611
612#[derive(Debug, Clone, Serialize, Deserialize)]
613#[serde(rename_all = "camelCase", deny_unknown_fields)]
614pub struct GetOperationRequest {
615    pub operation_id: String,
616}
617
618#[derive(Debug, Clone, Serialize, Deserialize)]
619#[serde(rename_all = "camelCase", deny_unknown_fields)]
620pub struct CancelRunRequest {
621    pub run_id: String,
622}
623
624#[derive(Debug, Clone, Serialize, Deserialize)]
625#[serde(rename_all = "camelCase", deny_unknown_fields)]
626pub struct ListCaseRunsRequest {
627    pub run_id: String,
628    #[serde(default, skip_serializing_if = "Option::is_none")]
629    pub cursor: Option<String>,
630    #[serde(default, skip_serializing_if = "Option::is_none")]
631    pub state: Option<CaseRunState>,
632    #[serde(default = "default_case_page")]
633    pub limit: usize,
634}
635
636fn default_case_page() -> usize {
637    50
638}
639
640#[derive(Debug, Clone, Serialize, Deserialize)]
641#[serde(rename_all = "camelCase")]
642pub struct CaseRunSummary {
643    pub id: String,
644    pub run_id: String,
645    pub case_id: String,
646    pub case_version: String,
647    pub position: usize,
648    pub state: CaseRunState,
649    pub attempt: u32,
650    pub duration_ms: u64,
651    #[serde(default, skip_serializing_if = "Option::is_none")]
652    pub error: Option<String>,
653    #[serde(default)]
654    pub evidence_refs: Vec<EvidenceRef>,
655    #[serde(default)]
656    pub sample_index: u32,
657    #[serde(default, skip_serializing_if = "Option::is_none")]
658    pub variant_id: Option<String>,
659}
660
661#[derive(Debug, Clone, Serialize, Deserialize)]
662#[serde(rename_all = "camelCase")]
663pub struct CaseRunPage {
664    pub items: Vec<CaseRunSummary>,
665    #[serde(default, skip_serializing_if = "Option::is_none")]
666    pub next_cursor: Option<String>,
667}
668
669#[derive(Debug, Clone, Serialize, Deserialize)]
670#[serde(rename_all = "camelCase", deny_unknown_fields)]
671pub struct EvalAccessScope {
672    pub tenant_id: String,
673    pub project_id: String,
674    pub subject: String,
675}
676
677#[derive(Debug, Clone, Serialize, Deserialize)]
678#[serde(rename_all = "camelCase", deny_unknown_fields)]
679pub struct SandboxExecutionRequest {
680    /// Tenant/project ownership is passed explicitly so the isolated executor
681    /// can only finalize evidence in the namespace later validated by the
682    /// control plane.
683    pub scope: EvalAccessScope,
684    pub operation_id: String,
685    pub run_id: String,
686    pub case_run_id: String,
687    pub subject_version: String,
688    pub executor_profile: String,
689    #[serde(default, skip_serializing_if = "Option::is_none")]
690    pub subject: Option<EvaluationSubject>,
691    #[serde(default, skip_serializing_if = "Option::is_none")]
692    pub environment_profile: Option<EnvironmentProfileVersion>,
693    #[serde(default)]
694    pub evaluator_versions: Vec<EvaluatorVersion>,
695    pub seed: u64,
696    pub case: EvalCaseVersion,
697    pub fencing_token: i64,
698    #[serde(default)]
699    pub sample_index: u32,
700    #[serde(default, skip_serializing_if = "Option::is_none")]
701    pub variant: Option<ExperimentVariant>,
702}
703
704#[derive(Debug, Clone, Serialize, Deserialize)]
705#[serde(rename_all = "camelCase", deny_unknown_fields)]
706pub struct PluginEvaluationEvidence {
707    pub plugin_ref: String,
708    pub passed: bool,
709    #[serde(default)]
710    pub detail: String,
711    /// Attestation produced by the configured sandbox boundary. Production
712    /// evaluation fails closed when it is absent.
713    pub attestation: String,
714}
715
716/// Attested outcome of a model-graded evaluator. Produced by the isolated
717/// executor; the control plane never calls a judge model itself.
718#[derive(Debug, Clone, Serialize, Deserialize)]
719#[serde(rename_all = "camelCase", deny_unknown_fields)]
720pub struct JudgeEvaluationEvidence {
721    pub evaluator_id: String,
722    pub evaluator_version: String,
723    pub passed: bool,
724    /// Judge model actually used, checked against the pinned reference.
725    pub model_ref: String,
726    #[serde(default)]
727    pub detail: String,
728    #[serde(default)]
729    pub cost_micros: u64,
730    pub attestation: String,
731}
732
733/// Stable input passed to a repository-owned Agent evaluation function.
734/// It is deliberately independent from the function implementation language.
735#[derive(Debug, Clone, Default, Serialize, Deserialize)]
736#[serde(rename_all = "camelCase", deny_unknown_fields)]
737pub struct AgentEvaluationInput {
738    pub trace_id: String,
739    #[serde(default, skip_serializing_if = "Option::is_none")]
740    pub snapshot_id: Option<String>,
741    #[serde(default, skip_serializing_if = "Option::is_none")]
742    pub computer_id: Option<String>,
743    #[serde(default, skip_serializing_if = "Option::is_none")]
744    pub space_id: Option<String>,
745    #[serde(default)]
746    pub responses: Vec<String>,
747    #[serde(default)]
748    pub workspace_files: std::collections::BTreeMap<String, String>,
749    #[serde(default)]
750    pub command_exit_codes: std::collections::BTreeMap<String, i32>,
751    #[serde(default)]
752    pub trace_event_counts: std::collections::BTreeMap<String, usize>,
753    #[serde(default)]
754    pub artifact_kinds: Vec<String>,
755}
756
757/// Stable output produced by one repository-owned Agent evaluation function.
758#[derive(Debug, Clone, Serialize, Deserialize)]
759#[serde(rename_all = "camelCase", deny_unknown_fields)]
760pub struct AgentEvaluationOutput {
761    pub evaluator_ref: String,
762    pub passed: bool,
763    #[serde(default)]
764    pub assertions: Vec<AssertionVerdict>,
765    #[serde(default, skip_serializing_if = "String::is_empty")]
766    pub detail: String,
767}
768
769/// Immutable evidence object owned by the scoped Evaluate namespace.
770#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
771#[serde(rename_all = "camelCase", deny_unknown_fields)]
772pub struct EvidenceRef {
773    pub reference: String,
774    pub tenant_id: String,
775    pub project_id: String,
776    pub run_id: String,
777    pub case_run_id: String,
778    /// Lower-case `sha256:<64 hex>` digest of the finalized bytes.
779    pub digest: String,
780    pub size_bytes: u64,
781    /// Attestation bound to the isolated executor response boundary.
782    pub attestation: String,
783}
784
785/// Bounded evidence returned by the isolated executor. Verdict calculation
786/// remains owned by Evaluate and is deterministic from this typed structure.
787#[derive(Debug, Clone, Default, Serialize, Deserialize)]
788#[serde(rename_all = "camelCase", deny_unknown_fields)]
789pub struct SandboxExecutionEvidence {
790    #[serde(default)]
791    pub responses: Vec<String>,
792    #[serde(default)]
793    pub workspace_files: std::collections::BTreeMap<String, String>,
794    #[serde(default)]
795    pub command_exit_codes: std::collections::BTreeMap<String, i32>,
796    #[serde(default)]
797    pub trace_event_counts: std::collections::BTreeMap<String, usize>,
798    #[serde(default)]
799    pub artifact_kinds: Vec<String>,
800    /// Immutable blob/object references owned by Evaluate's evidence namespace.
801    #[serde(default)]
802    pub evidence_refs: Vec<EvidenceRef>,
803    #[serde(default)]
804    pub plugin_evaluations: Vec<PluginEvaluationEvidence>,
805    #[serde(default)]
806    pub judge_evaluations: Vec<JudgeEvaluationEvidence>,
807    #[serde(default)]
808    pub agent_evaluations: Vec<AgentEvaluationOutput>,
809    #[serde(default, skip_serializing_if = "Option::is_none")]
810    pub infrastructure_error: Option<String>,
811    /// A sandbox may explicitly reject an immutable case as unsupported or
812    /// invalid. This is deliberately typed so provider text is never parsed as
813    /// a terminal-state code.
814    #[serde(default, skip_serializing_if = "Option::is_none")]
815    pub invalid_case_error: Option<String>,
816    #[serde(default)]
817    pub duration_ms: u64,
818    /// Executor-measured cost in millionths of the configured billing unit.
819    #[serde(default)]
820    pub cost_micros: u64,
821}
822
823/// Runtime-facing pull protocol. `evaluate_task_id` is the durable
824/// [`EvalOperation::id`] returned when the SDK/CLI configures a run. Infra
825/// never needs a callback URL for the Runtime.
826#[derive(Debug, Clone, Serialize, Deserialize)]
827#[serde(rename_all = "camelCase", deny_unknown_fields)]
828pub struct ClaimEvaluationCaseRequest {
829    pub evaluate_task_id: String,
830    #[serde(default = "default_driver_lease_ms")]
831    pub lease_ms: i64,
832}
833
834fn default_driver_lease_ms() -> i64 {
835    60_000
836}
837
838#[derive(Debug, Clone, Serialize, Deserialize)]
839#[serde(rename_all = "camelCase", deny_unknown_fields)]
840pub struct EvaluationCaseLease {
841    pub execution: SandboxExecutionRequest,
842    pub lease_until_ms: i64,
843}
844
845#[derive(Debug, Clone, Serialize, Deserialize)]
846#[serde(rename_all = "camelCase", deny_unknown_fields)]
847pub struct ClaimEvaluationCaseResponse {
848    pub operation: EvalOperation,
849    #[serde(default, skip_serializing_if = "Option::is_none")]
850    pub case: Option<EvaluationCaseLease>,
851}
852
853#[derive(Debug, Clone, Serialize, Deserialize)]
854#[serde(rename_all = "camelCase", deny_unknown_fields)]
855pub struct RenewEvaluationCaseRequest {
856    pub evaluate_task_id: String,
857    pub case_run_id: String,
858    pub fencing_token: i64,
859    #[serde(default = "default_driver_lease_ms")]
860    pub lease_ms: i64,
861}
862
863#[derive(Debug, Clone, Serialize, Deserialize)]
864#[serde(rename_all = "camelCase", deny_unknown_fields)]
865pub struct RenewEvaluationCaseResponse {
866    pub renewed: bool,
867    pub lease_until_ms: i64,
868}
869
870#[derive(Debug, Clone, Serialize, Deserialize)]
871#[serde(rename_all = "camelCase", deny_unknown_fields)]
872pub struct SubmitEvaluationEvidenceRequest {
873    pub evaluate_task_id: String,
874    pub case_run_id: String,
875    pub fencing_token: i64,
876    pub evidence: SandboxExecutionEvidence,
877}
878
879#[derive(Debug, Clone, Serialize, Deserialize)]
880#[serde(rename_all = "camelCase", deny_unknown_fields)]
881pub struct SubmitEvaluationEvidenceResponse {
882    pub accepted: bool,
883    pub operation: EvalOperation,
884}
885
886#[derive(Debug, Clone, Serialize, Deserialize)]
887#[serde(rename_all = "camelCase", deny_unknown_fields)]
888pub struct GetReportRequest {
889    pub report_id: String,
890}
891
892// ---------------------------------------------------------------------------
893// Evaluate administration query model
894// ---------------------------------------------------------------------------
895
896pub const DEFAULT_ADMIN_PAGE_SIZE: usize = 50;
897pub const MAX_ADMIN_PAGE_SIZE: usize = 200;
898
899fn default_admin_page() -> usize {
900    DEFAULT_ADMIN_PAGE_SIZE
901}
902
903#[derive(Debug, Clone, Serialize, Deserialize)]
904#[serde(rename_all = "camelCase", deny_unknown_fields)]
905pub struct ListAdminCasesRequest {
906    #[serde(default, skip_serializing_if = "Option::is_none")]
907    pub cursor: Option<String>,
908    #[serde(default = "default_admin_page")]
909    pub limit: usize,
910}
911
912impl Default for ListAdminCasesRequest {
913    fn default() -> Self {
914        Self {
915            cursor: None,
916            limit: DEFAULT_ADMIN_PAGE_SIZE,
917        }
918    }
919}
920
921#[derive(Debug, Clone, Serialize, Deserialize)]
922#[serde(rename_all = "camelCase", deny_unknown_fields)]
923pub struct ListAdminSuitesRequest {
924    #[serde(default, skip_serializing_if = "Option::is_none")]
925    pub cursor: Option<String>,
926    #[serde(default = "default_admin_page")]
927    pub limit: usize,
928}
929
930impl Default for ListAdminSuitesRequest {
931    fn default() -> Self {
932        Self {
933            cursor: None,
934            limit: DEFAULT_ADMIN_PAGE_SIZE,
935        }
936    }
937}
938
939#[derive(Debug, Clone, Serialize, Deserialize)]
940#[serde(rename_all = "camelCase", deny_unknown_fields)]
941pub struct ListAdminRunsRequest {
942    #[serde(default, skip_serializing_if = "Option::is_none")]
943    pub cursor: Option<String>,
944    #[serde(default, skip_serializing_if = "Option::is_none")]
945    pub state: Option<OperationState>,
946    #[serde(default = "default_admin_page")]
947    pub limit: usize,
948}
949
950impl Default for ListAdminRunsRequest {
951    fn default() -> Self {
952        Self {
953            cursor: None,
954            state: None,
955            limit: DEFAULT_ADMIN_PAGE_SIZE,
956        }
957    }
958}
959
960#[derive(Debug, Clone, Serialize, Deserialize)]
961#[serde(rename_all = "camelCase", deny_unknown_fields)]
962pub struct ListAdminReviewsRequest {
963    #[serde(default, skip_serializing_if = "Option::is_none")]
964    pub cursor: Option<String>,
965    #[serde(default, skip_serializing_if = "Option::is_none")]
966    pub state: Option<ReviewState>,
967    #[serde(default = "default_admin_page")]
968    pub limit: usize,
969}
970
971impl Default for ListAdminReviewsRequest {
972    fn default() -> Self {
973        Self {
974            cursor: None,
975            state: None,
976            limit: DEFAULT_ADMIN_PAGE_SIZE,
977        }
978    }
979}
980
981#[derive(Debug, Clone, Serialize, Deserialize)]
982#[serde(rename_all = "camelCase")]
983pub struct EvaluateAdminOverview {
984    pub case_versions: u64,
985    pub suite_versions: u64,
986    pub run_states: std::collections::BTreeMap<String, u64>,
987    pub case_states: std::collections::BTreeMap<String, u64>,
988    pub review_states: std::collections::BTreeMap<String, u64>,
989}
990
991#[derive(Debug, Clone, Serialize, Deserialize)]
992#[serde(rename_all = "camelCase")]
993pub struct AdminCaseSummary {
994    pub case_id: String,
995    pub version: String,
996    pub digest: String,
997    pub category: String,
998    pub turns: usize,
999    pub evaluator_versions: Vec<VersionedResourceRef>,
1000    #[serde(default, skip_serializing_if = "Option::is_none")]
1001    pub environment_profile: Option<VersionedResourceRef>,
1002    pub created_at_ms: i64,
1003}
1004
1005#[derive(Debug, Clone, Serialize, Deserialize)]
1006#[serde(rename_all = "camelCase")]
1007pub struct AdminCasePage {
1008    pub items: Vec<AdminCaseSummary>,
1009    #[serde(default, skip_serializing_if = "Option::is_none")]
1010    pub next_cursor: Option<String>,
1011}
1012
1013#[derive(Debug, Clone, Serialize, Deserialize)]
1014#[serde(rename_all = "camelCase")]
1015pub struct AdminSuiteSummary {
1016    pub suite_id: String,
1017    pub version: String,
1018    pub manifest_digest: String,
1019    pub schema_version: u32,
1020    pub case_count: usize,
1021    pub created_at_ms: i64,
1022}
1023
1024#[derive(Debug, Clone, Serialize, Deserialize)]
1025#[serde(rename_all = "camelCase")]
1026pub struct AdminSuitePage {
1027    pub items: Vec<AdminSuiteSummary>,
1028    #[serde(default, skip_serializing_if = "Option::is_none")]
1029    pub next_cursor: Option<String>,
1030}
1031
1032#[derive(Debug, Clone, Serialize, Deserialize)]
1033#[serde(rename_all = "camelCase")]
1034pub struct AdminRunSummary {
1035    pub run_id: String,
1036    pub operation_id: String,
1037    /// Present only when the run was expanded from a Suite version.
1038    #[serde(default, skip_serializing_if = "Option::is_none")]
1039    pub suite: Option<VersionedResourceRef>,
1040    pub subject_version: String,
1041    pub executor_profile: String,
1042    #[serde(default, skip_serializing_if = "Option::is_none")]
1043    pub subject: Option<EvaluationSubject>,
1044    pub state: OperationState,
1045    pub phase: String,
1046    pub completed_cases: usize,
1047    pub total_cases: usize,
1048    #[serde(default, skip_serializing_if = "Option::is_none")]
1049    pub report_id: Option<String>,
1050    pub created_at_ms: i64,
1051    pub updated_at_ms: i64,
1052    pub version: i64,
1053}
1054
1055#[derive(Debug, Clone, Serialize, Deserialize)]
1056#[serde(rename_all = "camelCase")]
1057pub struct AdminRunPage {
1058    pub items: Vec<AdminRunSummary>,
1059    #[serde(default, skip_serializing_if = "Option::is_none")]
1060    pub next_cursor: Option<String>,
1061}
1062
1063#[derive(Debug, Clone, Serialize, Deserialize)]
1064#[serde(rename_all = "camelCase")]
1065pub struct AdminReviewPage {
1066    pub items: Vec<HumanReview>,
1067    #[serde(default, skip_serializing_if = "Option::is_none")]
1068    pub next_cursor: Option<String>,
1069}
1070
1071/// Error body shared by every non-2xx response.
1072#[derive(Debug, Clone, Serialize, Deserialize)]
1073pub struct ApiError {
1074    pub status: u16,
1075    pub message: String,
1076}
1077
1078#[cfg(test)]
1079mod tests {
1080    use super::{
1081        DEFAULT_ADMIN_PAGE_SIZE, ListAdminCasesRequest, ListAdminReviewsRequest,
1082        ListAdminRunsRequest, ListAdminSuitesRequest, TypedAssertion,
1083    };
1084    use serde_json::json;
1085
1086    #[test]
1087    fn typed_assertion_uses_snake_case_tags_and_camel_case_fields() {
1088        let assertion = TypedAssertion::TraceEvent {
1089            event_type: "task.completed".into(),
1090            minimum_count: 1,
1091        };
1092        let value = serde_json::to_value(&assertion).unwrap();
1093        assert_eq!(
1094            value,
1095            json!({
1096                "kind": "trace_event",
1097                "eventType": "task.completed",
1098                "minimumCount": 1
1099            })
1100        );
1101        assert!(serde_json::from_value::<TypedAssertion>(value).is_ok());
1102    }
1103
1104    #[test]
1105    fn admin_queries_use_the_same_bounded_default_in_rust_and_serde() {
1106        assert_eq!(
1107            ListAdminCasesRequest::default().limit,
1108            DEFAULT_ADMIN_PAGE_SIZE
1109        );
1110        assert_eq!(
1111            ListAdminSuitesRequest::default().limit,
1112            DEFAULT_ADMIN_PAGE_SIZE
1113        );
1114        assert_eq!(
1115            ListAdminRunsRequest::default().limit,
1116            DEFAULT_ADMIN_PAGE_SIZE
1117        );
1118        assert_eq!(
1119            ListAdminReviewsRequest::default().limit,
1120            DEFAULT_ADMIN_PAGE_SIZE
1121        );
1122        let decoded: ListAdminRunsRequest = serde_json::from_value(json!({})).unwrap();
1123        assert_eq!(decoded.limit, DEFAULT_ADMIN_PAGE_SIZE);
1124    }
1125}
agent_evaluate_contract/lib.rs

agent_evaluate_contract/
lib.rs
