1use crate::measurement_data_asset::{
8 EvalPreparedSourceBinding, MeasurementDataAssetPreparationRequest,
9 MeasurementDataAssetPreparationResult,
10};
11use schemars::JsonSchema;
12use serde::{Deserialize, Serialize};
13use std::collections::BTreeMap;
14use std::path::PathBuf;
15
16#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
22pub enum ProtocolVersion {
23 #[serde(rename = "10")]
25 V10,
26}
27
28impl ProtocolVersion {
29 pub const CURRENT: Self = Self::V10;
31
32 #[must_use]
36 pub const fn as_str(self) -> &'static str {
37 match self {
38 Self::V10 => "10",
39 }
40 }
41}
42
43#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
46#[serde(untagged)]
47pub enum SettingValue {
48 Bool(bool),
50 Integer(i64),
52 Float(f64),
54 String(String),
56 Array(Vec<SettingValue>),
58 Object(BTreeMap<String, SettingValue>),
60}
61
62#[derive(
65 Clone, Copy, Debug, Default, Deserialize, Eq, JsonSchema, Ord, PartialEq, PartialOrd, Serialize,
66)]
67#[serde(rename_all = "snake_case")]
68pub enum CaptureMechanism {
69 #[default]
72 ManagedCollection,
73 EngineTrace,
76}
77
78#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
84#[serde(rename_all = "snake_case", deny_unknown_fields)]
85pub enum SyntheticAcceptanceInput {
86 Explicit {
88 acceptance_length: f64,
90 },
91 Curve(SyntheticAcceptanceCurveInput),
95}
96
97#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
99#[serde(deny_unknown_fields)]
100pub struct SyntheticAcceptanceCurveInput {
101 pub model_key: String,
102 #[serde(default, skip_serializing_if = "Option::is_none")]
106 pub thinking_mode: Option<String>,
107 pub text: String,
109 pub sha256: String,
112}
113
114#[derive(Clone, Copy, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
118#[serde(deny_unknown_fields)]
119pub struct SyntheticAcceptanceOutcome {
120 pub acceptance_length: f64,
122 #[serde(default, skip_serializing_if = "Option::is_none")]
126 pub draft_count: Option<u32>,
127}
128
129#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
131#[serde(deny_unknown_fields)]
132pub struct EndpointAssignment {
133 pub host: String,
134 pub port: u16,
135}
136
137#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
139#[serde(rename_all = "snake_case")]
140pub enum EndpointProtocol {
141 Http,
143}
144
145#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
147#[serde(rename_all = "snake_case")]
148pub enum HttpMethod {
149 Post,
151}
152
153#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
158#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
159pub enum AdapterRequest {
160 PlanServe {
163 protocol_version: ProtocolVersion,
164 input: PlanServeInput,
165 },
166 RenderServe {
169 protocol_version: ProtocolVersion,
170 input: RenderServeInput,
171 },
172}
173
174impl AdapterRequest {
175 #[must_use]
177 pub const fn protocol_version(&self) -> ProtocolVersion {
178 match self {
179 Self::PlanServe {
180 protocol_version, ..
181 }
182 | Self::RenderServe {
183 protocol_version, ..
184 } => *protocol_version,
185 }
186 }
187}
188
189#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
191#[serde(deny_unknown_fields)]
192pub struct PlanServeInput {
193 pub model: ServeModelInput,
194 pub topology: ServeTopology,
195 pub state_dir: String,
201 #[serde(default, skip_serializing_if = "Option::is_none")]
202 pub gateway_backend: Option<String>,
203 #[serde(default, skip_serializing_if = "Option::is_none")]
204 pub pd_router_backend: Option<String>,
205 #[serde(default, skip_serializing_if = "Option::is_none")]
206 pub kv_transfer: Option<KvTransferMechanism>,
207 pub roles: Vec<ServeRoleInput>,
208 #[serde(default, skip_serializing_if = "Option::is_none")]
211 pub profiling: Option<CaptureMechanism>,
212 #[serde(default, skip_serializing_if = "Option::is_none")]
215 pub synthetic_acceptance: Option<SyntheticAcceptanceInput>,
216 #[serde(default, skip_serializing_if = "Vec::is_empty")]
219 pub auxiliary_models: Vec<AuxiliaryModelInput>,
220}
221
222#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
225#[serde(deny_unknown_fields)]
226pub struct RenderServeInput {
227 pub model: ServeModelInput,
228 pub topology: ServeTopology,
229 pub state_dir: String,
234 #[serde(default, skip_serializing_if = "Option::is_none")]
235 pub gateway_backend: Option<String>,
236 #[serde(default, skip_serializing_if = "Option::is_none")]
237 pub pd_router_backend: Option<String>,
238 #[serde(default, skip_serializing_if = "Option::is_none")]
239 pub kv_transfer: Option<KvTransferMechanism>,
240 pub allocations: Vec<ServeProcessAllocation>,
241 #[serde(default, skip_serializing_if = "Option::is_none")]
244 pub profiling: Option<CaptureMechanism>,
245 #[serde(default, skip_serializing_if = "Option::is_none")]
248 pub synthetic_acceptance: Option<SyntheticAcceptanceInput>,
249 #[serde(default, skip_serializing_if = "Vec::is_empty")]
252 pub auxiliary_models: Vec<AuxiliaryModelInput>,
253}
254
255#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
257#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
258pub enum AdapterResponse {
259 Ok {
261 protocol_version: ProtocolVersion,
262 result: Box<AdapterResult>,
263 },
264 Error {
266 protocol_version: ProtocolVersion,
267 error: AdapterError,
268 },
269}
270
271impl AdapterResponse {
272 #[must_use]
274 pub const fn protocol_version(&self) -> ProtocolVersion {
275 match self {
276 Self::Ok {
277 protocol_version, ..
278 }
279 | Self::Error {
280 protocol_version, ..
281 } => *protocol_version,
282 }
283 }
284}
285
286#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
289#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
290pub enum AdapterResult {
291 PlanServe { output: Box<PlanServeResult> },
293 RenderServe { output: Box<RenderServeResult> },
295}
296
297#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
299#[serde(deny_unknown_fields)]
300pub struct AdapterError {
301 pub code: AdapterErrorCode,
302 pub message: String,
303}
304
305#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
307#[serde(rename_all = "snake_case")]
308pub enum AdapterErrorCode {
309 InvalidRequest,
311 UnsupportedProtocolVersion,
313 InvalidSettings,
315 Internal,
317 UnsupportedOperation,
319}
320
321#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
324#[serde(deny_unknown_fields)]
325pub struct IntegrationIdentity {
326 pub adapter_id: String,
327 pub adapter_version: String,
328 pub framework: String,
329 pub framework_version: String,
330}
331
332#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
336#[serde(rename_all = "snake_case")]
337pub enum ServeTopology {
338 Single,
340 PrefillDecode,
342}
343
344#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
346#[serde(rename_all = "snake_case")]
347pub enum ServeRoleKind {
348 Serve,
350 Prefill,
352 Decode,
354}
355
356impl ServeRoleKind {
357 #[must_use]
361 pub const fn as_str(self) -> &'static str {
362 match self {
363 Self::Serve => "serve",
364 Self::Prefill => "prefill",
365 Self::Decode => "decode",
366 }
367 }
368}
369
370#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
372#[serde(rename_all = "snake_case")]
373pub enum KvTransferMechanism {
374 Mooncake,
376 Nixl,
378}
379
380#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
385#[serde(default, deny_unknown_fields)]
386pub struct Parallelism {
387 #[serde(skip_serializing_if = "Option::is_none")]
389 pub outer: Option<ParallelismOuter>,
390 #[serde(skip_serializing_if = "Option::is_none")]
392 pub attention: Option<ParallelismAttention>,
393 #[serde(skip_serializing_if = "Option::is_none")]
395 pub experts: Option<ParallelismExperts>,
396}
397
398impl Parallelism {
399 pub fn field_values(&self) -> [(&'static str, Option<u32>); 9] {
404 [
405 (
406 "outer.tensor_parallel_size",
407 self.outer
408 .as_ref()
409 .and_then(|value| value.tensor_parallel_size),
410 ),
411 (
412 "outer.pipeline_parallel_size",
413 self.outer
414 .as_ref()
415 .and_then(|value| value.pipeline_parallel_size),
416 ),
417 (
418 "attention.tensor_parallel_size",
419 self.attention
420 .as_ref()
421 .and_then(|value| value.tensor_parallel_size),
422 ),
423 (
424 "attention.data_parallel_size",
425 self.attention
426 .as_ref()
427 .and_then(|value| value.data_parallel_size),
428 ),
429 (
430 "attention.context_parallel_size",
431 self.attention
432 .as_ref()
433 .and_then(|value| value.context_parallel_size),
434 ),
435 (
436 "experts.tensor_parallel_size",
437 self.experts
438 .as_ref()
439 .and_then(|value| value.tensor_parallel_size),
440 ),
441 (
442 "experts.data_parallel_size",
443 self.experts
444 .as_ref()
445 .and_then(|value| value.data_parallel_size),
446 ),
447 (
448 "experts.expert_parallel_size",
449 self.experts
450 .as_ref()
451 .and_then(|value| value.expert_parallel_size),
452 ),
453 (
454 "experts.dense_tensor_parallel_size",
455 self.experts
456 .as_ref()
457 .and_then(|value| value.dense_tensor_parallel_size),
458 ),
459 ]
460 }
461
462 pub fn merge_from(&mut self, other: &Self) {
465 if let Some(other) = &other.outer {
466 self.outer.get_or_insert_default().merge_from(other);
467 }
468 if let Some(other) = &other.attention {
469 self.attention.get_or_insert_default().merge_from(other);
470 }
471 if let Some(other) = &other.experts {
472 self.experts.get_or_insert_default().merge_from(other);
473 }
474 }
475}
476
477#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
479#[serde(default, deny_unknown_fields)]
480pub struct ParallelismOuter {
481 #[schemars(range(min = 1))]
482 #[serde(skip_serializing_if = "Option::is_none")]
483 pub tensor_parallel_size: Option<u32>,
484 #[schemars(range(min = 1))]
485 #[serde(skip_serializing_if = "Option::is_none")]
486 pub pipeline_parallel_size: Option<u32>,
487}
488
489impl ParallelismOuter {
490 fn merge_from(&mut self, other: &Self) {
491 if other.tensor_parallel_size.is_some() {
492 self.tensor_parallel_size = other.tensor_parallel_size;
493 }
494 if other.pipeline_parallel_size.is_some() {
495 self.pipeline_parallel_size = other.pipeline_parallel_size;
496 }
497 }
498}
499
500#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
502#[serde(default, deny_unknown_fields)]
503pub struct ParallelismAttention {
504 #[schemars(range(min = 1))]
505 #[serde(skip_serializing_if = "Option::is_none")]
506 pub tensor_parallel_size: Option<u32>,
507 #[schemars(range(min = 1))]
508 #[serde(skip_serializing_if = "Option::is_none")]
509 pub data_parallel_size: Option<u32>,
510 #[schemars(range(min = 1))]
511 #[serde(skip_serializing_if = "Option::is_none")]
512 pub context_parallel_size: Option<u32>,
513}
514
515impl ParallelismAttention {
516 fn merge_from(&mut self, other: &Self) {
517 if other.tensor_parallel_size.is_some() {
518 self.tensor_parallel_size = other.tensor_parallel_size;
519 }
520 if other.data_parallel_size.is_some() {
521 self.data_parallel_size = other.data_parallel_size;
522 }
523 if other.context_parallel_size.is_some() {
524 self.context_parallel_size = other.context_parallel_size;
525 }
526 }
527}
528
529#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
531#[serde(default, deny_unknown_fields)]
532pub struct ParallelismExperts {
533 #[schemars(range(min = 1))]
534 #[serde(skip_serializing_if = "Option::is_none")]
535 pub tensor_parallel_size: Option<u32>,
536 #[schemars(range(min = 1))]
537 #[serde(skip_serializing_if = "Option::is_none")]
538 pub data_parallel_size: Option<u32>,
539 #[schemars(range(min = 1))]
540 #[serde(skip_serializing_if = "Option::is_none")]
541 pub expert_parallel_size: Option<u32>,
542 #[schemars(range(min = 1))]
543 #[serde(skip_serializing_if = "Option::is_none")]
544 pub dense_tensor_parallel_size: Option<u32>,
545}
546
547impl ParallelismExperts {
548 fn merge_from(&mut self, other: &Self) {
549 if other.tensor_parallel_size.is_some() {
550 self.tensor_parallel_size = other.tensor_parallel_size;
551 }
552 if other.data_parallel_size.is_some() {
553 self.data_parallel_size = other.data_parallel_size;
554 }
555 if other.expert_parallel_size.is_some() {
556 self.expert_parallel_size = other.expert_parallel_size;
557 }
558 if other.dense_tensor_parallel_size.is_some() {
559 self.dense_tensor_parallel_size = other.dense_tensor_parallel_size;
560 }
561 }
562}
563
564#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
566#[serde(deny_unknown_fields)]
567pub struct ServeModelInput {
568 pub id: String,
569 pub served_name: String,
570}
571
572#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
575#[serde(rename_all = "kebab-case")]
576pub enum AuxiliaryModelKind {
577 DraftModel,
580}
581
582#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
586#[serde(deny_unknown_fields)]
587pub struct AuxiliaryModelInput {
588 pub kind: AuxiliaryModelKind,
589 pub model: ServeModelInput,
590}
591
592#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
595#[serde(deny_unknown_fields)]
596pub struct AuxiliaryModelLocator {
597 pub kind: AuxiliaryModelKind,
598 pub locator: String,
599}
600
601#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
604#[serde(deny_unknown_fields)]
605pub struct ServeRoleInput {
606 pub id: String,
607 pub kind: ServeRoleKind,
608 pub replica_count: u32,
609 pub parallelism: Parallelism,
610 pub settings: BTreeMap<String, SettingValue>,
611}
612
613#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
616#[serde(deny_unknown_fields)]
617pub struct ServeRoleResult {
618 pub id: String,
619 pub kind: ServeRoleKind,
620 pub declared_replica_count: u32,
621 pub effective_replica_count: u32,
622 pub effective_settings: BTreeMap<String, SettingValue>,
623 pub effective_parallelism: Parallelism,
624 #[serde(default, skip_serializing_if = "Option::is_none")]
628 pub public_endpoint: Option<EndpointDeclaration>,
629 #[serde(default)]
630 pub render_inputs: Vec<RenderInputDeclaration>,
631}
632
633#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
637#[serde(deny_unknown_fields)]
638pub struct PlanServeResult {
639 pub integration: IntegrationIdentity,
640 #[serde(default, skip_serializing_if = "Option::is_none")]
644 pub synthetic_acceptance: Option<SyntheticAcceptanceOutcome>,
645 pub roles: Vec<ServeRoleResult>,
646 pub replicas: Vec<ServeReplicaRequirement>,
647 pub links: Vec<ServeRoleLink>,
648 #[serde(default, skip_serializing_if = "Option::is_none")]
649 pub gateway: Option<GatewayPlan>,
650 #[serde(default, skip_serializing_if = "Option::is_none")]
651 pub pd_router: Option<PdRouterPlan>,
652}
653
654#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
657#[serde(deny_unknown_fields)]
658pub struct ServeReplicaRequirement {
659 pub id: String,
660 pub role_id: String,
661 pub replica_index: u32,
662 pub device_count: u32,
663 pub ports: Vec<String>,
664 pub primary_ports: Vec<String>,
665 pub primary_readiness: ReadinessProbe,
666 pub worker_readiness: ReadinessProbe,
667 #[serde(default, skip_serializing_if = "Option::is_none")]
668 pub capture_target: Option<CaptureTargetRequirement>,
669}
670
671#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
674#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
675pub enum ServeRoleLink {
676 RequestRouting {
678 source: String,
679 targets: Vec<String>,
680 },
681 KvTransfer {
683 source: String,
684 target: String,
685 mechanism: KvTransferMechanism,
686 },
687 Bootstrap {
689 source: String,
690 target: String,
691 port: String,
692 },
693 SideChannel {
695 source: String,
696 target: String,
697 port: String,
698 },
699}
700
701#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
704#[serde(deny_unknown_fields)]
705pub struct RenderInputDeclaration {
706 pub source_path: String,
707}
708
709#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
712#[serde(deny_unknown_fields)]
713pub struct SuppliedRenderInput {
714 pub source_path: String,
715 pub text: String,
716 pub sha256: String,
717}
718
719#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
726#[serde(deny_unknown_fields)]
727pub struct EndpointDeclaration {
728 pub protocol: EndpointProtocol,
729 #[serde(default, skip_serializing_if = "Option::is_none")]
730 pub server_metrics: Option<ServerMetricsEndpointRequirement>,
731 #[serde(default, skip_serializing_if = "Option::is_none")]
732 pub prefix_cache_reset: Option<HttpActionSpec>,
733 #[serde(default, skip_serializing_if = "Option::is_none")]
737 pub prefix_cache_conditioning: Option<HttpActionSpec>,
738 #[serde(default, skip_serializing_if = "Option::is_none")]
739 pub prompt_cache_read_zero_representation: Option<PromptCacheReadZeroRepresentation>,
740}
741
742#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
748#[serde(deny_unknown_fields)]
749pub struct EndpointRequirement {
750 pub protocol: EndpointProtocol,
751 pub completions_path: String,
752 pub chat_completions_path: String,
753 #[serde(default, skip_serializing_if = "Option::is_none")]
754 pub server_metrics: Option<ServerMetricsEndpointRequirement>,
755 #[serde(default, skip_serializing_if = "Option::is_none")]
756 pub prefix_cache_reset: Option<HttpActionSpec>,
757 #[serde(default, skip_serializing_if = "Option::is_none")]
761 pub prefix_cache_conditioning: Option<HttpActionSpec>,
762 #[serde(default, skip_serializing_if = "Option::is_none")]
763 pub prompt_cache_read_zero_representation: Option<PromptCacheReadZeroRepresentation>,
764}
765
766#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
768#[serde(deny_unknown_fields)]
769pub struct ServerMetricsEndpointRequirement {
770 pub path: String,
771 #[serde(default, skip_serializing_if = "Option::is_none")]
772 pub port: Option<String>,
773}
774
775#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
777#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
778pub enum ReadinessProbe {
779 Http { path: String },
781 HttpTargetRegistry(Box<HttpTargetRegistryReadiness>),
784 ProcessAlive,
786}
787
788#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
790#[serde(deny_unknown_fields)]
791pub struct HttpTargetRegistryReadiness {
792 pub target_scheme: TargetEndpointScheme,
793 pub readiness_path: String,
794 pub registry_path: String,
795 pub targets_field: String,
796 pub target_url_field: String,
797 pub target_role_field: String,
798 pub target_healthy_field: String,
799 pub target_bootstrap_port_field: String,
800 pub prefill_role_value: String,
801 pub decode_role_value: String,
802 pub prefill_bootstrap_port: String,
803}
804
805#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
807#[serde(deny_unknown_fields)]
808pub struct HttpActionSpec {
809 pub method: HttpMethod,
810 pub path: String,
811}
812
813#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
818#[serde(deny_unknown_fields)]
819pub struct CaptureTargetRequirement {
820 pub mechanism: CaptureMechanism,
823 pub window_control: CaptureWindowControlRequirement,
824}
825
826#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
829#[serde(deny_unknown_fields)]
830pub struct CaptureWindowControlRequirement {
831 pub endpoint: CaptureWindowControlEndpoint,
832 pub start: CaptureWindowHttpActionSpec,
833 pub stop: CaptureWindowHttpActionSpec,
834}
835
836#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
839#[serde(rename_all = "snake_case")]
840pub enum CaptureWindowControlEndpoint {
841 ReplicaEntry,
843 Gateway,
845}
846
847#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
851#[serde(deny_unknown_fields)]
852pub struct CaptureWindowHttpActionSpec {
853 pub method: HttpMethod,
854 pub path: String,
855 #[serde(default, skip_serializing_if = "Option::is_none")]
856 pub body: Option<BTreeMap<String, SettingValue>>,
857}
858
859#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
863#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
864pub enum ServeProcessAllocation {
865 ModelRank {
866 process: String,
867 role: String,
868 role_kind: ServeRoleKind,
869 replica: u32,
870 rank: u32,
871 rank_count: u32,
872 machine: String,
873 devices: Vec<u32>,
874 model_locator: String,
875 #[serde(default, skip_serializing_if = "Vec::is_empty")]
878 auxiliary_model_locators: Vec<AuxiliaryModelLocator>,
879 #[serde(default, skip_serializing_if = "Option::is_none")]
880 endpoint: Option<EndpointAssignment>,
881 ports: BTreeMap<String, EndpointAssignment>,
882 cache: String,
883 #[serde(default, skip_serializing_if = "Option::is_none")]
887 capture_storage: Option<String>,
888 launch: AllocationLaunch,
889 effective_settings: BTreeMap<String, SettingValue>,
890 effective_parallelism: Parallelism,
891 #[serde(default)]
892 links: Vec<ServeRoleLink>,
893 #[serde(default)]
894 dependencies: Vec<String>,
895 #[serde(default)]
896 render_inputs: Vec<SuppliedRenderInput>,
897 },
898 Frontend {
899 process: String,
900 process_role: FrontendProcessRole,
901 components: FrontendComponents,
902 machine: String,
903 devices: Vec<u32>,
904 endpoint: EndpointAssignment,
905 ports: BTreeMap<String, EndpointAssignment>,
906 cache: String,
907 launch: AllocationLaunch,
908 gateway: Box<GatewayPlan>,
909 #[serde(default, skip_serializing_if = "Option::is_none")]
910 pd_router: Option<Box<PdRouterPlan>>,
911 #[serde(default)]
912 links: Vec<ServeRoleLink>,
913 #[serde(default)]
914 dependencies: Vec<String>,
915 #[serde(default)]
916 render_inputs: Vec<SuppliedRenderInput>,
917 },
918}
919
920#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
922#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
923pub enum AllocationLaunch {
924 Local,
925 Ssh { target: String },
926}
927
928#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
931#[serde(deny_unknown_fields)]
932pub struct RenderServeResult {
933 pub integration: IntegrationIdentity,
934 pub processes: Vec<RenderedServeProcess>,
935}
936
937#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
940#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
941pub enum RenderedServeProcess {
942 ModelRank {
943 process: String,
944 role: String,
945 replica: u32,
946 rank: u32,
947 rank_count: u32,
948 launch_files: Vec<LaunchFileDeclaration>,
949 command: ProcessSpec,
950 },
951 Frontend {
952 process: String,
953 process_role: FrontendProcessRole,
954 components: FrontendComponents,
955 launch_files: Vec<LaunchFileDeclaration>,
956 command: ProcessSpec,
957 },
958}
959
960#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
962#[serde(deny_unknown_fields)]
963pub struct LaunchFileDeclaration {
964 pub relative_path: String,
965 pub text: String,
966 pub sha256: String,
967}
968
969#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
971#[serde(deny_unknown_fields)]
972pub struct ProcessSpec {
973 pub argv: Vec<String>,
974 pub env: BTreeMap<String, String>,
975}
976
977#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
983#[serde(rename_all = "snake_case")]
984pub enum RenderSource {
985 ControlPlane,
986 Integration,
987}
988
989#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
991#[serde(rename_all = "snake_case")]
992pub enum FrontendProcessRole {
993 Gateway,
994}
995
996#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
998#[serde(deny_unknown_fields)]
999pub struct FrontendCoRendering {
1000 pub process_role: FrontendProcessRole,
1001}
1002
1003#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1005#[serde(rename_all = "snake_case")]
1006pub enum FrontendGatewayComponent {
1007 Gateway,
1008}
1009
1010#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1012#[serde(rename_all = "snake_case")]
1013pub enum FrontendPdRouterComponent {
1014 PdRouter,
1015}
1016
1017#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1019#[serde(transparent)]
1020pub struct GatewayFrontendBinding(pub [FrontendGatewayComponent; 1]);
1021
1022#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1024#[serde(transparent)]
1025pub struct GatewayPdRouterFrontendBinding(
1026 pub (FrontendGatewayComponent, FrontendPdRouterComponent),
1027);
1028
1029#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1033#[serde(untagged)]
1034pub enum FrontendComponents {
1035 Gateway(GatewayFrontendBinding),
1036 GatewayPdRouter(GatewayPdRouterFrontendBinding),
1037}
1038
1039impl FrontendComponents {
1040 #[must_use]
1041 pub const fn gateway() -> Self {
1042 Self::Gateway(GatewayFrontendBinding([FrontendGatewayComponent::Gateway]))
1043 }
1044
1045 #[must_use]
1046 pub const fn gateway_pd_router() -> Self {
1047 Self::GatewayPdRouter(GatewayPdRouterFrontendBinding((
1048 FrontendGatewayComponent::Gateway,
1049 FrontendPdRouterComponent::PdRouter,
1050 )))
1051 }
1052
1053 #[must_use]
1054 pub const fn includes_pd_router(&self) -> bool {
1055 matches!(self, Self::GatewayPdRouter(_))
1056 }
1057}
1058
1059#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1061#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
1062pub enum GatewayTarget {
1063 Engine { role: String },
1065 PdRouter,
1067}
1068
1069#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1072#[serde(deny_unknown_fields)]
1073pub struct GatewayPlan {
1074 pub backend: String,
1075 pub implementation: String,
1076 pub implementation_version: String,
1077 pub effective_settings: BTreeMap<String, SettingValue>,
1078 pub endpoint: EndpointDeclaration,
1079 pub readiness: ReadinessProbe,
1080 #[serde(default)]
1081 pub ports: Vec<String>,
1082 pub targets: Vec<GatewayTarget>,
1083 #[serde(default)]
1084 pub render_inputs: Vec<RenderInputDeclaration>,
1085 pub render_source: RenderSource,
1086 pub co_rendering: FrontendCoRendering,
1087}
1088
1089#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1091#[serde(deny_unknown_fields)]
1092pub struct PdRoutingPolicies {
1093 pub prefill: String,
1094 pub decode: String,
1095}
1096
1097#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1100#[serde(rename_all = "snake_case")]
1101pub enum FrontendHandoff {
1102 InProcess,
1103}
1104
1105#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1108#[serde(deny_unknown_fields)]
1109pub struct PdRouterPlan {
1110 pub backend: String,
1111 pub implementation: String,
1112 pub implementation_version: String,
1113 pub effective_settings: BTreeMap<String, SettingValue>,
1114 pub policies: PdRoutingPolicies,
1115 pub prefill_role: String,
1116 pub decode_role: String,
1117 pub target_scheme: TargetEndpointScheme,
1118 #[serde(default)]
1119 pub ports: Vec<String>,
1120 pub readiness: ReadinessProbe,
1121 pub handoff: FrontendHandoff,
1122 #[serde(default)]
1123 pub render_inputs: Vec<RenderInputDeclaration>,
1124 pub render_source: RenderSource,
1125 pub co_rendering: FrontendCoRendering,
1126}
1127
1128#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1130#[serde(rename_all = "snake_case")]
1131pub enum TargetEndpointScheme {
1132 Http,
1134 Grpc,
1136}
1137
1138#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1144#[serde(deny_unknown_fields)]
1145pub struct MeasurementModelInput {
1146 pub locator: String,
1147 pub served_name: String,
1148}
1149
1150#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1152#[serde(deny_unknown_fields)]
1153pub struct ClientEndpointInput {
1154 pub protocol: EndpointProtocol,
1155 pub host: String,
1156 pub port: u16,
1157 pub completions_path: String,
1158 pub chat_completions_path: String,
1159 #[serde(default)]
1160 pub server_metrics: Option<ServerMetricsEndpointInput>,
1161 #[serde(default)]
1162 pub prompt_cache_read_zero_representation: Option<PromptCacheReadZeroRepresentation>,
1163}
1164
1165#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1167#[serde(deny_unknown_fields)]
1168pub struct ServerMetricsEndpointInput {
1169 pub path: String,
1170 #[serde(default)]
1171 pub port_name: Option<String>,
1172 pub url: String,
1173}
1174
1175#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1177#[serde(rename_all = "snake_case")]
1178pub enum PromptCacheReadZeroRepresentation {
1179 Explicit,
1180 Omitted,
1181}
1182
1183#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1185#[serde(rename_all = "snake_case")]
1186pub enum ClientStatus {
1187 Succeeded,
1189 Failed,
1191}
1192
1193#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1195#[serde(deny_unknown_fields)]
1196pub struct RawArtifact {
1197 pub name: String,
1198 pub kind: String,
1199 pub path: PathBuf,
1200}
1201
1202#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1207#[serde(deny_unknown_fields)]
1208pub struct EvalClientRequest {
1209 pub protocol_version: ProtocolVersion,
1210 pub workspace_root: PathBuf,
1211 pub workspace_source_exclusions: Vec<PathBuf>,
1212 pub endpoint: ClientEndpointInput,
1213 pub model: MeasurementModelInput,
1214 pub definition: EvalDefinitionInput,
1215 #[serde(default)]
1216 pub prepared_source: Option<EvalPreparedSourceBinding>,
1217 pub case_budget_seconds: f64,
1219 pub artifact_dir: PathBuf,
1220}
1221
1222#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1224#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
1225pub enum EvalDefinitionInput {
1226 #[serde(rename = "openai_smoke")]
1228 OpenAiSmoke {
1229 prompt: String,
1230 max_tokens: u32,
1231 timeout_seconds: u64,
1232 },
1233 LmEval {
1235 task: Box<EvalTaskSourceInput>,
1236 prompt: EvalPromptInput,
1238 #[serde(default)]
1240 declared_prompt: Option<EvalPromptInput>,
1241 #[serde(default)]
1242 request_body: BTreeMap<String, SettingValue>,
1243 limit: Option<u32>,
1244 few_shot: Option<u32>,
1245 seed: Option<u64>,
1248 base_seed: u64,
1252 trials: u32,
1253 max_tokens: Option<u32>,
1254 concurrency: Option<u32>,
1255 metric: String,
1256 #[serde(default)]
1257 metric_filter: Option<String>,
1258 threshold: f64,
1259 timeout_seconds: u64,
1260 },
1261}
1262
1263#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1269#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
1270pub enum EvalPromptInput {
1271 Flat,
1273 ServerChat,
1275}
1276
1277#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1279#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
1280pub enum EvalTaskSourceInput {
1281 BuiltIn { name: String },
1283 Bundled {
1285 name: String,
1286 task_identity: String,
1287 path: PathBuf,
1288 task_closure_sha256: String,
1289 task_definition_sha256: String,
1290 prompt_asset_sha256: String,
1291 dataset_asset_sha256: String,
1292 scorer_sha256: String,
1293 },
1294 WorkspaceYaml { path: PathBuf },
1296}
1297
1298#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1300#[serde(deny_unknown_fields)]
1301pub struct EvalClientResult {
1302 pub schema_version: u32,
1305 pub status: ClientStatus,
1306 pub metrics: BTreeMap<String, f64>,
1307 #[serde(default)]
1308 pub normalized_metrics: BTreeMap<String, EvalNormalizedMetric>,
1309 #[serde(default)]
1310 pub gate: Option<EvalMetricGate>,
1311 #[serde(default)]
1312 pub trial_summary: Option<EvalTrialSummary>,
1313 pub native_command: Vec<String>,
1314 #[serde(default)]
1315 pub native_exit_code: Option<i32>,
1316 #[serde(default)]
1317 pub native_timed_out: bool,
1318 pub raw_artifacts: Vec<RawArtifact>,
1319 pub failure_kind: Option<EvalFailureKind>,
1320 pub error: Option<String>,
1321}
1322
1323#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1325#[serde(rename_all = "snake_case")]
1326pub enum EvalFailureKind {
1327 TaskResolution,
1328 ProbeTokenizer,
1329 ProbeTransport,
1330 ProbeHttp,
1331 ProbeMalformedResponse,
1332 ProbeGeneratedOnlyLogprobs,
1333 ProbeTokenizerAlignment,
1334 MetricNormalization,
1335}
1336
1337#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1339#[serde(rename_all = "snake_case")]
1340pub enum EvalMetricComparison {
1341 AtLeast,
1342 AtMost,
1343}
1344
1345#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1347#[serde(rename_all = "snake_case")]
1348pub enum EvalMetricGateConclusion {
1349 Passed,
1350 Failed,
1351}
1352
1353#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1355#[serde(deny_unknown_fields)]
1356pub struct EvalNormalizedMetric {
1357 pub source_identity: String,
1358 pub metric: String,
1359 pub filter: Option<String>,
1360 pub native_metric_key: String,
1361 pub value: f64,
1362 pub higher_is_better: bool,
1363 pub prompt_authority: EvalPromptInput,
1367}
1368
1369#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1371#[serde(deny_unknown_fields)]
1372pub struct EvalMetricGate {
1373 pub metric: EvalNormalizedMetric,
1374 pub threshold: f64,
1375 pub comparison: EvalMetricComparison,
1376 pub conclusion: EvalMetricGateConclusion,
1377}
1378
1379#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1381#[serde(deny_unknown_fields)]
1382pub struct EvalTrialSummary {
1383 pub requested_trials: u32,
1384 pub issued_trials: u32,
1385 pub unissued_trials: u32,
1386 pub completed_trials: u32,
1387 pub request_failure_trials: u32,
1388 pub passed_trials: u32,
1389 pub pass_rate: Option<f64>,
1390 pub per_trial_metric: String,
1391 pub per_trial_filter: Option<String>,
1392 pub higher_is_better: bool,
1393}
1394
1395#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1401#[serde(deny_unknown_fields)]
1402pub struct BenchClientRequest {
1403 pub protocol_version: ProtocolVersion,
1404 pub endpoint: ClientEndpointInput,
1405 pub model: MeasurementModelInput,
1406 pub definition: BenchDefinitionInput,
1407 #[serde(default)]
1408 pub population: Option<BenchPopulationInput>,
1409 pub case: BenchCaseInput,
1410 pub case_budget_seconds: f64,
1412 pub artifact_dir: PathBuf,
1413}
1414
1415#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1417#[serde(deny_unknown_fields)]
1418pub struct BenchDefinitionInput {
1419 #[serde(default)]
1420 pub request_source: Option<BenchRequestSourceInput>,
1421 #[serde(default)]
1422 pub session_source: Option<BenchSessionSourceInput>,
1423 #[serde(default)]
1424 pub agentic_source: Option<BenchAgenticSourceInput>,
1425 pub prompt: BenchPromptInput,
1426 #[serde(default)]
1427 pub server_metrics: bool,
1428 #[serde(default)]
1429 pub artifact_level: BenchArtifactLevelInput,
1430 pub seed: u64,
1431 #[serde(default)]
1432 pub request_body: BTreeMap<String, SettingValue>,
1433 #[serde(default)]
1434 pub request_slo: Option<BenchRequestSloInput>,
1435 pub timeout_seconds: u64,
1436 pub cache_start: BenchCacheStartInput,
1437}
1438
1439#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1444#[serde(rename_all = "snake_case")]
1445pub enum BenchArtifactLevelInput {
1446 Performance,
1447 #[default]
1448 Diagnostic,
1449}
1450
1451#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1452#[serde(rename_all = "snake_case")]
1453pub enum BenchCacheStartInput {
1454 Uncontrolled,
1455 Cold,
1456 Primed,
1457}
1458
1459#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1462#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
1463pub enum BenchPromptInput {
1464 Flat {
1465 request_representation: BenchRequestRepresentationInput,
1466 route: BenchPromptRouteInput,
1467 rendering_authority: BenchRenderingAuthorityInput,
1468 },
1469 RenderedChat {
1470 #[serde(default)]
1471 chat_template: Option<String>,
1472 #[serde(default)]
1473 chat_template_kwargs: BTreeMap<String, SettingValue>,
1474 request_representation: BenchRequestRepresentationInput,
1475 route: BenchPromptRouteInput,
1476 rendering_authority: BenchRenderingAuthorityInput,
1477 },
1478 ServerChat {
1479 request_representation: BenchRequestRepresentationInput,
1480 route: BenchPromptRouteInput,
1481 rendering_authority: BenchRenderingAuthorityInput,
1482 },
1483}
1484
1485#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1487#[serde(rename_all = "snake_case")]
1488pub enum BenchRequestRepresentationInput {
1489 FlatPrompt,
1490 StructuredMessages,
1491}
1492
1493#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1495#[serde(rename_all = "snake_case")]
1496pub enum BenchPromptRouteInput {
1497 Completions,
1498 ChatCompletions,
1499}
1500
1501#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1503#[serde(rename_all = "snake_case")]
1504pub enum BenchRenderingAuthorityInput {
1505 LocalFlat,
1506 LocalTemplate,
1507 Server,
1508}
1509
1510#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1512#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
1513pub enum BenchRequestSourceInput {
1514 Random {
1517 input_tokens: BenchTokenSelectorInput,
1518 output_tokens: BenchTokenSelectorInput,
1519 #[serde(default)]
1520 prefix_sharing: Option<BenchPrefixSharingInput>,
1521 #[serde(default)]
1522 shared_system_content: Option<BenchSharedSystemContentInput>,
1523 #[serde(default)]
1524 corpus: Option<BenchCorpusInput>,
1525 },
1526 RandomMixture {
1529 shapes: Vec<BenchRandomShapeInput>,
1530 total_weight: u64,
1531 #[serde(default)]
1532 prefix_sharing: Option<BenchPrefixSharingInput>,
1533 },
1534 Dataset {
1537 dataset: String,
1538 #[serde(default)]
1539 profile: Option<String>,
1540 max_input_tokens: u32,
1541 output_tokens: Option<u32>,
1542 catalog: Box<BenchDatasetCatalogInput>,
1543 },
1544 Replay {
1548 path: String,
1551 #[serde(default)]
1552 expected_sha256: Option<String>,
1553 #[serde(default)]
1554 prefix_sharing: Option<BenchPrefixSharingInput>,
1555 },
1556}
1557
1558#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1560#[serde(untagged)]
1561pub enum BenchTokenSelectorInput {
1562 Fixed(u32),
1563 InclusiveUniform(BenchInclusiveUniformInput),
1564}
1565
1566#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1568#[serde(deny_unknown_fields)]
1569pub struct BenchInclusiveUniformInput {
1570 pub kind: BenchTokenDistributionKindInput,
1571 pub min: u32,
1572 pub max: u32,
1573}
1574
1575#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1576#[serde(rename_all = "snake_case")]
1577pub enum BenchTokenDistributionKindInput {
1578 InclusiveUniform,
1579}
1580
1581#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1583#[serde(untagged)]
1584pub enum BenchPrefixSharingInput {
1585 Tokens { shared_prefix_tokens: u32 },
1586 Ratio { shared_prefix_ratio: f64 },
1587}
1588
1589#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1592#[serde(untagged)]
1593pub enum BenchSharedSystemContentInput {
1594 Tokens { tokens: u32 },
1595 Ratio { ratio: f64 },
1596}
1597
1598#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1602#[serde(deny_unknown_fields)]
1603pub struct BenchCorpusInput {
1604 pub path: String,
1607 #[serde(default)]
1608 pub expected_sha256: Option<String>,
1609}
1610
1611#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1613#[serde(deny_unknown_fields)]
1614pub struct BenchRandomShapeInput {
1615 pub input_tokens: u32,
1616 pub output_tokens: u32,
1617 pub weight: u32,
1618}
1619
1620#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1622#[serde(deny_unknown_fields)]
1623pub struct BenchDatasetCatalogInput {
1624 pub dataset: String,
1625 #[serde(default)]
1626 pub profile: Option<String>,
1627 pub source: String,
1628 pub upstream_identity: String,
1629 pub url: String,
1630 pub sha256: String,
1631 pub source_format: String,
1632 pub aiperf_format: String,
1633 #[serde(default)]
1634 pub configuration: Option<String>,
1635 #[serde(default)]
1636 pub split: Option<String>,
1637 #[serde(default)]
1638 pub filter: Option<BenchDatasetFilterInput>,
1639 pub license: String,
1640 pub cache_path: PathBuf,
1641 pub cache_state: BenchDatasetCacheState,
1642 pub materialization_identity: String,
1643 pub provides_output_targets: bool,
1644}
1645
1646#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1647#[serde(deny_unknown_fields)]
1648pub struct BenchDatasetFilterInput {
1649 pub field: String,
1650 pub value: String,
1651}
1652
1653#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1655#[serde(rename_all = "snake_case")]
1656pub enum BenchDatasetCacheState {
1657 Missing,
1658 Present,
1659}
1660
1661#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1663#[serde(deny_unknown_fields)]
1664pub struct BenchSessionSourceInput {
1665 pub dataset: String,
1666 #[serde(default)]
1667 pub profile: Option<String>,
1668 pub max_input_tokens: u32,
1669 pub output_tokens: Option<u32>,
1670 pub inter_turn_delay_scale: f64,
1671 pub max_inter_turn_delay_seconds: Option<f64>,
1672 pub catalog: Box<BenchSessionDatasetCatalogInput>,
1673}
1674
1675#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1678#[serde(deny_unknown_fields)]
1679pub struct BenchSessionDatasetCatalogInput {
1680 pub dataset: String,
1681 #[serde(default)]
1682 pub profile: Option<String>,
1683 pub source: String,
1684 pub upstream_identity: String,
1685 pub url: String,
1686 pub sha256: String,
1687 pub source_format: String,
1688 #[serde(default)]
1689 pub configuration: Option<String>,
1690 #[serde(default)]
1691 pub split: Option<String>,
1692 #[serde(default)]
1693 pub filter: Option<BenchDatasetFilterInput>,
1694 pub license: String,
1695 pub cache_path: PathBuf,
1696 pub cache_state: BenchDatasetCacheState,
1697 pub materialization_identity: String,
1698 pub provides_output_targets: bool,
1699}
1700
1701#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1703#[serde(deny_unknown_fields)]
1704pub struct BenchAgenticSourceInput {
1705 pub dataset: String,
1706 pub profile: String,
1707 pub catalog: Box<BenchAgenticCatalogInput>,
1708}
1709
1710#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1711#[serde(deny_unknown_fields)]
1712pub struct BenchAgenticCatalogInput {
1713 pub repository: String,
1714 pub revision: String,
1715 pub filename: String,
1716 pub sha256: String,
1717 #[serde(default)]
1718 pub cache_path: Option<PathBuf>,
1719 #[serde(default)]
1720 pub cache_state: Option<BenchDatasetCacheState>,
1721 pub trace_count: u32,
1722 pub approximate_bytes: u64,
1723 pub license: String,
1724 pub source_format: String,
1725 pub aiperf_loader: String,
1726 pub materialization_identity: String,
1727 pub scenario: String,
1728 pub concurrency_semantics: String,
1729 pub replay_semantics: String,
1730 pub cache_bust: String,
1731 pub trajectory_start_min: f64,
1732 pub trajectory_start_max: f64,
1733 pub global_idle_gap_cap_seconds: f64,
1734 pub trace_idle_gap_cap_seconds: f64,
1735 pub cache_warmup_requests_per_lane: u64,
1736 pub warmup_grace_seconds: u64,
1737 pub dataset_configuration_timeout_seconds: u64,
1738 pub service_profile_configuration_timeout_seconds: u64,
1739 pub default_duration_seconds: u64,
1740 pub minimum_duration_seconds: u64,
1741 pub failure_threshold: f64,
1742 pub dataset_entries: u32,
1743 pub streaming: bool,
1744 pub ignore_eos: bool,
1745 pub use_server_token_count: bool,
1746 pub gpu_telemetry: bool,
1747 pub server_metric_slice_seconds: u64,
1748 pub required_artifacts: Vec<String>,
1749 pub unavailable_dimensions: Vec<String>,
1750 pub inferencex_repository: String,
1751 pub inferencex_revision: String,
1752 pub inferencex_reference: String,
1753 pub aiperf_revision: String,
1754 pub aiperf_version: String,
1755}
1756
1757#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1759#[serde(deny_unknown_fields)]
1760pub struct BenchCaseInput {
1761 pub load_shape: BenchLoadInput,
1762 pub request_count: u32,
1763 #[serde(default)]
1764 pub warmup_request_count: u32,
1765 #[serde(default)]
1766 pub duration_seconds: Option<u64>,
1767 #[serde(default)]
1768 pub session_count: Option<u32>,
1769 #[serde(default)]
1770 pub warmup_session_count: Option<u32>,
1771}
1772
1773#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1775#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
1776pub enum BenchLoadInput {
1777 ConcurrencyLimited { concurrency: u32 },
1779 RequestRateLimited {
1781 request_rate: f64,
1782 burstiness: Option<f64>,
1783 },
1784 UnboundedRequestRate,
1786}
1787
1788#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1790#[serde(deny_unknown_fields)]
1791pub struct BenchRequestSloInput {
1792 #[serde(default)]
1793 pub request_latency_ms: Option<f64>,
1794 #[serde(default)]
1795 pub ttft_ms: Option<f64>,
1796 #[serde(default)]
1797 pub tpot_ms: Option<f64>,
1798 pub minimum_good_request_ratio: f64,
1799}
1800
1801#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1803#[serde(deny_unknown_fields)]
1804pub struct BenchPopulationInput {
1805 pub path: PathBuf,
1806 pub evidence_path: PathBuf,
1807 pub sha256: String,
1808 pub entries: u32,
1809 pub tpot_applicable: bool,
1810 #[serde(default)]
1811 pub session_templates: Vec<BenchSessionTemplateInput>,
1812}
1813
1814#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1817#[serde(deny_unknown_fields)]
1818pub struct BenchSessionTemplateInput {
1819 pub template_identity: String,
1820 pub turn_count: u32,
1821}
1822
1823#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1826#[serde(deny_unknown_fields)]
1827pub struct BenchPopulationPreparationRequest {
1828 pub protocol_version: ProtocolVersion,
1829 pub model: MeasurementModelInput,
1830 pub tokenizer_backend: String,
1831 pub transformers_version: String,
1832 #[serde(default)]
1833 pub request_source: Option<BenchRequestSourceInput>,
1834 #[serde(default)]
1835 pub session_source: Option<BenchSessionSourceInput>,
1836 pub prompt: BenchPromptInput,
1837 pub cache_start: BenchCacheStartInput,
1838 #[serde(default)]
1839 pub source_path: Option<PathBuf>,
1840 pub required_entries: u32,
1841 pub seed: u64,
1842 #[serde(default)]
1843 pub request_body: BTreeMap<String, SettingValue>,
1844 pub artifact_dir: PathBuf,
1845}
1846
1847#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1850#[serde(deny_unknown_fields)]
1851pub struct BenchTokenCountSummary {
1852 pub minimum: u32,
1853 pub maximum: u32,
1854 pub mean: f64,
1855}
1856
1857#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1860#[serde(rename_all = "snake_case")]
1861pub enum BenchPromptTemplateSource {
1862 RequestBody,
1863 PromptTable,
1864 TokenizerDefault,
1865}
1866
1867#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1869#[serde(deny_unknown_fields)]
1870pub struct BenchPromptTemplateProjection {
1871 pub source: BenchPromptTemplateSource,
1872 pub content: String,
1873 pub sha256: String,
1874}
1875
1876#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1878#[serde(deny_unknown_fields)]
1879pub struct BenchPromptTokenTargetingSummary {
1880 pub selected_prompt_tokens: BenchTokenCountSummary,
1881 pub pre_template_content_tokens: BenchTokenCountSummary,
1882 #[serde(default)]
1883 pub projection_template: Option<BenchPromptTemplateProjection>,
1884 pub exact_entries: u32,
1885 pub fallback_entries: u32,
1886 #[serde(default)]
1887 pub fallback_reasons: BTreeMap<String, u32>,
1888}
1889
1890#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1892#[serde(deny_unknown_fields)]
1893pub struct BenchPrefixGeometrySummary {
1894 pub shared_prefix_tokens: BenchTokenCountSummary,
1895 pub unique_suffix_tokens: BenchTokenCountSummary,
1896 pub maximum_shared_prefix_tokens: u32,
1897 pub canonical_prefix_sha256: String,
1898 pub full_prompt_entries: u32,
1899}
1900
1901#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1903#[serde(deny_unknown_fields)]
1904pub struct BenchPrefixConditioningInput {
1905 pub path: PathBuf,
1906 pub sha256: String,
1907 pub prompt_tokens: u32,
1908}
1909
1910#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1913#[serde(deny_unknown_fields)]
1914pub struct BenchSharedSystemContentSummary {
1915 pub system_content_tokens: BenchTokenCountSummary,
1916 pub user_content_tokens: BenchTokenCountSummary,
1917 pub canonical_system_content_sha256: String,
1918}
1919
1920#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1922#[serde(deny_unknown_fields)]
1923pub struct BenchPopulationPreparationResult {
1924 pub schema_version: u32,
1925 pub status: ClientStatus,
1926 pub materialization_identity: String,
1927 pub requested_entries: u32,
1928 pub candidate_entries: u64,
1929 pub admitted_entries: u64,
1930 pub ineligible_entries: u64,
1931 #[serde(default)]
1932 pub ineligible_reasons: BTreeMap<String, u64>,
1933 pub population: Option<BenchPopulationInput>,
1934 pub input_tokens: Option<BenchTokenCountSummary>,
1935 pub output_tokens: Option<BenchTokenCountSummary>,
1936 #[serde(default)]
1937 pub prompt_token_targeting: Option<BenchPromptTokenTargetingSummary>,
1938 #[serde(default)]
1939 pub prefix_geometry: Option<BenchPrefixGeometrySummary>,
1940 #[serde(default)]
1941 pub prefix_conditioning: Option<BenchPrefixConditioningInput>,
1942 #[serde(default)]
1943 pub shared_system_content: Option<BenchSharedSystemContentSummary>,
1944 pub evidence_path: Option<PathBuf>,
1945 pub error: Option<String>,
1946}
1947
1948#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1950#[serde(deny_unknown_fields)]
1951pub struct BenchClientResult {
1952 pub schema_version: u32,
1955 pub status: ClientStatus,
1956 pub completed_requests: u64,
1957 pub failed_requests: u64,
1958 pub normalization_schema: String,
1959 pub metrics: BTreeMap<String, f64>,
1960 #[serde(default)]
1961 pub request_slo: Option<BenchRequestSloResult>,
1962 #[serde(default)]
1963 pub session_evidence: Option<BenchSessionResultEvidence>,
1964 #[serde(default)]
1965 pub agentic_evidence: Option<Box<BenchAgenticResultEvidence>>,
1966 #[serde(default)]
1967 pub prompt_token_reconciliation: Vec<BenchPromptTokenReconciliation>,
1968 #[serde(default)]
1969 pub prompt_cache_observations: Vec<BenchPromptCacheObservation>,
1970 pub native_command: Vec<String>,
1971 pub native_exit_code: Option<i32>,
1972 #[serde(default)]
1973 pub report_invocations: Vec<BenchNativeInvocation>,
1974 pub raw_artifacts: Vec<RawArtifact>,
1975 pub error: Option<String>,
1976}
1977
1978#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1980#[serde(deny_unknown_fields)]
1981pub struct BenchPromptCacheObservation {
1982 pub request_id: u64,
1983 pub prompt_tokens: u64,
1984 pub cache_read_tokens: u64,
1985 pub uncached_prompt_tokens: u64,
1986 pub cache_read_ratio: f64,
1987}
1988
1989#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1990#[serde(deny_unknown_fields)]
1991pub struct BenchAgenticSourceVerification {
1992 pub repository: String,
1993 pub expected_revision: String,
1994 #[serde(default)]
1995 pub observed_revision: Option<String>,
1996 pub filename: String,
1997 pub expected_sha256: String,
1998 #[serde(default)]
1999 pub observed_sha256: Option<String>,
2000 #[serde(default)]
2001 pub cache_path: Option<PathBuf>,
2002 #[serde(default)]
2003 pub cache_state_before: Option<BenchDatasetCacheState>,
2004 #[serde(default)]
2005 pub acquisition_outcome: Option<BenchAgenticAcquisitionOutcome>,
2006}
2007
2008#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
2009#[serde(rename_all = "snake_case")]
2010pub enum BenchAgenticAcquisitionOutcome {
2011 Reused,
2012 Downloaded,
2013}
2014
2015#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
2016#[serde(deny_unknown_fields)]
2017pub struct BenchAgenticBranchStats {
2018 pub children_spawned: u64,
2019 pub children_completed: u64,
2020 pub children_errored: u64,
2021 pub children_truncated: u64,
2022 pub children_delayed: u64,
2023 pub parents_suspended: u64,
2024 pub parents_resumed: u64,
2025 pub parents_failed_due_to_child_error: u64,
2026 pub joins_suppressed: u64,
2027}
2028
2029#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
2030#[serde(deny_unknown_fields)]
2031pub struct BenchAgenticResultEvidence {
2032 pub source: BenchAgenticSourceVerification,
2033 #[serde(default)]
2034 pub run: Option<Box<BenchAgenticRunEvidence>>,
2035}
2036
2037#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
2038#[serde(deny_unknown_fields)]
2039pub struct BenchAgenticRunEvidence {
2040 pub native_run_id: String,
2041 pub scenario: String,
2042 pub submission_valid: bool,
2043 #[serde(default)]
2044 pub submission_invalid_reasons: Vec<String>,
2045 pub warmup_records: u64,
2046 pub warmup_error_records: u64,
2047 pub warmup_succeeded: bool,
2051 pub profiling_records: u64,
2052 pub distinct_runtime_conversations: u64,
2053 pub distinct_transport_requests: u64,
2054 pub context_overflow_count: u64,
2055 pub ordinary_failure_count: u64,
2056 pub branch_stats: BenchAgenticBranchStats,
2057 pub aggregate_artifact: PathBuf,
2058 #[serde(default)]
2061 pub raw_records_artifact: Option<PathBuf>,
2062 #[serde(default)]
2063 pub unavailable_dimensions: Vec<String>,
2064 #[serde(default)]
2067 pub warmup_source_coordinate_records: Option<u64>,
2068 #[serde(default)]
2071 pub source_coordinate_records: Option<u64>,
2072 #[serde(default)]
2075 pub distinct_source_traces: Option<u64>,
2076 #[serde(default)]
2079 pub cache_bust_records: Option<u64>,
2080}
2081
2082#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2084#[serde(deny_unknown_fields)]
2085pub struct BenchPromptTokenReconciliation {
2086 pub population_index: u32,
2087 pub native_session_num: u64,
2088 pub planned_prompt_tokens: u32,
2089 #[serde(default)]
2090 pub observed_prompt_tokens: Option<u32>,
2091 pub reconciled: bool,
2092}
2093
2094#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2096#[serde(deny_unknown_fields)]
2097pub struct BenchSessionPhaseSummary {
2098 pub planned_sessions: u32,
2099 pub started_sessions: u32,
2100 pub succeeded_sessions: u32,
2101 pub failed_sessions: u32,
2102 pub planned_requests: u32,
2103 pub attempted_requests: u32,
2104 pub completed_requests: u32,
2105 pub failed_requests: u32,
2106 pub reconciled: bool,
2107}
2108
2109#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2111#[serde(deny_unknown_fields)]
2112pub struct BenchRuntimeSessionResult {
2113 pub phase: String,
2114 pub runtime_session_id: String,
2115 pub template_identity: String,
2116 pub planned_turns: u32,
2117 pub attempted_turns: u32,
2118 pub status: ClientStatus,
2119 #[serde(default)]
2120 pub failure_classification: Option<String>,
2121 #[serde(default)]
2122 pub diagnostic: Option<String>,
2123 #[serde(default)]
2124 pub failing_turn: Option<u32>,
2125 pub suppressed_later_turns: u32,
2126}
2127
2128#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2130#[serde(deny_unknown_fields)]
2131pub struct BenchSessionTurnResult {
2132 pub phase: String,
2133 pub runtime_session_id: String,
2134 pub turn_index: u32,
2135 #[serde(default)]
2138 pub pre_template_content_tokens: Option<u32>,
2139 #[serde(default)]
2140 pub observed_prompt_tokens: Option<u32>,
2141 pub native_session_num: u64,
2142 #[serde(default)]
2143 pub preceding_native_session_num: Option<u64>,
2144 #[serde(default)]
2145 pub preceding_terminal_response_receipt_ns: Option<u64>,
2146 #[serde(default)]
2147 pub effective_inter_turn_delay_seconds: Option<f64>,
2148 pub request_start_ns: u64,
2149 #[serde(default)]
2150 pub inter_turn_delay_reconciled: Option<bool>,
2151 #[serde(default)]
2152 pub post_failure_continuation: bool,
2153 pub native_artifact_name: String,
2154}
2155
2156#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2158#[serde(deny_unknown_fields)]
2159pub struct BenchSessionResultEvidence {
2160 pub warmup: BenchSessionPhaseSummary,
2161 pub profiling: BenchSessionPhaseSummary,
2162 pub sessions: Vec<BenchRuntimeSessionResult>,
2163 pub turns: Vec<BenchSessionTurnResult>,
2164 pub population_slice_reconciled: bool,
2165 pub sessions_reconciled: bool,
2166 pub turn_order_reconciled: bool,
2167 pub inter_turn_delays_reconciled: bool,
2168 #[serde(default)]
2171 pub native_requests_reconciled: Option<bool>,
2172 pub counts_reconciled: bool,
2173 #[serde(default)]
2176 pub unavailable_dimensions: Vec<String>,
2177}
2178
2179#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2181#[serde(deny_unknown_fields)]
2182pub struct BenchNativeInvocation {
2183 pub purpose: String,
2184 pub command: Vec<String>,
2185 pub exit_code: Option<i32>,
2186 pub interrupted: bool,
2187 pub timed_out: bool,
2188}
2189
2190#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2192#[serde(deny_unknown_fields)]
2193pub struct BenchRequestSloResult {
2194 pub good_requests: u64,
2195 pub good_request_ratio: f64,
2196 pub goodput: f64,
2197 pub profiling_duration_seconds: f64,
2198 pub profiling_duration_source: String,
2199 pub request_count_reconciled: bool,
2200 #[serde(default)]
2201 pub native_aggregate_good_request_count: Option<u64>,
2202 #[serde(default)]
2203 pub native_aggregate_good_request_count_consistent: Option<bool>,
2204}
2205
2206#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2210#[serde(deny_unknown_fields)]
2211pub struct AdapterProtocol {
2212 pub request: AdapterRequest,
2213 pub response: AdapterResponse,
2214}
2215
2216#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
2220#[serde(deny_unknown_fields)]
2221pub struct MeasurementProtocol {
2222 #[serde(default, skip_serializing_if = "Option::is_none")]
2223 pub eval_client_request: Option<EvalClientRequest>,
2224 #[serde(default, skip_serializing_if = "Option::is_none")]
2225 pub eval_client_result: Option<EvalClientResult>,
2226 #[serde(default, skip_serializing_if = "Option::is_none")]
2227 pub bench_client_request: Option<BenchClientRequest>,
2228 #[serde(default, skip_serializing_if = "Option::is_none")]
2229 pub bench_client_result: Option<BenchClientResult>,
2230 #[serde(default, skip_serializing_if = "Option::is_none")]
2231 pub bench_population_preparation_request: Option<BenchPopulationPreparationRequest>,
2232 #[serde(default, skip_serializing_if = "Option::is_none")]
2233 pub bench_population_preparation_result: Option<BenchPopulationPreparationResult>,
2234 #[serde(default, skip_serializing_if = "Option::is_none")]
2235 pub data_asset_preparation_request: Option<MeasurementDataAssetPreparationRequest>,
2236 #[serde(default, skip_serializing_if = "Option::is_none")]
2237 pub data_asset_preparation_result: Option<MeasurementDataAssetPreparationResult>,
2238}