use crate::measurement_data_asset::{
EvalPreparedSourceBinding, MeasurementDataAssetPreparationRequest,
MeasurementDataAssetPreparationResult,
};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::path::PathBuf;
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
pub enum ProtocolVersion {
#[serde(rename = "11")]
V11,
}
impl ProtocolVersion {
pub const CURRENT: Self = Self::V11;
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::V11 => "11",
}
}
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(untagged)]
pub enum SettingValue {
Bool(bool),
Integer(i64),
Float(f64),
String(String),
Array(Vec<SettingValue>),
Object(BTreeMap<String, SettingValue>),
}
#[derive(
Clone, Copy, Debug, Default, Deserialize, Eq, JsonSchema, Ord, PartialEq, PartialOrd, Serialize,
)]
#[serde(rename_all = "snake_case")]
pub enum CaptureMechanism {
#[default]
ManagedCollection,
EngineTrace,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub enum SyntheticAcceptanceInput {
Explicit {
acceptance_length: f64,
},
Curve(SyntheticAcceptanceCurveInput),
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SyntheticAcceptanceCurveInput {
pub model_key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thinking_mode: Option<String>,
pub text: String,
pub sha256: String,
}
#[derive(Clone, Copy, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SyntheticAcceptanceOutcome {
pub acceptance_length: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub draft_count: Option<u32>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EndpointAssignment {
pub host: String,
pub port: u16,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum EndpointProtocol {
Http,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum HttpMethod {
Post,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
pub enum AdapterRequest {
PlanServe {
protocol_version: ProtocolVersion,
input: PlanServeInput,
},
RenderServe {
protocol_version: ProtocolVersion,
input: RenderServeInput,
},
}
impl AdapterRequest {
#[must_use]
pub const fn protocol_version(&self) -> ProtocolVersion {
match self {
Self::PlanServe {
protocol_version, ..
}
| Self::RenderServe {
protocol_version, ..
} => *protocol_version,
}
}
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct PlanServeInput {
pub model: ServeModelInput,
pub topology: ServeTopology,
pub state_dir: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gateway_backend: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pd_router_backend: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub kv_transfer: Option<KvTransferMechanism>,
pub roles: Vec<ServeRoleInput>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profiling: Option<CaptureMechanism>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub synthetic_acceptance: Option<SyntheticAcceptanceInput>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub auxiliary_models: Vec<AuxiliaryModelInput>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RenderServeInput {
pub model: ServeModelInput,
pub topology: ServeTopology,
pub state_dir: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gateway_backend: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pd_router_backend: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub kv_transfer: Option<KvTransferMechanism>,
pub allocations: Vec<ServeProcessAllocation>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profiling: Option<CaptureMechanism>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub synthetic_acceptance: Option<SyntheticAcceptanceInput>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub auxiliary_models: Vec<AuxiliaryModelInput>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
pub enum AdapterResponse {
Ok {
protocol_version: ProtocolVersion,
result: Box<AdapterResult>,
},
Error {
protocol_version: ProtocolVersion,
error: AdapterError,
},
}
impl AdapterResponse {
#[must_use]
pub const fn protocol_version(&self) -> ProtocolVersion {
match self {
Self::Ok {
protocol_version, ..
}
| Self::Error {
protocol_version, ..
} => *protocol_version,
}
}
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
pub enum AdapterResult {
PlanServe { output: Box<PlanServeResult> },
RenderServe { output: Box<RenderServeResult> },
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AdapterError {
pub code: AdapterErrorCode,
pub message: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum AdapterErrorCode {
InvalidRequest,
UnsupportedProtocolVersion,
InvalidSettings,
Internal,
UnsupportedOperation,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct IntegrationIdentity {
pub adapter_id: String,
pub adapter_version: String,
pub framework: String,
pub framework_version: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ServeTopology {
Single,
PrefillDecode,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ServeRoleKind {
Serve,
Prefill,
Decode,
}
impl ServeRoleKind {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Serve => "serve",
Self::Prefill => "prefill",
Self::Decode => "decode",
}
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum KvTransferMechanism {
Mooncake,
Nixl,
}
#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct Parallelism {
#[serde(skip_serializing_if = "Option::is_none")]
pub outer: Option<ParallelismOuter>,
#[serde(skip_serializing_if = "Option::is_none")]
pub attention: Option<ParallelismAttention>,
#[serde(skip_serializing_if = "Option::is_none")]
pub experts: Option<ParallelismExperts>,
}
impl Parallelism {
pub fn field_values(&self) -> [(&'static str, Option<u32>); 9] {
[
(
"outer.tensor_parallel_size",
self.outer
.as_ref()
.and_then(|value| value.tensor_parallel_size),
),
(
"outer.pipeline_parallel_size",
self.outer
.as_ref()
.and_then(|value| value.pipeline_parallel_size),
),
(
"attention.tensor_parallel_size",
self.attention
.as_ref()
.and_then(|value| value.tensor_parallel_size),
),
(
"attention.data_parallel_size",
self.attention
.as_ref()
.and_then(|value| value.data_parallel_size),
),
(
"attention.context_parallel_size",
self.attention
.as_ref()
.and_then(|value| value.context_parallel_size),
),
(
"experts.tensor_parallel_size",
self.experts
.as_ref()
.and_then(|value| value.tensor_parallel_size),
),
(
"experts.data_parallel_size",
self.experts
.as_ref()
.and_then(|value| value.data_parallel_size),
),
(
"experts.expert_parallel_size",
self.experts
.as_ref()
.and_then(|value| value.expert_parallel_size),
),
(
"experts.dense_tensor_parallel_size",
self.experts
.as_ref()
.and_then(|value| value.dense_tensor_parallel_size),
),
]
}
pub fn merge_from(&mut self, other: &Self) {
if let Some(other) = &other.outer {
self.outer.get_or_insert_default().merge_from(other);
}
if let Some(other) = &other.attention {
self.attention.get_or_insert_default().merge_from(other);
}
if let Some(other) = &other.experts {
self.experts.get_or_insert_default().merge_from(other);
}
}
}
#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct ParallelismOuter {
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub tensor_parallel_size: Option<u32>,
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub pipeline_parallel_size: Option<u32>,
}
impl ParallelismOuter {
fn merge_from(&mut self, other: &Self) {
if other.tensor_parallel_size.is_some() {
self.tensor_parallel_size = other.tensor_parallel_size;
}
if other.pipeline_parallel_size.is_some() {
self.pipeline_parallel_size = other.pipeline_parallel_size;
}
}
}
#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct ParallelismAttention {
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub tensor_parallel_size: Option<u32>,
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub data_parallel_size: Option<u32>,
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub context_parallel_size: Option<u32>,
}
impl ParallelismAttention {
fn merge_from(&mut self, other: &Self) {
if other.tensor_parallel_size.is_some() {
self.tensor_parallel_size = other.tensor_parallel_size;
}
if other.data_parallel_size.is_some() {
self.data_parallel_size = other.data_parallel_size;
}
if other.context_parallel_size.is_some() {
self.context_parallel_size = other.context_parallel_size;
}
}
}
#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct ParallelismExperts {
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub tensor_parallel_size: Option<u32>,
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub data_parallel_size: Option<u32>,
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub expert_parallel_size: Option<u32>,
#[schemars(range(min = 1))]
#[serde(skip_serializing_if = "Option::is_none")]
pub dense_tensor_parallel_size: Option<u32>,
}
impl ParallelismExperts {
fn merge_from(&mut self, other: &Self) {
if other.tensor_parallel_size.is_some() {
self.tensor_parallel_size = other.tensor_parallel_size;
}
if other.data_parallel_size.is_some() {
self.data_parallel_size = other.data_parallel_size;
}
if other.expert_parallel_size.is_some() {
self.expert_parallel_size = other.expert_parallel_size;
}
if other.dense_tensor_parallel_size.is_some() {
self.dense_tensor_parallel_size = other.dense_tensor_parallel_size;
}
}
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ServeModelInput {
pub id: String,
pub served_name: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum AuxiliaryModelKind {
DraftModel,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AuxiliaryModelInput {
pub kind: AuxiliaryModelKind,
pub model: ServeModelInput,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AuxiliaryModelLocator {
pub kind: AuxiliaryModelKind,
pub locator: String,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ServeRoleInput {
pub id: String,
pub kind: ServeRoleKind,
pub replica_count: u32,
pub parallelism: Parallelism,
pub settings: BTreeMap<String, SettingValue>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ServeRoleResult {
pub id: String,
pub kind: ServeRoleKind,
pub declared_replica_count: u32,
pub effective_replica_count: u32,
pub effective_settings: BTreeMap<String, SettingValue>,
pub effective_parallelism: Parallelism,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub public_endpoint: Option<EndpointDeclaration>,
#[serde(default)]
pub render_inputs: Vec<RenderInputDeclaration>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct PlanServeResult {
pub integration: IntegrationIdentity,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub synthetic_acceptance: Option<SyntheticAcceptanceOutcome>,
pub roles: Vec<ServeRoleResult>,
pub replicas: Vec<ServeReplicaRequirement>,
pub links: Vec<ServeRoleLink>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gateway: Option<GatewayPlan>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pd_router: Option<PdRouterPlan>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ServeReplicaRequirement {
pub id: String,
pub role_id: String,
pub replica_index: u32,
pub device_count: u32,
pub ports: Vec<String>,
pub primary_ports: Vec<String>,
pub primary_readiness: ReadinessProbe,
pub worker_readiness: ReadinessProbe,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capture_target: Option<CaptureTargetRequirement>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum ServeRoleLink {
RequestRouting {
source: String,
targets: Vec<String>,
},
KvTransfer {
source: String,
target: String,
mechanism: KvTransferMechanism,
},
Bootstrap {
source: String,
target: String,
port: String,
},
SideChannel {
source: String,
target: String,
port: String,
},
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RenderInputDeclaration {
pub source_path: String,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct SuppliedRenderInput {
pub source_path: String,
pub text: String,
pub sha256: String,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EndpointDeclaration {
pub protocol: EndpointProtocol,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub server_metrics: Option<ServerMetricsEndpointRequirement>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prefix_cache_reset: Option<HttpActionSpec>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prefix_cache_conditioning: Option<HttpActionSpec>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prompt_cache_read_zero_representation: Option<PromptCacheReadZeroRepresentation>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EndpointRequirement {
pub protocol: EndpointProtocol,
pub completions_path: String,
pub chat_completions_path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub server_metrics: Option<ServerMetricsEndpointRequirement>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prefix_cache_reset: Option<HttpActionSpec>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prefix_cache_conditioning: Option<HttpActionSpec>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prompt_cache_read_zero_representation: Option<PromptCacheReadZeroRepresentation>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ServerMetricsEndpointRequirement {
pub path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub port: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum ReadinessProbe {
Http { path: String },
HttpTargetRegistry(Box<HttpTargetRegistryReadiness>),
ProcessAlive,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HttpTargetRegistryReadiness {
pub target_scheme: TargetEndpointScheme,
pub readiness_path: String,
pub registry_path: String,
pub targets_field: String,
pub target_url_field: String,
pub target_role_field: String,
pub target_healthy_field: String,
pub target_bootstrap_port_field: String,
pub prefill_role_value: String,
pub decode_role_value: String,
pub prefill_bootstrap_port: String,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct HttpActionSpec {
pub method: HttpMethod,
pub path: String,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CaptureTargetRequirement {
pub mechanism: CaptureMechanism,
pub window_control: CaptureWindowControlRequirement,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CaptureWindowControlRequirement {
pub endpoint: CaptureWindowControlEndpoint,
pub start: CaptureWindowHttpActionSpec,
pub stop: CaptureWindowHttpActionSpec,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum CaptureWindowControlEndpoint {
ReplicaEntry,
Gateway,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CaptureWindowHttpActionSpec {
pub method: HttpMethod,
pub path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub body: Option<BTreeMap<String, SettingValue>>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum ServeProcessAllocation {
ModelRank {
process: String,
role: String,
role_kind: ServeRoleKind,
replica: u32,
rank: u32,
rank_count: u32,
machine: String,
devices: Vec<u32>,
model_locator: String,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
auxiliary_model_locators: Vec<AuxiliaryModelLocator>,
#[serde(default, skip_serializing_if = "Option::is_none")]
endpoint: Option<EndpointAssignment>,
ports: BTreeMap<String, EndpointAssignment>,
cache: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
capture_storage: Option<String>,
launch: AllocationLaunch,
effective_settings: BTreeMap<String, SettingValue>,
effective_parallelism: Parallelism,
#[serde(default)]
links: Vec<ServeRoleLink>,
#[serde(default)]
dependencies: Vec<String>,
#[serde(default)]
render_inputs: Vec<SuppliedRenderInput>,
},
Frontend {
process: String,
process_role: FrontendProcessRole,
components: FrontendComponents,
machine: String,
devices: Vec<u32>,
endpoint: EndpointAssignment,
ports: BTreeMap<String, EndpointAssignment>,
cache: String,
launch: AllocationLaunch,
gateway: Box<GatewayPlan>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pd_router: Option<Box<PdRouterPlan>>,
#[serde(default)]
links: Vec<ServeRoleLink>,
#[serde(default)]
dependencies: Vec<String>,
#[serde(default)]
render_inputs: Vec<SuppliedRenderInput>,
},
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum AllocationLaunch {
Local,
Ssh { target: String },
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RenderServeResult {
pub integration: IntegrationIdentity,
pub processes: Vec<RenderedServeProcess>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum RenderedServeProcess {
ModelRank {
process: String,
role: String,
replica: u32,
rank: u32,
rank_count: u32,
launch_files: Vec<LaunchFileDeclaration>,
command: ProcessSpec,
},
Frontend {
process: String,
process_role: FrontendProcessRole,
components: FrontendComponents,
launch_files: Vec<LaunchFileDeclaration>,
command: ProcessSpec,
},
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct LaunchFileDeclaration {
pub relative_path: String,
pub text: String,
pub sha256: String,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ProcessSpec {
pub argv: Vec<String>,
pub env: BTreeMap<String, String>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum RenderSource {
ControlPlane,
Integration,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum FrontendProcessRole {
Gateway,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct FrontendCoRendering {
pub process_role: FrontendProcessRole,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum FrontendGatewayComponent {
Gateway,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum FrontendPdRouterComponent {
PdRouter,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(transparent)]
pub struct GatewayFrontendBinding(pub [FrontendGatewayComponent; 1]);
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(transparent)]
pub struct GatewayPdRouterFrontendBinding(
pub (FrontendGatewayComponent, FrontendPdRouterComponent),
);
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(untagged)]
pub enum FrontendComponents {
Gateway(GatewayFrontendBinding),
GatewayPdRouter(GatewayPdRouterFrontendBinding),
}
impl FrontendComponents {
#[must_use]
pub const fn gateway() -> Self {
Self::Gateway(GatewayFrontendBinding([FrontendGatewayComponent::Gateway]))
}
#[must_use]
pub const fn gateway_pd_router() -> Self {
Self::GatewayPdRouter(GatewayPdRouterFrontendBinding((
FrontendGatewayComponent::Gateway,
FrontendPdRouterComponent::PdRouter,
)))
}
#[must_use]
pub const fn includes_pd_router(&self) -> bool {
matches!(self, Self::GatewayPdRouter(_))
}
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum GatewayTarget {
Engine { role: String },
PdRouter,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct GatewayPlan {
pub backend: String,
pub implementation: String,
pub implementation_version: String,
pub effective_settings: BTreeMap<String, SettingValue>,
pub endpoint: EndpointDeclaration,
pub readiness: ReadinessProbe,
#[serde(default)]
pub ports: Vec<String>,
pub targets: Vec<GatewayTarget>,
#[serde(default)]
pub render_inputs: Vec<RenderInputDeclaration>,
pub render_source: RenderSource,
pub co_rendering: FrontendCoRendering,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct PdRoutingPolicies {
pub prefill: String,
pub decode: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum FrontendHandoff {
InProcess,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct PdRouterPlan {
pub backend: String,
pub implementation: String,
pub implementation_version: String,
pub effective_settings: BTreeMap<String, SettingValue>,
pub policies: PdRoutingPolicies,
pub prefill_role: String,
pub decode_role: String,
pub target_scheme: TargetEndpointScheme,
#[serde(default)]
pub ports: Vec<String>,
pub readiness: ReadinessProbe,
pub handoff: FrontendHandoff,
#[serde(default)]
pub render_inputs: Vec<RenderInputDeclaration>,
pub render_source: RenderSource,
pub co_rendering: FrontendCoRendering,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TargetEndpointScheme {
Http,
Grpc,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct MeasurementModelInput {
pub locator: String,
pub served_name: String,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ClientEndpointInput {
pub protocol: EndpointProtocol,
pub host: String,
pub port: u16,
pub completions_path: String,
pub chat_completions_path: String,
#[serde(default)]
pub server_metrics: Option<ServerMetricsEndpointInput>,
#[serde(default)]
pub prompt_cache_read_zero_representation: Option<PromptCacheReadZeroRepresentation>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ServerMetricsEndpointInput {
pub path: String,
#[serde(default)]
pub port_name: Option<String>,
pub url: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum PromptCacheReadZeroRepresentation {
Explicit,
Omitted,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ClientStatus {
Succeeded,
Failed,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RawArtifact {
pub name: String,
pub kind: String,
pub path: PathBuf,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EvalClientRequest {
pub protocol_version: ProtocolVersion,
pub workspace_root: PathBuf,
pub workspace_source_exclusions: Vec<PathBuf>,
pub endpoint: ClientEndpointInput,
pub model: MeasurementModelInput,
pub definition: EvalDefinitionInput,
#[serde(default)]
pub prepared_source: Option<EvalPreparedSourceBinding>,
pub case_budget_seconds: f64,
pub artifact_dir: PathBuf,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum EvalDefinitionInput {
#[serde(rename = "openai_smoke")]
OpenAiSmoke {
prompt: String,
max_tokens: u32,
timeout_seconds: u64,
#[serde(default)]
vision: bool,
},
LmEval {
task: Box<EvalTaskSourceInput>,
prompt: EvalPromptInput,
#[serde(default)]
declared_prompt: Option<EvalPromptInput>,
#[serde(default)]
request_body: BTreeMap<String, SettingValue>,
limit: Option<u32>,
few_shot: Option<u32>,
seed: Option<u64>,
base_seed: u64,
trials: u32,
max_tokens: Option<u32>,
concurrency: Option<u32>,
metric: String,
#[serde(default)]
metric_filter: Option<String>,
threshold: f64,
timeout_seconds: u64,
},
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum EvalPromptInput {
Flat,
ServerChat,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum EvalTaskSourceInput {
BuiltIn { name: String },
Bundled {
name: String,
task_identity: String,
path: PathBuf,
task_closure_sha256: String,
task_definition_sha256: String,
prompt_asset_sha256: String,
dataset_asset_sha256: String,
scorer_sha256: String,
},
WorkspaceYaml { path: PathBuf },
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EvalClientResult {
pub schema_version: u32,
pub status: ClientStatus,
pub metrics: BTreeMap<String, f64>,
#[serde(default)]
pub normalized_metrics: BTreeMap<String, EvalNormalizedMetric>,
#[serde(default)]
pub gate: Option<EvalMetricGate>,
#[serde(default)]
pub trial_summary: Option<EvalTrialSummary>,
pub native_command: Vec<String>,
#[serde(default)]
pub native_exit_code: Option<i32>,
#[serde(default)]
pub native_timed_out: bool,
pub raw_artifacts: Vec<RawArtifact>,
pub failure_kind: Option<EvalFailureKind>,
pub error: Option<String>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum EvalFailureKind {
TaskResolution,
ProbeTokenizer,
ProbeTransport,
ProbeHttp,
ProbeMalformedResponse,
ProbeGeneratedOnlyLogprobs,
ProbeTokenizerAlignment,
MetricNormalization,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum EvalMetricComparison {
AtLeast,
AtMost,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum EvalMetricGateConclusion {
Passed,
Failed,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EvalNormalizedMetric {
pub source_identity: String,
pub metric: String,
pub filter: Option<String>,
pub native_metric_key: String,
pub value: f64,
pub higher_is_better: bool,
pub prompt_authority: EvalPromptInput,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EvalMetricGate {
pub metric: EvalNormalizedMetric,
pub threshold: f64,
pub comparison: EvalMetricComparison,
pub conclusion: EvalMetricGateConclusion,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct EvalTrialSummary {
pub requested_trials: u32,
pub issued_trials: u32,
pub unissued_trials: u32,
pub completed_trials: u32,
pub request_failure_trials: u32,
pub passed_trials: u32,
pub pass_rate: Option<f64>,
pub per_trial_metric: String,
pub per_trial_filter: Option<String>,
pub higher_is_better: bool,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchClientRequest {
pub protocol_version: ProtocolVersion,
pub endpoint: ClientEndpointInput,
pub model: MeasurementModelInput,
pub definition: BenchDefinitionInput,
#[serde(default)]
pub population: Option<BenchPopulationInput>,
pub case: BenchCaseInput,
pub case_budget_seconds: f64,
pub artifact_dir: PathBuf,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchDefinitionInput {
#[serde(default)]
pub request_source: Option<BenchRequestSourceInput>,
#[serde(default)]
pub session_source: Option<BenchSessionSourceInput>,
#[serde(default)]
pub agentic_source: Option<BenchAgenticSourceInput>,
pub prompt: BenchPromptInput,
#[serde(default)]
pub server_metrics: bool,
#[serde(default)]
pub artifact_level: BenchArtifactLevelInput,
pub seed: u64,
#[serde(default)]
pub request_body: BTreeMap<String, SettingValue>,
#[serde(default)]
pub request_slo: Option<BenchRequestSloInput>,
pub timeout_seconds: u64,
pub cache_start: BenchCacheStartInput,
}
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchArtifactLevelInput {
Performance,
#[default]
Diagnostic,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchCacheStartInput {
Uncontrolled,
Cold,
Primed,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum BenchPromptInput {
Flat {
request_representation: BenchRequestRepresentationInput,
route: BenchPromptRouteInput,
rendering_authority: BenchRenderingAuthorityInput,
},
RenderedChat {
#[serde(default)]
chat_template: Option<String>,
#[serde(default)]
chat_template_kwargs: BTreeMap<String, SettingValue>,
request_representation: BenchRequestRepresentationInput,
route: BenchPromptRouteInput,
rendering_authority: BenchRenderingAuthorityInput,
},
ServerChat {
request_representation: BenchRequestRepresentationInput,
route: BenchPromptRouteInput,
rendering_authority: BenchRenderingAuthorityInput,
},
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchRequestRepresentationInput {
FlatPrompt,
StructuredMessages,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchPromptRouteInput {
Completions,
ChatCompletions,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchRenderingAuthorityInput {
LocalFlat,
LocalTemplate,
Server,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum BenchRequestSourceInput {
Random {
input_tokens: BenchTokenSelectorInput,
output_tokens: BenchTokenSelectorInput,
#[serde(default)]
prefix_sharing: Option<BenchPrefixSharingInput>,
#[serde(default)]
shared_system_content: Option<BenchSharedSystemContentInput>,
#[serde(default)]
corpus: Option<BenchCorpusInput>,
#[serde(default)]
images: Option<BenchImagesInput>,
},
RandomMixture {
shapes: Vec<BenchRandomShapeInput>,
total_weight: u64,
#[serde(default)]
prefix_sharing: Option<BenchPrefixSharingInput>,
},
Dataset {
dataset: String,
#[serde(default)]
profile: Option<String>,
max_input_tokens: u32,
output_tokens: Option<u32>,
catalog: Box<BenchDatasetCatalogInput>,
},
Replay {
path: String,
#[serde(default)]
expected_sha256: Option<String>,
#[serde(default)]
prefix_sharing: Option<BenchPrefixSharingInput>,
},
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(untagged)]
pub enum BenchTokenSelectorInput {
Fixed(u32),
InclusiveUniform(BenchInclusiveUniformInput),
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchInclusiveUniformInput {
pub kind: BenchTokenDistributionKindInput,
pub min: u32,
pub max: u32,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchTokenDistributionKindInput {
InclusiveUniform,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(untagged)]
pub enum BenchPrefixSharingInput {
Tokens { shared_prefix_tokens: u32 },
Ratio { shared_prefix_ratio: f64 },
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(untagged)]
pub enum BenchSharedSystemContentInput {
Tokens { tokens: u32 },
Ratio { ratio: f64 },
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchCorpusInput {
pub path: String,
#[serde(default)]
pub expected_sha256: Option<String>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchImagesInput {
pub width: u32,
pub height: u32,
pub count: u32,
#[serde(default)]
pub source: Option<BenchImageSourceInput>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchImageSourceInput {
pub path: String,
pub resolved_path: PathBuf,
#[serde(default)]
pub expected_sha256: Option<String>,
pub sampling: BenchImageSamplingInput,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum BenchImageSamplingInput {
RandomWithReplacement,
ShuffleCycle,
SequentialCycle,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchRandomShapeInput {
pub input_tokens: u32,
pub output_tokens: u32,
pub weight: u32,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchDatasetCatalogInput {
pub dataset: String,
#[serde(default)]
pub profile: Option<String>,
pub source: String,
pub upstream_identity: String,
pub url: String,
pub sha256: String,
pub source_format: String,
pub aiperf_format: String,
#[serde(default)]
pub configuration: Option<String>,
#[serde(default)]
pub split: Option<String>,
#[serde(default)]
pub filter: Option<BenchDatasetFilterInput>,
pub license: String,
pub cache_path: PathBuf,
pub cache_state: BenchDatasetCacheState,
pub materialization_identity: String,
pub provides_output_targets: bool,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchDatasetFilterInput {
pub field: String,
pub value: String,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchDatasetCacheState {
Missing,
Present,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSessionSourceInput {
pub dataset: String,
#[serde(default)]
pub profile: Option<String>,
pub max_input_tokens: u32,
pub output_tokens: Option<u32>,
pub inter_turn_delay_scale: f64,
pub max_inter_turn_delay_seconds: Option<f64>,
pub catalog: Box<BenchSessionDatasetCatalogInput>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSessionDatasetCatalogInput {
pub dataset: String,
#[serde(default)]
pub profile: Option<String>,
pub source: String,
pub upstream_identity: String,
pub url: String,
pub sha256: String,
pub source_format: String,
#[serde(default)]
pub configuration: Option<String>,
#[serde(default)]
pub split: Option<String>,
#[serde(default)]
pub filter: Option<BenchDatasetFilterInput>,
pub license: String,
pub cache_path: PathBuf,
pub cache_state: BenchDatasetCacheState,
pub materialization_identity: String,
pub provides_output_targets: bool,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchAgenticSourceInput {
pub dataset: String,
pub profile: String,
pub catalog: Box<BenchAgenticCatalogInput>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchAgenticCatalogInput {
pub repository: String,
pub revision: String,
pub filename: String,
pub sha256: String,
#[serde(default)]
pub cache_path: Option<PathBuf>,
#[serde(default)]
pub cache_state: Option<BenchDatasetCacheState>,
pub trace_count: u32,
pub approximate_bytes: u64,
pub license: String,
pub source_format: String,
pub aiperf_loader: String,
pub materialization_identity: String,
pub scenario: String,
pub concurrency_semantics: String,
pub replay_semantics: String,
pub cache_bust: String,
pub trajectory_start_min: f64,
pub trajectory_start_max: f64,
pub global_idle_gap_cap_seconds: f64,
pub trace_idle_gap_cap_seconds: f64,
pub cache_warmup_requests_per_lane: u64,
pub warmup_grace_seconds: u64,
pub dataset_configuration_timeout_seconds: u64,
pub service_profile_configuration_timeout_seconds: u64,
pub default_duration_seconds: u64,
pub minimum_duration_seconds: u64,
pub failure_threshold: f64,
pub dataset_entries: u32,
pub streaming: bool,
pub ignore_eos: bool,
pub use_server_token_count: bool,
pub gpu_telemetry: bool,
pub server_metric_slice_seconds: u64,
pub required_artifacts: Vec<String>,
pub unavailable_dimensions: Vec<String>,
pub inferencex_repository: String,
pub inferencex_revision: String,
pub inferencex_reference: String,
pub aiperf_revision: String,
pub aiperf_version: String,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchCaseInput {
pub load_shape: BenchLoadInput,
pub request_count: u32,
#[serde(default)]
pub warmup_request_count: u32,
#[serde(default)]
pub duration_seconds: Option<u64>,
#[serde(default)]
pub session_count: Option<u32>,
#[serde(default)]
pub warmup_session_count: Option<u32>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum BenchLoadInput {
ConcurrencyLimited { concurrency: u32 },
RequestRateLimited {
request_rate: f64,
burstiness: Option<f64>,
},
UnboundedRequestRate,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchRequestSloInput {
#[serde(default)]
pub request_latency_ms: Option<f64>,
#[serde(default)]
pub ttft_ms: Option<f64>,
#[serde(default)]
pub tpot_ms: Option<f64>,
pub minimum_good_request_ratio: f64,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPopulationInput {
pub path: PathBuf,
pub evidence_path: PathBuf,
pub sha256: String,
pub entries: u32,
pub tpot_applicable: bool,
#[serde(default)]
pub session_templates: Vec<BenchSessionTemplateInput>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSessionTemplateInput {
pub template_identity: String,
pub turn_count: u32,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPopulationPreparationRequest {
pub protocol_version: ProtocolVersion,
pub model: MeasurementModelInput,
pub tokenizer_backend: String,
pub transformers_version: String,
#[serde(default)]
pub request_source: Option<BenchRequestSourceInput>,
#[serde(default)]
pub session_source: Option<BenchSessionSourceInput>,
pub prompt: BenchPromptInput,
pub cache_start: BenchCacheStartInput,
#[serde(default)]
pub source_path: Option<PathBuf>,
pub required_entries: u32,
pub seed: u64,
#[serde(default)]
pub request_body: BTreeMap<String, SettingValue>,
pub artifact_dir: PathBuf,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchTokenCountSummary {
pub minimum: u32,
pub maximum: u32,
pub mean: f64,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchPromptTemplateSource {
RequestBody,
PromptTable,
TokenizerDefault,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPromptTemplateProjection {
pub source: BenchPromptTemplateSource,
pub content: String,
pub sha256: String,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPromptTokenTargetingSummary {
pub selected_prompt_tokens: BenchTokenCountSummary,
pub pre_template_content_tokens: BenchTokenCountSummary,
#[serde(default)]
pub projection_template: Option<BenchPromptTemplateProjection>,
pub exact_entries: u32,
pub fallback_entries: u32,
#[serde(default)]
pub fallback_reasons: BTreeMap<String, u32>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPrefixGeometrySummary {
pub shared_prefix_tokens: BenchTokenCountSummary,
pub unique_suffix_tokens: BenchTokenCountSummary,
pub maximum_shared_prefix_tokens: u32,
pub canonical_prefix_sha256: String,
pub full_prompt_entries: u32,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPrefixConditioningInput {
pub path: PathBuf,
pub sha256: String,
pub prompt_tokens: u32,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSharedSystemContentSummary {
pub system_content_tokens: BenchTokenCountSummary,
pub user_content_tokens: BenchTokenCountSummary,
pub canonical_system_content_sha256: String,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPopulationPreparationResult {
pub schema_version: u32,
pub status: ClientStatus,
pub materialization_identity: String,
pub requested_entries: u32,
pub candidate_entries: u64,
pub admitted_entries: u64,
pub ineligible_entries: u64,
#[serde(default)]
pub ineligible_reasons: BTreeMap<String, u64>,
pub population: Option<BenchPopulationInput>,
pub input_tokens: Option<BenchTokenCountSummary>,
pub output_tokens: Option<BenchTokenCountSummary>,
#[serde(default)]
pub prompt_token_targeting: Option<BenchPromptTokenTargetingSummary>,
#[serde(default)]
pub prefix_geometry: Option<BenchPrefixGeometrySummary>,
#[serde(default)]
pub prefix_conditioning: Option<BenchPrefixConditioningInput>,
#[serde(default)]
pub shared_system_content: Option<BenchSharedSystemContentSummary>,
pub evidence_path: Option<PathBuf>,
pub error: Option<String>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchClientResult {
pub schema_version: u32,
pub status: ClientStatus,
pub completed_requests: u64,
pub failed_requests: u64,
pub normalization_schema: String,
pub metrics: BTreeMap<String, f64>,
#[serde(default)]
pub request_slo: Option<BenchRequestSloResult>,
#[serde(default)]
pub session_evidence: Option<BenchSessionResultEvidence>,
#[serde(default)]
pub agentic_evidence: Option<Box<BenchAgenticResultEvidence>>,
#[serde(default)]
pub prompt_token_reconciliation: Vec<BenchPromptTokenReconciliation>,
#[serde(default)]
pub prompt_cache_observations: Vec<BenchPromptCacheObservation>,
pub native_command: Vec<String>,
pub native_exit_code: Option<i32>,
#[serde(default)]
pub report_invocations: Vec<BenchNativeInvocation>,
pub raw_artifacts: Vec<RawArtifact>,
pub error: Option<String>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPromptCacheObservation {
pub request_id: u64,
pub prompt_tokens: u64,
pub cache_read_tokens: u64,
pub uncached_prompt_tokens: u64,
pub cache_read_ratio: f64,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchAgenticSourceVerification {
pub repository: String,
pub expected_revision: String,
#[serde(default)]
pub observed_revision: Option<String>,
pub filename: String,
pub expected_sha256: String,
#[serde(default)]
pub observed_sha256: Option<String>,
#[serde(default)]
pub cache_path: Option<PathBuf>,
#[serde(default)]
pub cache_state_before: Option<BenchDatasetCacheState>,
#[serde(default)]
pub acquisition_outcome: Option<BenchAgenticAcquisitionOutcome>,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum BenchAgenticAcquisitionOutcome {
Reused,
Downloaded,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchAgenticBranchStats {
pub children_spawned: u64,
pub children_completed: u64,
pub children_errored: u64,
pub children_truncated: u64,
pub children_delayed: u64,
pub parents_suspended: u64,
pub parents_resumed: u64,
pub parents_failed_due_to_child_error: u64,
pub joins_suppressed: u64,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchAgenticResultEvidence {
pub source: BenchAgenticSourceVerification,
#[serde(default)]
pub run: Option<Box<BenchAgenticRunEvidence>>,
}
#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchAgenticRunEvidence {
pub native_run_id: String,
pub scenario: String,
pub submission_valid: bool,
#[serde(default)]
pub submission_invalid_reasons: Vec<String>,
pub warmup_records: u64,
pub warmup_error_records: u64,
pub warmup_succeeded: bool,
pub profiling_records: u64,
pub distinct_runtime_conversations: u64,
pub distinct_transport_requests: u64,
pub context_overflow_count: u64,
pub ordinary_failure_count: u64,
pub branch_stats: BenchAgenticBranchStats,
pub aggregate_artifact: PathBuf,
#[serde(default)]
pub raw_records_artifact: Option<PathBuf>,
#[serde(default)]
pub unavailable_dimensions: Vec<String>,
#[serde(default)]
pub warmup_source_coordinate_records: Option<u64>,
#[serde(default)]
pub source_coordinate_records: Option<u64>,
#[serde(default)]
pub distinct_source_traces: Option<u64>,
#[serde(default)]
pub cache_bust_records: Option<u64>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchPromptTokenReconciliation {
pub population_index: u32,
pub native_session_num: u64,
pub planned_prompt_tokens: u32,
#[serde(default)]
pub observed_prompt_tokens: Option<u32>,
pub reconciled: bool,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSessionPhaseSummary {
pub planned_sessions: u32,
pub started_sessions: u32,
pub succeeded_sessions: u32,
pub failed_sessions: u32,
pub planned_requests: u32,
pub attempted_requests: u32,
pub completed_requests: u32,
pub failed_requests: u32,
pub reconciled: bool,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchRuntimeSessionResult {
pub phase: String,
pub runtime_session_id: String,
pub template_identity: String,
pub planned_turns: u32,
pub attempted_turns: u32,
pub status: ClientStatus,
#[serde(default)]
pub failure_classification: Option<String>,
#[serde(default)]
pub diagnostic: Option<String>,
#[serde(default)]
pub failing_turn: Option<u32>,
pub suppressed_later_turns: u32,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSessionTurnResult {
pub phase: String,
pub runtime_session_id: String,
pub turn_index: u32,
#[serde(default)]
pub pre_template_content_tokens: Option<u32>,
#[serde(default)]
pub observed_prompt_tokens: Option<u32>,
pub native_session_num: u64,
#[serde(default)]
pub preceding_native_session_num: Option<u64>,
#[serde(default)]
pub preceding_terminal_response_receipt_ns: Option<u64>,
#[serde(default)]
pub effective_inter_turn_delay_seconds: Option<f64>,
pub request_start_ns: u64,
#[serde(default)]
pub inter_turn_delay_reconciled: Option<bool>,
#[serde(default)]
pub post_failure_continuation: bool,
pub native_artifact_name: String,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSessionResultEvidence {
pub warmup: BenchSessionPhaseSummary,
pub profiling: BenchSessionPhaseSummary,
pub sessions: Vec<BenchRuntimeSessionResult>,
pub turns: Vec<BenchSessionTurnResult>,
pub population_slice_reconciled: bool,
pub sessions_reconciled: bool,
pub turn_order_reconciled: bool,
pub inter_turn_delays_reconciled: bool,
#[serde(default)]
pub native_requests_reconciled: Option<bool>,
pub counts_reconciled: bool,
#[serde(default)]
pub unavailable_dimensions: Vec<String>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchNativeInvocation {
pub purpose: String,
pub command: Vec<String>,
pub exit_code: Option<i32>,
pub interrupted: bool,
pub timed_out: bool,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct BenchRequestSloResult {
pub good_requests: u64,
pub good_request_ratio: f64,
pub goodput: f64,
pub profiling_duration_seconds: f64,
pub profiling_duration_source: String,
pub request_count_reconciled: bool,
#[serde(default)]
pub native_aggregate_good_request_count: Option<u64>,
#[serde(default)]
pub native_aggregate_good_request_count_consistent: Option<bool>,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct AdapterProtocol {
pub request: AdapterRequest,
pub response: AdapterResponse,
}
#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct MeasurementProtocol {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub eval_client_request: Option<EvalClientRequest>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub eval_client_result: Option<EvalClientResult>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bench_client_request: Option<BenchClientRequest>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bench_client_result: Option<BenchClientResult>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bench_population_preparation_request: Option<BenchPopulationPreparationRequest>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bench_population_preparation_result: Option<BenchPopulationPreparationResult>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data_asset_preparation_request: Option<MeasurementDataAssetPreparationRequest>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data_asset_preparation_result: Option<MeasurementDataAssetPreparationResult>,
}