inferlab_protocol/
wire.rs

1//! The versioned wire types of the framework integration protocol
2//! ([[RFC-0006:C-INTEGRATIONS]]): the one-shot stdin/stdout JSON contract for
3//! the plan/render serve operations, plus the client request/result surfaces
4//! the release-owned Eval and Bench measurement runtimes exchange with their
5//! clients. [`AdapterProtocol`] is the schema root from which the committed
6//! JSON schema and the Python SDK models are generated.
7
8use schemars::JsonSchema;
9use serde::{Deserialize, Serialize};
10use std::collections::BTreeMap;
11use std::path::PathBuf;
12
13/// The shared protocol version used by framework integrations and release-owned
14/// measurement clients. The only accepted value is `6` (serialized as the
15/// string `"6"`); a mismatch is rejected before lowering.
16#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
17pub enum ProtocolVersion {
18    /// Protocol version 6.
19    #[serde(rename = "6")]
20    V6,
21}
22
23/// The one JSON request an integration reads from stdin, tagged by the
24/// requested operation.
25#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
26#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
27pub enum AdapterRequest {
28    /// Plan a serve topology: declare roles, per-replica requirements, links,
29    /// and endpoint requirements from the requested shape.
30    PlanServe {
31        protocol_version: ProtocolVersion,
32        input: PlanServeInput,
33    },
34    /// Render final process invocations for a planned topology, given the
35    /// control plane's concrete process allocations.
36    RenderServe {
37        protocol_version: ProtocolVersion,
38        input: RenderServeInput,
39    },
40}
41
42impl AdapterRequest {
43    /// The protocol version carried by this request, regardless of operation.
44    #[must_use]
45    pub const fn protocol_version(&self) -> ProtocolVersion {
46        match self {
47            Self::PlanServe {
48                protocol_version, ..
49            }
50            | Self::RenderServe {
51                protocol_version, ..
52            } => *protocol_version,
53        }
54    }
55}
56
57/// The requested serve shape a `PlanServe` operation lowers into a topology.
58#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
59#[serde(deny_unknown_fields)]
60pub struct PlanServeInput {
61    pub model: ServeModelInput,
62    pub topology: ServeTopology,
63    #[serde(default, skip_serializing_if = "Option::is_none")]
64    pub routing_backend: Option<String>,
65    #[serde(default, skip_serializing_if = "Option::is_none")]
66    pub kv_transfer: Option<KvTransferMechanism>,
67    pub roles: Vec<ServeRoleInput>,
68    #[serde(default)]
69    pub profiling: bool,
70}
71
72/// The planned topology plus the control plane's concrete allocations that a
73/// `RenderServe` operation turns into final process invocations.
74#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
75#[serde(deny_unknown_fields)]
76pub struct RenderServeInput {
77    pub model: ServeModelInput,
78    pub topology: ServeTopology,
79    #[serde(default, skip_serializing_if = "Option::is_none")]
80    pub routing_backend: Option<String>,
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub kv_transfer: Option<KvTransferMechanism>,
83    pub roles: Vec<ServeRoleResult>,
84    pub routing: RoutingResult,
85    pub links: Vec<ServeRoleLink>,
86    pub allocations: Vec<ServeProcessAllocation>,
87    #[serde(default)]
88    pub render_inputs: Vec<SuppliedRenderInput>,
89    #[serde(default)]
90    pub profiling: bool,
91}
92
93/// The serving deployment topology.
94#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
95#[serde(rename_all = "snake_case")]
96pub enum ServeTopology {
97    /// One aggregated serving role.
98    Single,
99    /// Disaggregated prefill and decode roles.
100    PrefillDecode,
101}
102
103/// The logical role a replica plays in a serve topology.
104#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
105#[serde(rename_all = "snake_case")]
106pub enum ServeRoleKind {
107    /// Aggregated serving role of a single topology.
108    Serve,
109    /// Prefill role of a disaggregated topology.
110    Prefill,
111    /// Decode role of a disaggregated topology.
112    Decode,
113    /// Request-routing role that does not execute model inference.
114    Router,
115}
116
117/// The KV-transfer mechanism connecting prefill and decode.
118#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
119#[serde(rename_all = "snake_case")]
120pub enum KvTransferMechanism {
121    /// Mooncake KV-cache transfer.
122    Mooncake,
123    /// NIXL KV-cache transfer.
124    Nixl,
125}
126
127/// A requested serving role: its identity, kind, replica cardinality, and
128/// declared (not-yet-completed) parallelism and settings.
129#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
130#[serde(deny_unknown_fields)]
131pub struct ServeRoleInput {
132    pub id: String,
133    pub kind: ServeRoleKind,
134    pub replica_count: u32,
135    pub parallelism: Parallelism,
136    pub settings: BTreeMap<String, SettingValue>,
137}
138
139/// A role as the integration resolved it: preserved identity and cardinality
140/// with the complete effective settings and parallelism.
141#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
142#[serde(deny_unknown_fields)]
143pub struct ServeRoleResult {
144    pub id: String,
145    pub kind: ServeRoleKind,
146    pub declared_replica_count: u32,
147    pub effective_replica_count: u32,
148    pub effective_settings: BTreeMap<String, SettingValue>,
149    pub effective_parallelism: Parallelism,
150}
151
152/// Framework-neutral component-aware parallelism ([[RFC-0003:C-SERVE-PARALLELISM]]).
153/// Every component is optional so an operator can override one without
154/// repeating the rest; omitted components are filled by the integration into a
155/// complete effective shape.
156#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
157#[serde(default, deny_unknown_fields)]
158pub struct Parallelism {
159    /// Outer deployment parallelism shared by attention and experts.
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub outer: Option<ParallelismOuter>,
162    /// Attention-block parallelism.
163    #[serde(skip_serializing_if = "Option::is_none")]
164    pub attention: Option<ParallelismAttention>,
165    /// MoE expert parallelism.
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub experts: Option<ParallelismExperts>,
168}
169
170impl Parallelism {
171    /// Overlay the components present in `other` onto `self`, leaving
172    /// components `other` omits untouched (the per-component precedence merge).
173    pub fn merge_from(&mut self, other: &Self) {
174        if let Some(other) = &other.outer {
175            self.outer.get_or_insert_default().merge_from(other);
176        }
177        if let Some(other) = &other.attention {
178            self.attention.get_or_insert_default().merge_from(other);
179        }
180        if let Some(other) = &other.experts {
181            self.experts.get_or_insert_default().merge_from(other);
182        }
183    }
184}
185
186/// Outer deployment parallelism: tensor and pipeline degrees.
187#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
188#[serde(default, deny_unknown_fields)]
189pub struct ParallelismOuter {
190    #[schemars(range(min = 1))]
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub tensor_parallel_size: Option<u32>,
193    #[schemars(range(min = 1))]
194    #[serde(skip_serializing_if = "Option::is_none")]
195    pub pipeline_parallel_size: Option<u32>,
196}
197
198impl ParallelismOuter {
199    fn merge_from(&mut self, other: &Self) {
200        if other.tensor_parallel_size.is_some() {
201            self.tensor_parallel_size = other.tensor_parallel_size;
202        }
203        if other.pipeline_parallel_size.is_some() {
204            self.pipeline_parallel_size = other.pipeline_parallel_size;
205        }
206    }
207}
208
209/// Attention-block parallelism: tensor, data, and context degrees.
210#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
211#[serde(default, deny_unknown_fields)]
212pub struct ParallelismAttention {
213    #[schemars(range(min = 1))]
214    #[serde(skip_serializing_if = "Option::is_none")]
215    pub tensor_parallel_size: Option<u32>,
216    #[schemars(range(min = 1))]
217    #[serde(skip_serializing_if = "Option::is_none")]
218    pub data_parallel_size: Option<u32>,
219    #[schemars(range(min = 1))]
220    #[serde(skip_serializing_if = "Option::is_none")]
221    pub context_parallel_size: Option<u32>,
222}
223
224impl ParallelismAttention {
225    fn merge_from(&mut self, other: &Self) {
226        if other.tensor_parallel_size.is_some() {
227            self.tensor_parallel_size = other.tensor_parallel_size;
228        }
229        if other.data_parallel_size.is_some() {
230            self.data_parallel_size = other.data_parallel_size;
231        }
232        if other.context_parallel_size.is_some() {
233            self.context_parallel_size = other.context_parallel_size;
234        }
235    }
236}
237
238/// MoE expert parallelism: tensor, data, expert, and dense-tensor degrees.
239#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
240#[serde(default, deny_unknown_fields)]
241pub struct ParallelismExperts {
242    #[schemars(range(min = 1))]
243    #[serde(skip_serializing_if = "Option::is_none")]
244    pub tensor_parallel_size: Option<u32>,
245    #[schemars(range(min = 1))]
246    #[serde(skip_serializing_if = "Option::is_none")]
247    pub data_parallel_size: Option<u32>,
248    #[schemars(range(min = 1))]
249    #[serde(skip_serializing_if = "Option::is_none")]
250    pub expert_parallel_size: Option<u32>,
251    #[schemars(range(min = 1))]
252    #[serde(skip_serializing_if = "Option::is_none")]
253    pub dense_tensor_parallel_size: Option<u32>,
254}
255
256impl ParallelismExperts {
257    fn merge_from(&mut self, other: &Self) {
258        if other.tensor_parallel_size.is_some() {
259            self.tensor_parallel_size = other.tensor_parallel_size;
260        }
261        if other.data_parallel_size.is_some() {
262            self.data_parallel_size = other.data_parallel_size;
263        }
264        if other.expert_parallel_size.is_some() {
265            self.expert_parallel_size = other.expert_parallel_size;
266        }
267        if other.dense_tensor_parallel_size.is_some() {
268            self.dense_tensor_parallel_size = other.dense_tensor_parallel_size;
269        }
270    }
271}
272
273/// The logical model supplied during serving planning and rendering.
274#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
275#[serde(deny_unknown_fields)]
276pub struct ServeModelInput {
277    pub id: String,
278    pub served_name: String,
279}
280
281/// The model identity used by measurement clients. Unlike integration
282/// planning, a benchmark client may need a controller-visible tokenizer
283/// locator.
284#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
285#[serde(deny_unknown_fields)]
286pub struct MeasurementModelInput {
287    pub locator: String,
288    pub served_name: String,
289}
290
291/// A concrete host/port endpoint the control plane allocated for a process.
292#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
293#[serde(deny_unknown_fields)]
294pub struct EndpointAssignment {
295    pub host: String,
296    pub port: u16,
297}
298
299/// The public workload endpoint an Eval or Bench client connects to.
300#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
301#[serde(deny_unknown_fields)]
302pub struct ClientEndpointInput {
303    pub protocol: EndpointProtocol,
304    pub host: String,
305    pub port: u16,
306    pub completions_path: String,
307    pub chat_completions_path: String,
308}
309
310/// The measurement an Eval client runs against the workload endpoint.
311#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
312#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
313pub enum EvalDefinitionInput {
314    /// A single-prompt liveness check.
315    #[serde(rename = "openai_smoke")]
316    OpenAiSmoke {
317        prompt: String,
318        max_tokens: u32,
319        timeout_seconds: u64,
320    },
321    /// An lm-eval task run with a pass threshold on the chosen metric.
322    LmEval {
323        task: Box<EvalTaskSourceInput>,
324        #[serde(default)]
325        request_body: BTreeMap<String, SettingValue>,
326        limit: Option<u32>,
327        few_shot: Option<u32>,
328        seed: Option<u64>,
329        trials: u32,
330        max_tokens: Option<u32>,
331        concurrency: Option<u32>,
332        metric: String,
333        #[serde(default)]
334        metric_filter: Option<String>,
335        threshold: f64,
336        timeout_seconds: u64,
337    },
338}
339
340/// The resolved lm-eval task source consumed by the release-owned Eval runner.
341#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
342#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
343pub enum EvalTaskSourceInput {
344    /// One individual task shipped by the pinned lm-eval runtime.
345    BuiltIn { name: String },
346    /// One task closure shipped and identity-bound by the Inferlab release.
347    Bundled {
348        name: String,
349        task_identity: String,
350        path: PathBuf,
351        task_closure_sha256: String,
352        task_definition_sha256: String,
353        prompt_asset_sha256: String,
354        dataset_asset_sha256: String,
355        scorer_sha256: String,
356    },
357    /// One validated workspace-local lm-eval YAML file.
358    WorkspaceYaml { path: PathBuf },
359}
360
361/// The workload shape a Bench client drives, shared across its load cases.
362#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
363#[serde(deny_unknown_fields)]
364pub struct BenchDefinitionInput {
365    pub request_source: BenchRequestSourceInput,
366    pub seed: u64,
367    #[serde(default)]
368    pub request_body: BTreeMap<String, SettingValue>,
369    #[serde(default)]
370    pub request_slo: Option<BenchRequestSloInput>,
371    pub timeout_seconds: u64,
372    #[serde(default)]
373    pub reset_prefix_cache: bool,
374}
375
376/// One closed request origin lowered by Inferlab for the Bench runtime.
377#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
378#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
379pub enum BenchRequestSourceInput {
380    /// AIPerf generates exact token-shape prompts from the release-pinned
381    /// synthetic generator.
382    Random {
383        input_tokens: u32,
384        output_tokens: u32,
385    },
386    /// Inferlab materializes a release-catalog conversation snapshot before
387    /// AIPerf starts.
388    Dataset {
389        dataset: BenchDatasetInput,
390        max_input_tokens: u32,
391        output_tokens: Option<u32>,
392        catalog: BenchDatasetCatalogInput,
393    },
394}
395
396/// Release-qualified public Bench datasets.
397#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
398#[serde(rename_all = "snake_case")]
399pub enum BenchDatasetInput {
400    Sharegpt,
401}
402
403/// Immutable release-catalog facts resolved by the Rust control plane.
404#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
405#[serde(deny_unknown_fields)]
406pub struct BenchDatasetCatalogInput {
407    pub upstream_identity: String,
408    pub url: String,
409    pub sha256: String,
410    pub source_format: String,
411    pub license: String,
412    pub cache_path: PathBuf,
413    pub cache_state: BenchDatasetCacheState,
414    pub materialization_identity: String,
415}
416
417/// Read-only cache state observed while resolving the Bench plan.
418#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
419#[serde(rename_all = "snake_case")]
420pub enum BenchDatasetCacheState {
421    Missing,
422    Present,
423}
424
425/// One frozen dataset population consumed sequentially by every Bench case.
426#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
427#[serde(deny_unknown_fields)]
428pub struct BenchPopulationInput {
429    pub path: PathBuf,
430    pub sha256: String,
431    pub entries: u32,
432    pub tpot_applicable: bool,
433}
434
435/// The bounded tokenizer-backed operation that materializes one dataset
436/// population before any Bench case starts.
437#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
438#[serde(deny_unknown_fields)]
439pub struct BenchDatasetPreparationRequest {
440    pub protocol_version: ProtocolVersion,
441    pub model: MeasurementModelInput,
442    pub request_source: BenchRequestSourceInput,
443    pub source_path: PathBuf,
444    pub required_entries: u32,
445    pub seed: u64,
446    #[serde(default)]
447    pub request_body: BTreeMap<String, SettingValue>,
448    pub artifact_dir: PathBuf,
449}
450
451/// Summary of the realized token counts. Exact per-entry values remain in the
452/// population evidence artifact.
453#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
454#[serde(deny_unknown_fields)]
455pub struct BenchTokenCountSummary {
456    pub minimum: u32,
457    pub maximum: u32,
458    pub mean: f64,
459}
460
461/// Terminal result of dataset population materialization.
462#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
463#[serde(deny_unknown_fields)]
464pub struct BenchDatasetPreparationResult {
465    pub schema_version: u32,
466    pub status: ClientStatus,
467    pub materialization_identity: String,
468    pub requested_entries: u32,
469    pub candidate_entries: u64,
470    pub admitted_entries: u64,
471    pub ineligible_entries: u64,
472    #[serde(default)]
473    pub ineligible_reasons: BTreeMap<String, u64>,
474    pub population: Option<BenchPopulationInput>,
475    pub input_tokens: Option<BenchTokenCountSummary>,
476    pub output_tokens: Option<BenchTokenCountSummary>,
477    pub evidence_path: Option<PathBuf>,
478    pub error: Option<String>,
479}
480
481/// Per-request latency bounds lowered to the release-owned Bench runner.
482#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
483#[serde(deny_unknown_fields)]
484pub struct BenchRequestSloInput {
485    #[serde(default)]
486    pub request_latency_ms: Option<f64>,
487    #[serde(default)]
488    pub ttft_ms: Option<f64>,
489    #[serde(default)]
490    pub tpot_ms: Option<f64>,
491    pub minimum_good_request_ratio: f64,
492}
493
494/// A framework-specific server setting value carried as structured JSON data
495/// (never a pre-rendered shell fragment) across the integration boundary.
496#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
497#[serde(untagged)]
498pub enum SettingValue {
499    /// A JSON boolean.
500    Bool(bool),
501    /// A JSON integer.
502    Integer(i64),
503    /// A JSON floating-point number.
504    Float(f64),
505    /// A JSON string.
506    String(String),
507    /// A JSON array of setting values.
508    Array(Vec<SettingValue>),
509    /// A JSON object of named setting values.
510    Object(BTreeMap<String, SettingValue>),
511}
512
513/// The one JSON response an integration writes to stdout, tagged by outcome.
514#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
515#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
516pub enum AdapterResponse {
517    /// The operation succeeded and carries its result.
518    Ok {
519        protocol_version: ProtocolVersion,
520        result: Box<AdapterResult>,
521    },
522    /// The operation was rejected with a structured error.
523    Error {
524        protocol_version: ProtocolVersion,
525        error: AdapterError,
526    },
527}
528
529impl AdapterResponse {
530    /// The protocol version carried by this response, regardless of outcome.
531    #[must_use]
532    pub const fn protocol_version(&self) -> ProtocolVersion {
533        match self {
534            Self::Ok {
535                protocol_version, ..
536            }
537            | Self::Error {
538                protocol_version, ..
539            } => *protocol_version,
540        }
541    }
542}
543
544/// The successful result of an [`AdapterRequest`], tagged by the operation it
545/// answers.
546#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
547#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
548pub enum AdapterResult {
549    /// The planned topology from a `PlanServe` request.
550    PlanServe { output: Box<PlanServeResult> },
551    /// The rendered process invocations from a `RenderServe` request.
552    RenderServe { output: Box<RenderServeResult> },
553}
554
555/// The lowered topology returned by a `PlanServe`: the effective server-level
556/// shape, per-role resolution, whole-replica requirements, role links, and the
557/// public and per-role endpoint requirements the control plane then allocates.
558#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
559#[serde(deny_unknown_fields)]
560pub struct PlanServeResult {
561    pub integration: IntegrationIdentity,
562    pub roles: Vec<ServeRoleResult>,
563    pub replicas: Vec<ServeReplicaRequirement>,
564    pub links: Vec<ServeRoleLink>,
565    pub routing: RoutingResult,
566    pub endpoint: EndpointRequirement,
567    #[serde(default)]
568    pub render_inputs: Vec<RenderInputDeclaration>,
569}
570
571/// One workspace-authored UTF-8 source file an integration declares during
572/// planning for the control plane to supply during final rendering.
573#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
574#[serde(deny_unknown_fields)]
575pub struct RenderInputDeclaration {
576    pub source_path: String,
577}
578
579/// The original declared path plus the exact UTF-8 contents and digest the
580/// control plane supplies to an integration during final rendering.
581#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
582#[serde(deny_unknown_fields)]
583pub struct SuppliedRenderInput {
584    pub source_path: String,
585    pub text: String,
586    pub sha256: String,
587}
588
589/// A whole-replica resource and readiness requirement the integration declares
590/// without choosing placement, ranks, or concrete endpoints.
591#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
592#[serde(deny_unknown_fields)]
593pub struct ServeReplicaRequirement {
594    pub id: String,
595    pub role_id: String,
596    pub replica_index: u32,
597    pub device_count: u32,
598    pub ports: Vec<String>,
599    pub primary_ports: Vec<String>,
600    pub primary_readiness: ReadinessProbe,
601    pub worker_readiness: ReadinessProbe,
602    #[serde(default, skip_serializing_if = "Option::is_none")]
603    pub capture_target: Option<CaptureTargetRequirement>,
604}
605
606/// One concrete process the control plane placed, supplied to `RenderServe`:
607/// its identity, rank, machine, devices, model locator, and allocated
608/// endpoints and named ports.
609#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
610#[serde(deny_unknown_fields)]
611pub struct ServeProcessAllocation {
612    pub process: String,
613    pub role: String,
614    pub replica: u32,
615    pub rank: u32,
616    pub rank_count: u32,
617    pub machine: String,
618    pub devices: Vec<u32>,
619    #[serde(default, skip_serializing_if = "Option::is_none")]
620    pub model_locator: Option<String>,
621    #[serde(default, skip_serializing_if = "Option::is_none")]
622    pub endpoint: Option<EndpointAssignment>,
623    pub ports: BTreeMap<String, EndpointAssignment>,
624    pub cache: String,
625    pub launch: AllocationLaunch,
626    #[serde(default)]
627    pub dependencies: Vec<String>,
628}
629
630/// The machine-local launch channel selected by the control plane.
631#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
632#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
633pub enum AllocationLaunch {
634    Local,
635    Ssh { target: String },
636}
637
638/// A directed link between serve roles the integration declares as part of the
639/// topology.
640#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
641#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
642pub enum ServeRoleLink {
643    /// The source role routes requests to the target roles.
644    RequestRouting {
645        source: String,
646        targets: Vec<String>,
647    },
648    /// KV cache is transferred from source to target over `mechanism`.
649    KvTransfer {
650        source: String,
651        target: String,
652        mechanism: KvTransferMechanism,
653    },
654    /// The source discovers the target through a bootstrap port.
655    Bootstrap {
656        source: String,
657        target: String,
658        port: String,
659    },
660    /// The source and target exchange out-of-band data over a side-channel port.
661    SideChannel {
662        source: String,
663        target: String,
664        port: String,
665    },
666}
667
668/// The closed routing owner selected during integration planning.
669#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
670#[serde(tag = "owner", rename_all = "snake_case", deny_unknown_fields)]
671pub enum RoutingResult {
672    Direct {
673        role: String,
674        replica: u32,
675    },
676    InferlabBuiltin {
677        implementation: BuiltinRouterKind,
678        policy: String,
679        prefill_role: String,
680        decode_role: String,
681        #[serde(default)]
682        ports: Vec<String>,
683        readiness: ReadinessProbe,
684    },
685    IntegrationNative {
686        role: String,
687        replica: u32,
688        policy: String,
689    },
690}
691
692/// An Inferlab-owned routing implementation with stable target semantics.
693#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
694#[serde(rename_all = "snake_case")]
695pub enum BuiltinRouterKind {
696    VllmMooncake,
697    VllmNixl,
698    Sglang,
699    Trtllm,
700}
701
702/// Marks a replica as a profiling capture target and carries its window
703/// control ([[RFC-0004:C-WORKLOAD-PROFILING]]).
704#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
705#[serde(deny_unknown_fields)]
706pub struct CaptureTargetRequirement {
707    pub control: CaptureControlRequirement,
708}
709
710/// The HTTP paths a capture target exposes to open and close a capture window.
711#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
712#[serde(deny_unknown_fields)]
713pub struct CaptureControlRequirement {
714    pub start_path: String,
715    pub stop_path: String,
716}
717
718/// The final process invocations returned by a `RenderServe`, one per supplied
719/// allocation.
720#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
721#[serde(deny_unknown_fields)]
722pub struct RenderServeResult {
723    pub integration: IntegrationIdentity,
724    pub processes: Vec<RenderedServeProcess>,
725}
726
727/// A rendered process bound to the allocation `id` it was produced for.
728#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
729#[serde(deny_unknown_fields)]
730pub struct RenderedServeProcess {
731    pub process: String,
732    pub role: String,
733    pub replica: u32,
734    pub rank: u32,
735    pub rank_count: u32,
736    pub launch_files: Vec<LaunchFileDeclaration>,
737    pub command: ProcessSpec,
738}
739
740/// One immutable text input a rendered process requires before it can launch.
741#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
742#[serde(deny_unknown_fields)]
743pub struct LaunchFileDeclaration {
744    pub relative_path: String,
745    pub text: String,
746    pub sha256: String,
747}
748
749/// An HTTP action (method and path) invoked against the workload endpoint.
750#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
751#[serde(deny_unknown_fields)]
752pub struct HttpActionSpec {
753    pub method: HttpMethod,
754    pub path: String,
755}
756
757/// The HTTP method of an [`HttpActionSpec`].
758#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
759#[serde(rename_all = "snake_case")]
760pub enum HttpMethod {
761    /// HTTP POST.
762    Post,
763}
764
765/// The integration's identity recorded on its results: adapter id, adapter
766/// version, and the framework it lowers to.
767#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
768#[serde(deny_unknown_fields)]
769pub struct IntegrationIdentity {
770    pub adapter_id: String,
771    pub adapter_version: String,
772    pub framework: String,
773    pub framework_version: String,
774}
775
776/// A launchable process: its argument vector and environment.
777#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
778#[serde(deny_unknown_fields)]
779pub struct ProcessSpec {
780    pub argv: Vec<String>,
781    pub env: BTreeMap<String, String>,
782}
783
784/// How the control plane decides a process is ready.
785#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
786#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
787pub enum ReadinessProbe {
788    /// Ready when an HTTP GET of `path` succeeds.
789    Http { path: String },
790    /// Ready when the public endpoint succeeds and its HTTP target registry
791    /// contains every control-plane-derived serving target.
792    HttpTargetRegistry(Box<HttpTargetRegistryReadiness>),
793    /// Ready as soon as the process is alive.
794    ProcessAlive,
795}
796
797/// The integration-owned HTTP registry contract for target-aware readiness.
798#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
799#[serde(deny_unknown_fields)]
800pub struct HttpTargetRegistryReadiness {
801    pub target_scheme: TargetEndpointScheme,
802    pub readiness_path: String,
803    pub registry_path: String,
804    pub targets_field: String,
805    pub target_url_field: String,
806    pub target_role_field: String,
807    pub target_healthy_field: String,
808    pub target_bootstrap_port_field: String,
809    pub prefill_role_value: String,
810    pub decode_role_value: String,
811    pub prefill_bootstrap_port: String,
812}
813
814/// The application protocol used to identify serving targets in a registry.
815#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
816#[serde(rename_all = "snake_case")]
817pub enum TargetEndpointScheme {
818    /// HTTP serving endpoint.
819    Http,
820    /// gRPC serving endpoint.
821    Grpc,
822}
823
824/// The workload endpoint's protocol and named OpenAI paths, plus an optional
825/// prefix-cache-reset action a Bench case can invoke between runs.
826#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
827#[serde(deny_unknown_fields)]
828pub struct EndpointRequirement {
829    pub protocol: EndpointProtocol,
830    pub completions_path: String,
831    pub chat_completions_path: String,
832    #[serde(default, skip_serializing_if = "Option::is_none")]
833    pub prefix_cache_reset: Option<HttpActionSpec>,
834}
835
836/// The application protocol a workload endpoint speaks.
837#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
838#[serde(rename_all = "snake_case")]
839pub enum EndpointProtocol {
840    /// HTTP.
841    Http,
842}
843
844/// A structured rejection an integration returns in an [`AdapterResponse::Error`].
845#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
846#[serde(deny_unknown_fields)]
847pub struct AdapterError {
848    pub code: AdapterErrorCode,
849    pub message: String,
850}
851
852/// Machine-readable failure category an adapter reports in an [`AdapterError`].
853#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
854#[serde(rename_all = "snake_case")]
855pub enum AdapterErrorCode {
856    /// The request was malformed or missing required fields.
857    InvalidRequest,
858    /// The request's protocol version is not accepted.
859    UnsupportedProtocolVersion,
860    /// A framework setting was unknown or invalid.
861    InvalidSettings,
862    /// An unexpected internal failure occurred in the integration.
863    Internal,
864    /// The requested operation is not supported by this integration.
865    UnsupportedOperation,
866}
867
868/// The request the Eval measurement runtime passes to its client: the endpoint
869/// to hit, the model, the eval definition, and where to write artifacts.
870#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
871#[serde(deny_unknown_fields)]
872pub struct EvalClientRequest {
873    pub protocol_version: ProtocolVersion,
874    pub workspace_root: PathBuf,
875    pub workspace_source_exclusions: Vec<PathBuf>,
876    pub endpoint: ClientEndpointInput,
877    pub model: MeasurementModelInput,
878    pub definition: EvalDefinitionInput,
879    /// Remaining control-plane case budget when the client is released.
880    pub case_budget_seconds: f64,
881    pub artifact_dir: PathBuf,
882}
883
884/// The request the Bench measurement runtime passes to its client: the
885/// endpoint, model, bench definition, the load case to run, and the artifact
886/// directory.
887#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
888#[serde(deny_unknown_fields)]
889pub struct BenchClientRequest {
890    pub protocol_version: ProtocolVersion,
891    pub endpoint: ClientEndpointInput,
892    pub model: MeasurementModelInput,
893    pub definition: BenchDefinitionInput,
894    #[serde(default)]
895    pub population: Option<BenchPopulationInput>,
896    pub case: BenchCaseInput,
897    /// Remaining control-plane case budget when the client is released.
898    pub case_budget_seconds: f64,
899    pub artifact_dir: PathBuf,
900}
901
902/// A single Bench case: its load shape and the number of requests to send.
903#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
904#[serde(deny_unknown_fields)]
905pub struct BenchCaseInput {
906    pub load_shape: BenchLoadInput,
907    pub request_count: u32,
908    #[serde(default)]
909    pub warmup_request_count: u32,
910}
911
912/// How a Bench case paces its requests.
913#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
914#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
915pub enum BenchLoadInput {
916    /// A fixed number of in-flight requests.
917    ConcurrencyLimited { concurrency: u32 },
918    /// A target arrival rate, optionally shaped by a burstiness factor.
919    RequestRateLimited {
920        request_rate: f64,
921        burstiness: Option<f64>,
922    },
923    /// All requests issued as fast as possible.
924    UnboundedRequestRate,
925}
926
927/// The terminal outcome a measurement client reports.
928#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
929#[serde(rename_all = "snake_case")]
930pub enum ClientStatus {
931    /// The client completed its measurement successfully.
932    Succeeded,
933    /// The client did not complete successfully.
934    Failed,
935}
936
937/// A typed Eval failure category preserved across the client boundary.
938#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
939#[serde(rename_all = "snake_case")]
940pub enum EvalFailureKind {
941    TaskResolution,
942    ProbeTokenizer,
943    ProbeTransport,
944    ProbeHttp,
945    ProbeMalformedResponse,
946    ProbeGeneratedOnlyLogprobs,
947    ProbeTokenizerAlignment,
948    MetricNormalization,
949}
950
951/// The threshold comparison selected from lm-eval's scoring direction.
952#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
953#[serde(rename_all = "snake_case")]
954pub enum EvalMetricComparison {
955    AtLeast,
956    AtMost,
957}
958
959/// The terminal conclusion of an lm-eval metric threshold gate.
960#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
961#[serde(rename_all = "snake_case")]
962pub enum EvalMetricGateConclusion {
963    Passed,
964    Failed,
965}
966
967/// One finite lm-eval metric with its exact native provenance.
968#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
969#[serde(deny_unknown_fields)]
970pub struct EvalNormalizedMetric {
971    pub source_identity: String,
972    pub metric: String,
973    pub filter: Option<String>,
974    pub native_metric_key: String,
975    pub value: f64,
976    pub higher_is_better: bool,
977}
978
979/// The effective threshold comparison for the configured primary metric.
980#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
981#[serde(deny_unknown_fields)]
982pub struct EvalMetricGate {
983    pub metric: EvalNormalizedMetric,
984    pub threshold: f64,
985    pub comparison: EvalMetricComparison,
986    pub conclusion: EvalMetricGateConclusion,
987}
988
989/// Reconstructible aggregate counts for fixed outer Eval trials.
990#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
991#[serde(deny_unknown_fields)]
992pub struct EvalTrialSummary {
993    pub requested_trials: u32,
994    pub issued_trials: u32,
995    pub unissued_trials: u32,
996    pub completed_trials: u32,
997    pub request_failure_trials: u32,
998    pub passed_trials: u32,
999    pub pass_rate: Option<f64>,
1000    pub per_trial_metric: String,
1001    pub per_trial_filter: Option<String>,
1002    pub higher_is_better: bool,
1003}
1004
1005/// The result an Eval client writes for the measurement runtime to consume.
1006#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1007#[serde(deny_unknown_fields)]
1008pub struct EvalClientResult {
1009    /// Result envelope version; clients write `1`. The measurement runtime
1010    /// rejects an eval result whose version is not `1`.
1011    pub schema_version: u32,
1012    pub status: ClientStatus,
1013    pub metrics: BTreeMap<String, f64>,
1014    #[serde(default)]
1015    pub normalized_metrics: BTreeMap<String, EvalNormalizedMetric>,
1016    #[serde(default)]
1017    pub gate: Option<EvalMetricGate>,
1018    #[serde(default)]
1019    pub trial_summary: Option<EvalTrialSummary>,
1020    pub native_command: Vec<String>,
1021    #[serde(default)]
1022    pub native_exit_code: Option<i32>,
1023    #[serde(default)]
1024    pub native_timed_out: bool,
1025    pub raw_artifacts: Vec<RawArtifact>,
1026    pub failure_kind: Option<EvalFailureKind>,
1027    pub error: Option<String>,
1028}
1029
1030/// The result a Bench client writes for the measurement runtime to consume.
1031#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1032#[serde(deny_unknown_fields)]
1033pub struct BenchClientResult {
1034    /// Result envelope version; clients write `1`. The measurement runtime
1035    /// rejects a bench result whose version is not `1`.
1036    pub schema_version: u32,
1037    pub status: ClientStatus,
1038    pub completed_requests: u64,
1039    pub failed_requests: u64,
1040    pub normalization_schema: String,
1041    pub metrics: BTreeMap<String, f64>,
1042    #[serde(default)]
1043    pub request_slo: Option<BenchRequestSloResult>,
1044    pub native_command: Vec<String>,
1045    pub native_exit_code: Option<i32>,
1046    pub raw_artifacts: Vec<RawArtifact>,
1047    pub error: Option<String>,
1048}
1049
1050/// File-bound request-SLO evidence derived from AIPerf profiling records.
1051#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1052#[serde(deny_unknown_fields)]
1053pub struct BenchRequestSloResult {
1054    pub good_requests: u64,
1055    pub good_request_ratio: f64,
1056    pub goodput: f64,
1057    pub profiling_duration_seconds: f64,
1058    pub profiling_duration_source: String,
1059    pub request_count_reconciled: bool,
1060    #[serde(default)]
1061    pub native_aggregate_good_request_count: Option<u64>,
1062    #[serde(default)]
1063    pub native_aggregate_good_request_count_consistent: Option<bool>,
1064}
1065
1066/// A raw output file a client produced, retained as workload evidence.
1067#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
1068#[serde(deny_unknown_fields)]
1069pub struct RawArtifact {
1070    pub name: String,
1071    pub kind: String,
1072    pub path: PathBuf,
1073}
1074
1075/// The schema root aggregating every wire type. It exists to generate one
1076/// committed JSON schema (and the Python SDK models); its optional client
1077/// fields are never all populated in a single message.
1078#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
1079#[serde(deny_unknown_fields)]
1080pub struct AdapterProtocol {
1081    pub request: AdapterRequest,
1082    pub response: AdapterResponse,
1083    #[serde(default, skip_serializing_if = "Option::is_none")]
1084    pub eval_client_request: Option<EvalClientRequest>,
1085    #[serde(default, skip_serializing_if = "Option::is_none")]
1086    pub eval_client_result: Option<EvalClientResult>,
1087    #[serde(default, skip_serializing_if = "Option::is_none")]
1088    pub bench_client_request: Option<BenchClientRequest>,
1089    #[serde(default, skip_serializing_if = "Option::is_none")]
1090    pub bench_client_result: Option<BenchClientResult>,
1091    #[serde(default, skip_serializing_if = "Option::is_none")]
1092    pub bench_dataset_preparation_request: Option<BenchDatasetPreparationRequest>,
1093    #[serde(default, skip_serializing_if = "Option::is_none")]
1094    pub bench_dataset_preparation_result: Option<BenchDatasetPreparationResult>,
1095}
inferlab_protocol/wire.rs

inferlab_protocol/
wire.rs
