inferlab_protocol/
wire.rs

1//! The versioned wire types of the framework integration protocol
2//! ([[RFC-0006:C-INTEGRATIONS]]): the one-shot stdin/stdout JSON contract for
3//! the plan/render serve operations, plus the client request/result surfaces
4//! the release-owned Eval and Bench measurement runtimes exchange with their
5//! clients. [`AdapterProtocol`] is the schema root from which the committed
6//! JSON schema and the Python SDK models are generated.
7
8use schemars::JsonSchema;
9use serde::{Deserialize, Serialize};
10use std::collections::BTreeMap;
11use std::path::PathBuf;
12
13/// The shared protocol version used by framework integrations and release-owned
14/// measurement clients. The only accepted value is `4` (serialized as the
15/// string `"4"`); a mismatch is rejected before lowering.
16#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
17pub enum ProtocolVersion {
18    /// Protocol version 4.
19    #[serde(rename = "4")]
20    V4,
21}
22
23/// The one JSON request an integration reads from stdin, tagged by the
24/// requested operation.
25#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
26#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
27pub enum AdapterRequest {
28    /// Plan a serve topology: declare roles, per-replica requirements, links,
29    /// and endpoint requirements from the requested shape.
30    PlanServe {
31        protocol_version: ProtocolVersion,
32        input: PlanServeInput,
33    },
34    /// Render final process invocations for a planned topology, given the
35    /// control plane's concrete process allocations.
36    RenderServe {
37        protocol_version: ProtocolVersion,
38        input: RenderServeInput,
39    },
40}
41
42impl AdapterRequest {
43    /// The protocol version carried by this request, regardless of operation.
44    #[must_use]
45    pub const fn protocol_version(&self) -> ProtocolVersion {
46        match self {
47            Self::PlanServe {
48                protocol_version, ..
49            }
50            | Self::RenderServe {
51                protocol_version, ..
52            } => *protocol_version,
53        }
54    }
55}
56
57/// The requested serve shape a `PlanServe` operation lowers into a topology.
58#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
59#[serde(deny_unknown_fields)]
60pub struct PlanServeInput {
61    pub model: ServeModelInput,
62    pub topology: ServeTopology,
63    pub routing_backend: String,
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub kv_transfer: Option<KvTransferMechanism>,
66    pub parallelism: Parallelism,
67    pub settings: BTreeMap<String, SettingValue>,
68    pub roles: Vec<ServeRoleInput>,
69    #[serde(default)]
70    pub profiling: bool,
71}
72
73/// The planned topology plus the control plane's concrete allocations that a
74/// `RenderServe` operation turns into final process invocations.
75#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
76#[serde(deny_unknown_fields)]
77pub struct RenderServeInput {
78    pub model: ServeModelInput,
79    pub topology: ServeTopology,
80    pub routing_backend: String,
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub kv_transfer: Option<KvTransferMechanism>,
83    pub parallelism: Parallelism,
84    pub settings: BTreeMap<String, SettingValue>,
85    pub roles: Vec<ServeRoleResult>,
86    pub links: Vec<ServeRoleLink>,
87    pub allocations: Vec<ServeProcessAllocation>,
88    #[serde(default)]
89    pub render_inputs: Vec<SuppliedRenderInput>,
90    #[serde(default)]
91    pub profiling: bool,
92}
93
94/// The serving deployment topology.
95#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
96#[serde(rename_all = "snake_case")]
97pub enum ServeTopology {
98    /// One aggregated serving role.
99    Single,
100    /// Disaggregated prefill and decode roles.
101    PrefillDecode,
102}
103
104/// The logical role a replica plays in a serve topology.
105#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
106#[serde(rename_all = "snake_case")]
107pub enum ServeRoleKind {
108    /// Aggregated serving role of a single topology.
109    Serve,
110    /// Prefill role of a disaggregated topology.
111    Prefill,
112    /// Decode role of a disaggregated topology.
113    Decode,
114    /// Request-routing role that does not execute model inference.
115    Router,
116}
117
118/// The KV-transfer mechanism connecting prefill and decode.
119#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
120#[serde(rename_all = "snake_case")]
121pub enum KvTransferMechanism {
122    /// Mooncake KV-cache transfer.
123    Mooncake,
124    /// NIXL KV-cache transfer.
125    Nixl,
126}
127
128/// A requested serving role: its identity, kind, replica cardinality, and
129/// declared (not-yet-completed) parallelism and settings.
130#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
131#[serde(deny_unknown_fields)]
132pub struct ServeRoleInput {
133    pub id: String,
134    pub kind: ServeRoleKind,
135    pub replica_count: u32,
136    pub parallelism: Parallelism,
137    pub settings: BTreeMap<String, SettingValue>,
138}
139
140/// A role as the integration resolved it: preserved identity and cardinality
141/// with the complete effective settings and parallelism.
142#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
143#[serde(deny_unknown_fields)]
144pub struct ServeRoleResult {
145    pub id: String,
146    pub kind: ServeRoleKind,
147    pub replica_count: u32,
148    pub effective_settings: BTreeMap<String, SettingValue>,
149    pub effective_parallelism: Parallelism,
150}
151
152/// Framework-neutral component-aware parallelism ([[RFC-0003:C-SERVE-PARALLELISM]]).
153/// Every component is optional so an operator can override one without
154/// repeating the rest; omitted components are filled by the integration into a
155/// complete effective shape.
156#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
157#[serde(default, deny_unknown_fields)]
158pub struct Parallelism {
159    /// Outer deployment parallelism shared by attention and experts.
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub outer: Option<ParallelismOuter>,
162    /// Attention-block parallelism.
163    #[serde(skip_serializing_if = "Option::is_none")]
164    pub attention: Option<ParallelismAttention>,
165    /// MoE expert parallelism.
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub experts: Option<ParallelismExperts>,
168}
169
170impl Parallelism {
171    /// Overlay the components present in `other` onto `self`, leaving
172    /// components `other` omits untouched (the per-component precedence merge).
173    pub fn merge_from(&mut self, other: &Self) {
174        if let Some(other) = &other.outer {
175            self.outer.get_or_insert_default().merge_from(other);
176        }
177        if let Some(other) = &other.attention {
178            self.attention.get_or_insert_default().merge_from(other);
179        }
180        if let Some(other) = &other.experts {
181            self.experts.get_or_insert_default().merge_from(other);
182        }
183    }
184}
185
186/// Outer deployment parallelism: tensor and pipeline degrees.
187#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
188#[serde(default, deny_unknown_fields)]
189pub struct ParallelismOuter {
190    #[schemars(range(min = 1))]
191    pub tensor_parallel_size: Option<u32>,
192    #[schemars(range(min = 1))]
193    pub pipeline_parallel_size: Option<u32>,
194}
195
196impl ParallelismOuter {
197    fn merge_from(&mut self, other: &Self) {
198        if other.tensor_parallel_size.is_some() {
199            self.tensor_parallel_size = other.tensor_parallel_size;
200        }
201        if other.pipeline_parallel_size.is_some() {
202            self.pipeline_parallel_size = other.pipeline_parallel_size;
203        }
204    }
205}
206
207/// Attention-block parallelism: tensor, data, and context degrees.
208#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
209#[serde(default, deny_unknown_fields)]
210pub struct ParallelismAttention {
211    #[schemars(range(min = 1))]
212    pub tensor_parallel_size: Option<u32>,
213    #[schemars(range(min = 1))]
214    pub data_parallel_size: Option<u32>,
215    #[schemars(range(min = 1))]
216    pub context_parallel_size: Option<u32>,
217}
218
219impl ParallelismAttention {
220    fn merge_from(&mut self, other: &Self) {
221        if other.tensor_parallel_size.is_some() {
222            self.tensor_parallel_size = other.tensor_parallel_size;
223        }
224        if other.data_parallel_size.is_some() {
225            self.data_parallel_size = other.data_parallel_size;
226        }
227        if other.context_parallel_size.is_some() {
228            self.context_parallel_size = other.context_parallel_size;
229        }
230    }
231}
232
233/// MoE expert parallelism: tensor, data, expert, and dense-tensor degrees.
234#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
235#[serde(default, deny_unknown_fields)]
236pub struct ParallelismExperts {
237    #[schemars(range(min = 1))]
238    pub tensor_parallel_size: Option<u32>,
239    #[schemars(range(min = 1))]
240    pub data_parallel_size: Option<u32>,
241    #[schemars(range(min = 1))]
242    pub expert_parallel_size: Option<u32>,
243    #[schemars(range(min = 1))]
244    pub dense_tensor_parallel_size: Option<u32>,
245}
246
247impl ParallelismExperts {
248    fn merge_from(&mut self, other: &Self) {
249        if other.tensor_parallel_size.is_some() {
250            self.tensor_parallel_size = other.tensor_parallel_size;
251        }
252        if other.data_parallel_size.is_some() {
253            self.data_parallel_size = other.data_parallel_size;
254        }
255        if other.expert_parallel_size.is_some() {
256            self.expert_parallel_size = other.expert_parallel_size;
257        }
258        if other.dense_tensor_parallel_size.is_some() {
259            self.dense_tensor_parallel_size = other.dense_tensor_parallel_size;
260        }
261    }
262}
263
264/// The model to serve: its resolved locator and the name it is served under.
265#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
266#[serde(deny_unknown_fields)]
267pub struct ServeModelInput {
268    pub locator: String,
269    pub served_name: String,
270}
271
272/// A concrete host/port endpoint the control plane allocated for a process.
273#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
274#[serde(deny_unknown_fields)]
275pub struct EndpointAssignment {
276    pub host: String,
277    pub port: u16,
278}
279
280/// The public workload endpoint an Eval or Bench client connects to.
281#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
282#[serde(deny_unknown_fields)]
283pub struct ClientEndpointInput {
284    pub protocol: EndpointProtocol,
285    pub host: String,
286    pub port: u16,
287    pub api_path: String,
288}
289
290/// The measurement an Eval client runs against the workload endpoint.
291#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
292#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
293pub enum EvalDefinitionInput {
294    /// A single-prompt liveness check.
295    #[serde(rename = "openai_smoke")]
296    OpenAiSmoke {
297        prompt: String,
298        max_tokens: u32,
299        timeout_seconds: u64,
300    },
301    /// An lm-eval task run with a pass threshold on the chosen metric.
302    LmEval {
303        task: String,
304        dataset: Option<String>,
305        split: Option<String>,
306        limit: Option<u32>,
307        few_shot: Option<u32>,
308        seed: Option<u64>,
309        max_tokens: Option<u32>,
310        concurrency: Option<u32>,
311        metric: String,
312        threshold: f64,
313        timeout_seconds: u64,
314    },
315}
316
317/// The workload shape a Bench client drives, shared across its load cases.
318#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
319#[serde(deny_unknown_fields)]
320pub struct BenchDefinitionInput {
321    pub input_tokens: u32,
322    pub output_tokens: u32,
323    pub seed: u64,
324    pub temperature: f64,
325    pub timeout_seconds: u64,
326    #[serde(default)]
327    pub reset_prefix_cache: bool,
328}
329
330/// A framework-specific server setting value carried as structured JSON data
331/// (never a pre-rendered shell fragment) across the integration boundary.
332#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
333#[serde(untagged)]
334pub enum SettingValue {
335    /// A JSON boolean.
336    Bool(bool),
337    /// A JSON integer.
338    Integer(i64),
339    /// A JSON floating-point number.
340    Float(f64),
341    /// A JSON string.
342    String(String),
343    /// A JSON array of setting values.
344    Array(Vec<SettingValue>),
345    /// A JSON object of named setting values.
346    Object(BTreeMap<String, SettingValue>),
347}
348
349/// The one JSON response an integration writes to stdout, tagged by outcome.
350#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
351#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
352pub enum AdapterResponse {
353    /// The operation succeeded and carries its result.
354    Ok {
355        protocol_version: ProtocolVersion,
356        result: Box<AdapterResult>,
357    },
358    /// The operation was rejected with a structured error.
359    Error {
360        protocol_version: ProtocolVersion,
361        error: AdapterError,
362    },
363}
364
365impl AdapterResponse {
366    /// The protocol version carried by this response, regardless of outcome.
367    #[must_use]
368    pub const fn protocol_version(&self) -> ProtocolVersion {
369        match self {
370            Self::Ok {
371                protocol_version, ..
372            }
373            | Self::Error {
374                protocol_version, ..
375            } => *protocol_version,
376        }
377    }
378}
379
380/// The successful result of an [`AdapterRequest`], tagged by the operation it
381/// answers.
382#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
383#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
384pub enum AdapterResult {
385    /// The planned topology from a `PlanServe` request.
386    PlanServe { output: Box<PlanServeResult> },
387    /// The rendered process invocations from a `RenderServe` request.
388    RenderServe { output: Box<RenderServeResult> },
389}
390
391/// The lowered topology returned by a `PlanServe`: the effective server-level
392/// shape, per-role resolution, whole-replica requirements, role links, and the
393/// public and per-role endpoint requirements the control plane then allocates.
394#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
395#[serde(deny_unknown_fields)]
396pub struct PlanServeResult {
397    pub integration: IntegrationIdentity,
398    pub effective_settings: BTreeMap<String, SettingValue>,
399    pub effective_parallelism: Parallelism,
400    pub roles: Vec<ServeRoleResult>,
401    pub replicas: Vec<ServeReplicaRequirement>,
402    pub links: Vec<ServeRoleLink>,
403    pub public_endpoint: PublicEndpointRequirement,
404    pub endpoint: EndpointRequirement,
405    #[serde(default)]
406    pub render_inputs: Vec<RenderInputDeclaration>,
407}
408
409/// One workspace-authored UTF-8 source file an integration declares during
410/// planning for the control plane to supply during final rendering.
411#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
412#[serde(deny_unknown_fields)]
413pub struct RenderInputDeclaration {
414    pub source_path: String,
415}
416
417/// The original declared path plus the exact UTF-8 contents and digest the
418/// control plane supplies to an integration during final rendering.
419#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
420#[serde(deny_unknown_fields)]
421pub struct SuppliedRenderInput {
422    pub source_path: String,
423    pub text: String,
424    pub sha256: String,
425}
426
427/// A whole-replica resource and readiness requirement the integration declares
428/// without choosing placement, ranks, or concrete endpoints.
429#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
430#[serde(deny_unknown_fields)]
431pub struct ServeReplicaRequirement {
432    pub id: String,
433    pub role_id: String,
434    pub replica_index: u32,
435    pub accelerator_count: u32,
436    pub ports: Vec<String>,
437    pub primary_ports: Vec<String>,
438    pub primary_readiness: ReadinessProbe,
439    pub worker_readiness: ReadinessProbe,
440    #[serde(default, skip_serializing_if = "Option::is_none")]
441    pub capture_target: Option<CaptureTargetRequirement>,
442}
443
444/// One concrete process the control plane placed, supplied to `RenderServe`:
445/// its identity, rank, machine, devices, model locator, and allocated
446/// endpoints and named ports.
447#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
448#[serde(deny_unknown_fields)]
449pub struct ServeProcessAllocation {
450    pub process_id: String,
451    pub role_id: String,
452    pub replica_id: String,
453    pub replica_index: u32,
454    pub rank: u32,
455    pub machine_id: String,
456    pub model_locator: String,
457    pub runtime_cache_root: String,
458    pub devices: Vec<u32>,
459    pub endpoint: EndpointAssignment,
460    pub ports: BTreeMap<String, EndpointAssignment>,
461}
462
463/// A directed link between serve roles the integration declares as part of the
464/// topology.
465#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
466#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
467pub enum ServeRoleLink {
468    /// The source role routes requests to the target roles.
469    RequestRouting {
470        source: String,
471        targets: Vec<String>,
472    },
473    /// KV cache is transferred from source to target over `mechanism`.
474    KvTransfer {
475        source: String,
476        target: String,
477        mechanism: KvTransferMechanism,
478    },
479    /// The source discovers the target through a bootstrap port.
480    Bootstrap {
481        source: String,
482        target: String,
483        port: String,
484    },
485    /// The source and target exchange out-of-band data over a side-channel port.
486    SideChannel {
487        source: String,
488        target: String,
489        port: String,
490    },
491}
492
493/// How the topology's public workload endpoint is exposed.
494#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
495#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
496pub enum PublicEndpointRequirement {
497    /// The endpoint is served directly by a single replica.
498    Replica { replica_id: String },
499    /// The endpoint is fronted by an Inferlab-owned built-in proxy routing
500    /// across the named prefill and decode roles.
501    BuiltinProxy {
502        process_id: String,
503        role_id: String,
504        prefill_role: String,
505        decode_role: String,
506        readiness: ReadinessProbe,
507    },
508}
509
510/// Marks a replica as a profiling capture target and carries its window
511/// control ([[RFC-0004:C-WORKLOAD-PROFILING]]).
512#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
513#[serde(deny_unknown_fields)]
514pub struct CaptureTargetRequirement {
515    pub control: CaptureControlRequirement,
516}
517
518/// The HTTP paths a capture target exposes to open and close a capture window.
519#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
520#[serde(deny_unknown_fields)]
521pub struct CaptureControlRequirement {
522    pub start_path: String,
523    pub stop_path: String,
524}
525
526/// The final process invocations returned by a `RenderServe`, one per supplied
527/// allocation.
528#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
529#[serde(deny_unknown_fields)]
530pub struct RenderServeResult {
531    pub integration: IntegrationIdentity,
532    pub processes: Vec<RenderedServeProcess>,
533}
534
535/// A rendered process bound to the allocation `id` it was produced for.
536#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
537#[serde(deny_unknown_fields)]
538pub struct RenderedServeProcess {
539    pub id: String,
540    pub launch_files: Vec<LaunchFileDeclaration>,
541    pub process: ProcessSpec,
542}
543
544/// One immutable text input a rendered process requires before it can launch.
545#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
546#[serde(deny_unknown_fields)]
547pub struct LaunchFileDeclaration {
548    pub relative_path: String,
549    pub text: String,
550    pub sha256: String,
551}
552
553/// An HTTP action (method and path) invoked against the workload endpoint.
554#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
555#[serde(deny_unknown_fields)]
556pub struct HttpActionSpec {
557    pub method: HttpMethod,
558    pub path: String,
559}
560
561/// The HTTP method of an [`HttpActionSpec`].
562#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
563#[serde(rename_all = "snake_case")]
564pub enum HttpMethod {
565    /// HTTP POST.
566    Post,
567}
568
569/// The integration's identity recorded on its results: adapter id, adapter
570/// version, and the framework it lowers to.
571#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
572#[serde(deny_unknown_fields)]
573pub struct IntegrationIdentity {
574    pub adapter_id: String,
575    pub adapter_version: String,
576    pub framework: String,
577}
578
579/// A launchable process: its argument vector and environment.
580#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
581#[serde(deny_unknown_fields)]
582pub struct ProcessSpec {
583    pub argv: Vec<String>,
584    pub env: BTreeMap<String, String>,
585}
586
587/// How the control plane decides a process is ready.
588#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
589#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
590pub enum ReadinessProbe {
591    /// Ready when an HTTP GET of `path` succeeds.
592    Http { path: String },
593    /// Ready when the public endpoint succeeds and its HTTP target registry
594    /// contains every control-plane-derived serving target.
595    HttpTargetRegistry(Box<HttpTargetRegistryReadiness>),
596    /// Ready as soon as the process is alive.
597    ProcessAlive,
598}
599
600/// The integration-owned HTTP registry contract for target-aware readiness.
601#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
602#[serde(deny_unknown_fields)]
603pub struct HttpTargetRegistryReadiness {
604    pub target_scheme: TargetEndpointScheme,
605    pub readiness_path: String,
606    pub registry_path: String,
607    pub targets_field: String,
608    pub target_url_field: String,
609    pub target_role_field: String,
610    pub target_healthy_field: String,
611    pub target_bootstrap_port_field: String,
612    pub prefill_role_value: String,
613    pub decode_role_value: String,
614    pub prefill_bootstrap_port: String,
615}
616
617/// The application protocol used to identify serving targets in a registry.
618#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
619#[serde(rename_all = "snake_case")]
620pub enum TargetEndpointScheme {
621    /// HTTP serving endpoint.
622    Http,
623    /// gRPC serving endpoint.
624    Grpc,
625}
626
627/// The workload endpoint's protocol and API path, plus an optional
628/// prefix-cache-reset action a Bench case can invoke between runs.
629#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
630#[serde(deny_unknown_fields)]
631pub struct EndpointRequirement {
632    pub protocol: EndpointProtocol,
633    pub api_path: String,
634    #[serde(default, skip_serializing_if = "Option::is_none")]
635    pub prefix_cache_reset: Option<HttpActionSpec>,
636}
637
638/// The application protocol a workload endpoint speaks.
639#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
640#[serde(rename_all = "snake_case")]
641pub enum EndpointProtocol {
642    /// HTTP.
643    Http,
644}
645
646/// A structured rejection an integration returns in an [`AdapterResponse::Error`].
647#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
648#[serde(deny_unknown_fields)]
649pub struct AdapterError {
650    pub code: AdapterErrorCode,
651    pub message: String,
652}
653
654/// Machine-readable failure category an adapter reports in an [`AdapterError`].
655#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
656#[serde(rename_all = "snake_case")]
657pub enum AdapterErrorCode {
658    /// The request was malformed or missing required fields.
659    InvalidRequest,
660    /// The request's protocol version is not accepted.
661    UnsupportedProtocolVersion,
662    /// A framework setting was unknown or invalid.
663    InvalidSettings,
664    /// An unexpected internal failure occurred in the integration.
665    Internal,
666    /// The requested operation is not supported by this integration.
667    UnsupportedOperation,
668}
669
670/// The request the Eval measurement runtime passes to its client: the endpoint
671/// to hit, the model, the eval definition, and where to write artifacts.
672#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
673#[serde(deny_unknown_fields)]
674pub struct EvalClientRequest {
675    pub protocol_version: ProtocolVersion,
676    pub endpoint: ClientEndpointInput,
677    pub model: ServeModelInput,
678    pub definition: EvalDefinitionInput,
679    pub artifact_dir: PathBuf,
680}
681
682/// The request the Bench measurement runtime passes to its client: the
683/// endpoint, model, bench definition, the load case to run, and the artifact
684/// directory.
685#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
686#[serde(deny_unknown_fields)]
687pub struct BenchClientRequest {
688    pub protocol_version: ProtocolVersion,
689    pub endpoint: ClientEndpointInput,
690    pub model: ServeModelInput,
691    pub definition: BenchDefinitionInput,
692    pub case: BenchCaseInput,
693    pub artifact_dir: PathBuf,
694}
695
696/// A single Bench case: its load shape and the number of requests to send.
697#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
698#[serde(deny_unknown_fields)]
699pub struct BenchCaseInput {
700    pub load_shape: BenchLoadInput,
701    pub request_count: u32,
702}
703
704/// How a Bench case paces its requests.
705#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
706#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
707pub enum BenchLoadInput {
708    /// A fixed number of in-flight requests.
709    ConcurrencyLimited { concurrency: u32 },
710    /// A target arrival rate, optionally shaped by a burstiness factor.
711    RequestRateLimited {
712        request_rate: f64,
713        burstiness: Option<f64>,
714    },
715    /// All requests issued as fast as possible.
716    UnboundedRequestRate,
717}
718
719/// The terminal outcome a measurement client reports.
720#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
721#[serde(rename_all = "snake_case")]
722pub enum ClientStatus {
723    /// The client completed its measurement successfully.
724    Succeeded,
725    /// The client did not complete successfully.
726    Failed,
727}
728
729/// The result an Eval client writes for the measurement runtime to consume.
730#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
731#[serde(deny_unknown_fields)]
732pub struct EvalClientResult {
733    /// Result envelope version; clients write `1`. The measurement runtime
734    /// rejects an eval result whose version is not `1`.
735    pub schema_version: u32,
736    pub status: ClientStatus,
737    pub metrics: BTreeMap<String, f64>,
738    pub native_command: Vec<String>,
739    pub raw_artifacts: Vec<RawArtifact>,
740    pub error: Option<String>,
741}
742
743/// The result a Bench client writes for the measurement runtime to consume.
744#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
745#[serde(deny_unknown_fields)]
746pub struct BenchClientResult {
747    /// Result envelope version; clients write `1`. The measurement runtime
748    /// rejects a bench result whose version is not `1`.
749    pub schema_version: u32,
750    pub status: ClientStatus,
751    pub completed_requests: u64,
752    pub failed_requests: u64,
753    pub normalization_schema: String,
754    pub metrics: BTreeMap<String, f64>,
755    pub native_command: Vec<String>,
756    pub native_exit_code: Option<i32>,
757    pub raw_artifacts: Vec<RawArtifact>,
758    pub error: Option<String>,
759}
760
761/// A raw output file a client produced, retained as workload evidence.
762#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
763#[serde(deny_unknown_fields)]
764pub struct RawArtifact {
765    pub name: String,
766    pub kind: String,
767    pub path: PathBuf,
768}
769
770/// The schema root aggregating every wire type. It exists to generate one
771/// committed JSON schema (and the Python SDK models); its optional client
772/// fields are never all populated in a single message.
773#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
774#[serde(deny_unknown_fields)]
775pub struct AdapterProtocol {
776    pub request: AdapterRequest,
777    pub response: AdapterResponse,
778    #[serde(default, skip_serializing_if = "Option::is_none")]
779    pub eval_client_request: Option<EvalClientRequest>,
780    #[serde(default, skip_serializing_if = "Option::is_none")]
781    pub eval_client_result: Option<EvalClientResult>,
782    #[serde(default, skip_serializing_if = "Option::is_none")]
783    pub bench_client_request: Option<BenchClientRequest>,
784    #[serde(default, skip_serializing_if = "Option::is_none")]
785    pub bench_client_result: Option<BenchClientResult>,
786}
inferlab_protocol/wire.rs

inferlab_protocol/
wire.rs
