inferlab_protocol/
wire.rs

1//! The versioned wire types of the framework integration protocol
2//! ([[RFC-0006:C-INTEGRATIONS]]): the one-shot stdin/stdout JSON contract for
3//! the plan/render serve operations, plus the client request/result surfaces
4//! the release-owned Eval and Bench measurement runtimes exchange with their
5//! clients. [`AdapterProtocol`] is the schema root from which the committed
6//! JSON schema and the Python SDK models are generated.
7
8use schemars::JsonSchema;
9use serde::{Deserialize, Serialize};
10use std::collections::BTreeMap;
11use std::path::PathBuf;
12
13/// The adapter protocol version an integration speaks. The only accepted value
14/// is `3` (serialized as the string `"3"`); a mismatch is rejected before
15/// lowering.
16#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
17pub enum ProtocolVersion {
18    /// Protocol version 3.
19    #[serde(rename = "3")]
20    V3,
21}
22
23/// The one JSON request an integration reads from stdin, tagged by the
24/// requested operation.
25#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
26#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
27pub enum AdapterRequest {
28    /// Plan a serve topology: declare roles, per-replica requirements, links,
29    /// and endpoint requirements from the requested shape.
30    PlanServe {
31        protocol_version: ProtocolVersion,
32        input: PlanServeInput,
33    },
34    /// Render final process invocations for a planned topology, given the
35    /// control plane's concrete process allocations.
36    RenderServe {
37        protocol_version: ProtocolVersion,
38        input: RenderServeInput,
39    },
40}
41
42impl AdapterRequest {
43    /// The protocol version carried by this request, regardless of operation.
44    #[must_use]
45    pub const fn protocol_version(&self) -> ProtocolVersion {
46        match self {
47            Self::PlanServe {
48                protocol_version, ..
49            }
50            | Self::RenderServe {
51                protocol_version, ..
52            } => *protocol_version,
53        }
54    }
55}
56
57/// The requested serve shape a `PlanServe` operation lowers into a topology.
58#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
59#[serde(deny_unknown_fields)]
60pub struct PlanServeInput {
61    pub model: ServeModelInput,
62    pub topology: ServeTopology,
63    pub routing_backend: String,
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub kv_transfer: Option<KvTransferMechanism>,
66    pub parallelism: Parallelism,
67    pub settings: BTreeMap<String, SettingValue>,
68    pub roles: Vec<ServeRoleInput>,
69    #[serde(default)]
70    pub profiling: bool,
71}
72
73/// The planned topology plus the control plane's concrete allocations that a
74/// `RenderServe` operation turns into final process invocations.
75#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
76#[serde(deny_unknown_fields)]
77pub struct RenderServeInput {
78    pub model: ServeModelInput,
79    pub topology: ServeTopology,
80    pub routing_backend: String,
81    #[serde(default, skip_serializing_if = "Option::is_none")]
82    pub kv_transfer: Option<KvTransferMechanism>,
83    pub parallelism: Parallelism,
84    pub settings: BTreeMap<String, SettingValue>,
85    pub roles: Vec<ServeRoleResult>,
86    pub links: Vec<ServeRoleLink>,
87    pub allocations: Vec<ServeProcessAllocation>,
88    #[serde(default)]
89    pub profiling: bool,
90}
91
92/// The serving deployment topology.
93#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
94#[serde(rename_all = "snake_case")]
95pub enum ServeTopology {
96    /// One aggregated serving role.
97    Single,
98    /// Disaggregated prefill and decode roles.
99    PrefillDecode,
100}
101
102/// The logical role a replica plays in a serve topology.
103#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
104#[serde(rename_all = "snake_case")]
105pub enum ServeRoleKind {
106    /// Aggregated serving role of a single topology.
107    Serve,
108    /// Prefill role of a disaggregated topology.
109    Prefill,
110    /// Decode role of a disaggregated topology.
111    Decode,
112    /// Request-routing role that does not execute model inference.
113    Router,
114}
115
116/// The KV-transfer mechanism connecting prefill and decode.
117#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
118#[serde(rename_all = "snake_case")]
119pub enum KvTransferMechanism {
120    /// Mooncake KV-cache transfer.
121    Mooncake,
122    /// NIXL KV-cache transfer.
123    Nixl,
124}
125
126/// A requested serving role: its identity, kind, replica cardinality, and
127/// declared (not-yet-completed) parallelism and settings.
128#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
129#[serde(deny_unknown_fields)]
130pub struct ServeRoleInput {
131    pub id: String,
132    pub kind: ServeRoleKind,
133    pub replica_count: u32,
134    pub parallelism: Parallelism,
135    pub settings: BTreeMap<String, SettingValue>,
136}
137
138/// A role as the integration resolved it: preserved identity and cardinality
139/// with the complete effective settings and parallelism.
140#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
141#[serde(deny_unknown_fields)]
142pub struct ServeRoleResult {
143    pub id: String,
144    pub kind: ServeRoleKind,
145    pub replica_count: u32,
146    pub effective_settings: BTreeMap<String, SettingValue>,
147    pub effective_parallelism: Parallelism,
148}
149
150/// Framework-neutral component-aware parallelism ([[RFC-0003:C-SERVE-PARALLELISM]]).
151/// Every component is optional so an operator can override one without
152/// repeating the rest; omitted components are filled by the integration into a
153/// complete effective shape.
154#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
155#[serde(default, deny_unknown_fields)]
156pub struct Parallelism {
157    /// Outer deployment parallelism shared by attention and experts.
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub outer: Option<ParallelismOuter>,
160    /// Attention-block parallelism.
161    #[serde(skip_serializing_if = "Option::is_none")]
162    pub attention: Option<ParallelismAttention>,
163    /// MoE expert parallelism.
164    #[serde(skip_serializing_if = "Option::is_none")]
165    pub experts: Option<ParallelismExperts>,
166}
167
168impl Parallelism {
169    /// Overlay the components present in `other` onto `self`, leaving
170    /// components `other` omits untouched (the per-component precedence merge).
171    pub fn merge_from(&mut self, other: &Self) {
172        if let Some(other) = &other.outer {
173            self.outer.get_or_insert_default().merge_from(other);
174        }
175        if let Some(other) = &other.attention {
176            self.attention.get_or_insert_default().merge_from(other);
177        }
178        if let Some(other) = &other.experts {
179            self.experts.get_or_insert_default().merge_from(other);
180        }
181    }
182}
183
184/// Outer deployment parallelism: tensor and pipeline degrees.
185#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
186#[serde(default, deny_unknown_fields)]
187pub struct ParallelismOuter {
188    #[schemars(range(min = 1))]
189    pub tensor_parallel_size: Option<u32>,
190    #[schemars(range(min = 1))]
191    pub pipeline_parallel_size: Option<u32>,
192}
193
194impl ParallelismOuter {
195    fn merge_from(&mut self, other: &Self) {
196        if other.tensor_parallel_size.is_some() {
197            self.tensor_parallel_size = other.tensor_parallel_size;
198        }
199        if other.pipeline_parallel_size.is_some() {
200            self.pipeline_parallel_size = other.pipeline_parallel_size;
201        }
202    }
203}
204
205/// Attention-block parallelism: tensor, data, and context degrees.
206#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
207#[serde(default, deny_unknown_fields)]
208pub struct ParallelismAttention {
209    #[schemars(range(min = 1))]
210    pub tensor_parallel_size: Option<u32>,
211    #[schemars(range(min = 1))]
212    pub data_parallel_size: Option<u32>,
213    #[schemars(range(min = 1))]
214    pub context_parallel_size: Option<u32>,
215}
216
217impl ParallelismAttention {
218    fn merge_from(&mut self, other: &Self) {
219        if other.tensor_parallel_size.is_some() {
220            self.tensor_parallel_size = other.tensor_parallel_size;
221        }
222        if other.data_parallel_size.is_some() {
223            self.data_parallel_size = other.data_parallel_size;
224        }
225        if other.context_parallel_size.is_some() {
226            self.context_parallel_size = other.context_parallel_size;
227        }
228    }
229}
230
231/// MoE expert parallelism: tensor, data, expert, and dense-tensor degrees.
232#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
233#[serde(default, deny_unknown_fields)]
234pub struct ParallelismExperts {
235    #[schemars(range(min = 1))]
236    pub tensor_parallel_size: Option<u32>,
237    #[schemars(range(min = 1))]
238    pub data_parallel_size: Option<u32>,
239    #[schemars(range(min = 1))]
240    pub expert_parallel_size: Option<u32>,
241    #[schemars(range(min = 1))]
242    pub dense_tensor_parallel_size: Option<u32>,
243}
244
245impl ParallelismExperts {
246    fn merge_from(&mut self, other: &Self) {
247        if other.tensor_parallel_size.is_some() {
248            self.tensor_parallel_size = other.tensor_parallel_size;
249        }
250        if other.data_parallel_size.is_some() {
251            self.data_parallel_size = other.data_parallel_size;
252        }
253        if other.expert_parallel_size.is_some() {
254            self.expert_parallel_size = other.expert_parallel_size;
255        }
256        if other.dense_tensor_parallel_size.is_some() {
257            self.dense_tensor_parallel_size = other.dense_tensor_parallel_size;
258        }
259    }
260}
261
262/// The model to serve: its resolved locator and the name it is served under.
263#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
264#[serde(deny_unknown_fields)]
265pub struct ServeModelInput {
266    pub locator: String,
267    pub served_name: String,
268}
269
270/// A concrete host/port endpoint the control plane allocated for a process.
271#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
272#[serde(deny_unknown_fields)]
273pub struct EndpointAssignment {
274    pub host: String,
275    pub port: u16,
276}
277
278/// The public workload endpoint an Eval or Bench client connects to.
279#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
280#[serde(deny_unknown_fields)]
281pub struct ClientEndpointInput {
282    pub protocol: EndpointProtocol,
283    pub host: String,
284    pub port: u16,
285    pub api_path: String,
286}
287
288/// The measurement an Eval client runs against the workload endpoint.
289#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
290#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
291pub enum EvalDefinitionInput {
292    /// A single-prompt liveness check.
293    #[serde(rename = "openai_smoke")]
294    OpenAiSmoke {
295        prompt: String,
296        max_tokens: u32,
297        timeout_seconds: u64,
298    },
299    /// An lm-eval task run with a pass threshold on the chosen metric.
300    LmEval {
301        task: String,
302        dataset: Option<String>,
303        split: Option<String>,
304        limit: Option<u32>,
305        few_shot: Option<u32>,
306        seed: Option<u64>,
307        max_tokens: Option<u32>,
308        concurrency: Option<u32>,
309        metric: String,
310        threshold: f64,
311        timeout_seconds: u64,
312    },
313}
314
315/// The workload shape a Bench client drives, shared across its load cases.
316#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
317#[serde(deny_unknown_fields)]
318pub struct BenchDefinitionInput {
319    pub input_tokens: u32,
320    pub output_tokens: u32,
321    pub seed: u64,
322    pub temperature: f64,
323    pub timeout_seconds: u64,
324    #[serde(default)]
325    pub reset_prefix_cache: bool,
326}
327
328/// A framework-specific server setting value carried as structured JSON data
329/// (never a pre-rendered shell fragment) across the integration boundary.
330#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
331#[serde(untagged)]
332pub enum SettingValue {
333    /// A JSON boolean.
334    Bool(bool),
335    /// A JSON integer.
336    Integer(i64),
337    /// A JSON floating-point number.
338    Float(f64),
339    /// A JSON string.
340    String(String),
341    /// A JSON array of setting values.
342    Array(Vec<SettingValue>),
343    /// A JSON object of named setting values.
344    Object(BTreeMap<String, SettingValue>),
345}
346
347/// The one JSON response an integration writes to stdout, tagged by outcome.
348#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
349#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
350pub enum AdapterResponse {
351    /// The operation succeeded and carries its result.
352    Ok {
353        protocol_version: ProtocolVersion,
354        result: Box<AdapterResult>,
355    },
356    /// The operation was rejected with a structured error.
357    Error {
358        protocol_version: ProtocolVersion,
359        error: AdapterError,
360    },
361}
362
363impl AdapterResponse {
364    /// The protocol version carried by this response, regardless of outcome.
365    #[must_use]
366    pub const fn protocol_version(&self) -> ProtocolVersion {
367        match self {
368            Self::Ok {
369                protocol_version, ..
370            }
371            | Self::Error {
372                protocol_version, ..
373            } => *protocol_version,
374        }
375    }
376}
377
378/// The successful result of an [`AdapterRequest`], tagged by the operation it
379/// answers.
380#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
381#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)]
382pub enum AdapterResult {
383    /// The planned topology from a `PlanServe` request.
384    PlanServe { output: Box<PlanServeResult> },
385    /// The rendered process invocations from a `RenderServe` request.
386    RenderServe { output: Box<RenderServeResult> },
387}
388
389/// The lowered topology returned by a `PlanServe`: the effective server-level
390/// shape, per-role resolution, whole-replica requirements, role links, and the
391/// public and per-role endpoint requirements the control plane then allocates.
392#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
393#[serde(deny_unknown_fields)]
394pub struct PlanServeResult {
395    pub integration: IntegrationIdentity,
396    pub effective_settings: BTreeMap<String, SettingValue>,
397    pub effective_parallelism: Parallelism,
398    pub roles: Vec<ServeRoleResult>,
399    pub replicas: Vec<ServeReplicaRequirement>,
400    pub links: Vec<ServeRoleLink>,
401    pub public_endpoint: PublicEndpointRequirement,
402    pub endpoint: EndpointRequirement,
403}
404
405/// A whole-replica resource and readiness requirement the integration declares
406/// without choosing placement, ranks, or concrete endpoints.
407#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
408#[serde(deny_unknown_fields)]
409pub struct ServeReplicaRequirement {
410    pub id: String,
411    pub role_id: String,
412    pub replica_index: u32,
413    pub accelerator_count: u32,
414    pub ports: Vec<String>,
415    pub primary_ports: Vec<String>,
416    pub primary_readiness: ReadinessProbe,
417    pub worker_readiness: ReadinessProbe,
418    #[serde(default, skip_serializing_if = "Option::is_none")]
419    pub capture_target: Option<CaptureTargetRequirement>,
420}
421
422/// One concrete process the control plane placed, supplied to `RenderServe`:
423/// its identity, rank, machine, devices, model locator, and allocated
424/// endpoints and named ports.
425#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
426#[serde(deny_unknown_fields)]
427pub struct ServeProcessAllocation {
428    pub process_id: String,
429    pub role_id: String,
430    pub replica_id: String,
431    pub replica_index: u32,
432    pub rank: u32,
433    pub machine_id: String,
434    pub model_locator: String,
435    pub runtime_cache_root: String,
436    pub devices: Vec<u32>,
437    pub endpoint: EndpointAssignment,
438    pub ports: BTreeMap<String, EndpointAssignment>,
439}
440
441/// A directed link between serve roles the integration declares as part of the
442/// topology.
443#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
444#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
445pub enum ServeRoleLink {
446    /// The source role routes requests to the target roles.
447    RequestRouting {
448        source: String,
449        targets: Vec<String>,
450    },
451    /// KV cache is transferred from source to target over `mechanism`.
452    KvTransfer {
453        source: String,
454        target: String,
455        mechanism: KvTransferMechanism,
456    },
457    /// The source discovers the target through a bootstrap port.
458    Bootstrap {
459        source: String,
460        target: String,
461        port: String,
462    },
463    /// The source and target exchange out-of-band data over a side-channel port.
464    SideChannel {
465        source: String,
466        target: String,
467        port: String,
468    },
469}
470
471/// How the topology's public workload endpoint is exposed.
472#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
473#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
474pub enum PublicEndpointRequirement {
475    /// The endpoint is served directly by a single replica.
476    Replica { replica_id: String },
477    /// The endpoint is fronted by an Inferlab-owned built-in proxy routing
478    /// across the named prefill and decode roles.
479    BuiltinProxy {
480        process_id: String,
481        role_id: String,
482        prefill_role: String,
483        decode_role: String,
484        readiness: ReadinessProbe,
485    },
486}
487
488/// Marks a replica as a profiling capture target and carries its window
489/// control ([[RFC-0004:C-WORKLOAD-PROFILING]]).
490#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
491#[serde(deny_unknown_fields)]
492pub struct CaptureTargetRequirement {
493    pub control: CaptureControlRequirement,
494}
495
496/// The HTTP paths a capture target exposes to open and close a capture window.
497#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
498#[serde(deny_unknown_fields)]
499pub struct CaptureControlRequirement {
500    pub start_path: String,
501    pub stop_path: String,
502}
503
504/// The final process invocations returned by a `RenderServe`, one per supplied
505/// allocation.
506#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
507#[serde(deny_unknown_fields)]
508pub struct RenderServeResult {
509    pub integration: IntegrationIdentity,
510    pub processes: Vec<RenderedServeProcess>,
511}
512
513/// A rendered process bound to the allocation `id` it was produced for.
514#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
515#[serde(deny_unknown_fields)]
516pub struct RenderedServeProcess {
517    pub id: String,
518    pub process: ProcessSpec,
519}
520
521/// An HTTP action (method and path) invoked against the workload endpoint.
522#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
523#[serde(deny_unknown_fields)]
524pub struct HttpActionSpec {
525    pub method: HttpMethod,
526    pub path: String,
527}
528
529/// The HTTP method of an [`HttpActionSpec`].
530#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
531#[serde(rename_all = "snake_case")]
532pub enum HttpMethod {
533    /// HTTP POST.
534    Post,
535}
536
537/// The integration's identity recorded on its results: adapter id, adapter
538/// version, and the framework it lowers to.
539#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
540#[serde(deny_unknown_fields)]
541pub struct IntegrationIdentity {
542    pub adapter_id: String,
543    pub adapter_version: String,
544    pub framework: String,
545}
546
547/// A launchable process: its argument vector and environment.
548#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
549#[serde(deny_unknown_fields)]
550pub struct ProcessSpec {
551    pub argv: Vec<String>,
552    pub env: BTreeMap<String, String>,
553}
554
555/// How the control plane decides a process is ready.
556#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
557#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
558pub enum ReadinessProbe {
559    /// Ready when an HTTP GET of `path` succeeds.
560    Http { path: String },
561    /// Ready as soon as the process is alive.
562    ProcessAlive,
563}
564
565/// The workload endpoint's protocol and API path, plus an optional
566/// prefix-cache-reset action a Bench case can invoke between runs.
567#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
568#[serde(deny_unknown_fields)]
569pub struct EndpointRequirement {
570    pub protocol: EndpointProtocol,
571    pub api_path: String,
572    #[serde(default, skip_serializing_if = "Option::is_none")]
573    pub prefix_cache_reset: Option<HttpActionSpec>,
574}
575
576/// The application protocol a workload endpoint speaks.
577#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
578#[serde(rename_all = "snake_case")]
579pub enum EndpointProtocol {
580    /// HTTP.
581    Http,
582}
583
584/// A structured rejection an integration returns in an [`AdapterResponse::Error`].
585#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
586#[serde(deny_unknown_fields)]
587pub struct AdapterError {
588    pub code: AdapterErrorCode,
589    pub message: String,
590}
591
592/// Machine-readable failure category an adapter reports in an [`AdapterError`].
593#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
594#[serde(rename_all = "snake_case")]
595pub enum AdapterErrorCode {
596    /// The request was malformed or missing required fields.
597    InvalidRequest,
598    /// The request's protocol version is not accepted.
599    UnsupportedProtocolVersion,
600    /// A framework setting was unknown or invalid.
601    InvalidSettings,
602    /// An unexpected internal failure occurred in the integration.
603    Internal,
604    /// The requested operation is not supported by this integration.
605    UnsupportedOperation,
606}
607
608/// The request the Eval measurement runtime passes to its client: the endpoint
609/// to hit, the model, the eval definition, and where to write artifacts.
610#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
611#[serde(deny_unknown_fields)]
612pub struct EvalClientRequest {
613    pub protocol_version: ProtocolVersion,
614    pub endpoint: ClientEndpointInput,
615    pub model: ServeModelInput,
616    pub definition: EvalDefinitionInput,
617    pub artifact_dir: PathBuf,
618}
619
620/// The request the Bench measurement runtime passes to its client: the
621/// endpoint, model, bench definition, the load case to run, and the artifact
622/// directory.
623#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
624#[serde(deny_unknown_fields)]
625pub struct BenchClientRequest {
626    pub protocol_version: ProtocolVersion,
627    pub endpoint: ClientEndpointInput,
628    pub model: ServeModelInput,
629    pub definition: BenchDefinitionInput,
630    pub case: BenchCaseInput,
631    pub artifact_dir: PathBuf,
632}
633
634/// A single Bench case: its load shape and the number of requests to send.
635#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
636#[serde(deny_unknown_fields)]
637pub struct BenchCaseInput {
638    pub load_shape: BenchLoadInput,
639    pub request_count: u32,
640}
641
642/// How a Bench case paces its requests.
643#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
644#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
645pub enum BenchLoadInput {
646    /// A fixed number of in-flight requests.
647    ConcurrencyLimited { concurrency: u32 },
648    /// A target arrival rate, optionally shaped by a burstiness factor.
649    RequestRateLimited {
650        request_rate: f64,
651        burstiness: Option<f64>,
652    },
653    /// All requests issued as fast as possible.
654    UnboundedRequestRate,
655}
656
657/// The terminal outcome a measurement client reports.
658#[derive(Clone, Copy, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
659#[serde(rename_all = "snake_case")]
660pub enum ClientStatus {
661    /// The client completed its measurement successfully.
662    Succeeded,
663    /// The client did not complete successfully.
664    Failed,
665}
666
667/// The result an Eval client writes for the measurement runtime to consume.
668#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
669#[serde(deny_unknown_fields)]
670pub struct EvalClientResult {
671    /// Result envelope version; clients write `1`. The measurement runtime
672    /// rejects an eval result whose version is not `1`.
673    pub schema_version: u32,
674    pub status: ClientStatus,
675    pub metrics: BTreeMap<String, f64>,
676    pub native_command: Vec<String>,
677    pub raw_artifacts: Vec<RawArtifact>,
678    pub error: Option<String>,
679}
680
681/// The result a Bench client writes for the measurement runtime to consume.
682#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
683#[serde(deny_unknown_fields)]
684pub struct BenchClientResult {
685    /// Result envelope version; clients write `1`. The measurement runtime
686    /// rejects a bench result whose version is not `1`.
687    pub schema_version: u32,
688    pub status: ClientStatus,
689    pub completed_requests: u64,
690    pub failed_requests: u64,
691    pub normalization_schema: String,
692    pub metrics: BTreeMap<String, f64>,
693    pub native_command: Vec<String>,
694    pub native_exit_code: Option<i32>,
695    pub raw_artifacts: Vec<RawArtifact>,
696    pub error: Option<String>,
697}
698
699/// A raw output file a client produced, retained as workload evidence.
700#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
701#[serde(deny_unknown_fields)]
702pub struct RawArtifact {
703    pub name: String,
704    pub kind: String,
705    pub path: PathBuf,
706}
707
708/// The schema root aggregating every wire type. It exists to generate one
709/// committed JSON schema (and the Python SDK models); its optional client
710/// fields are never all populated in a single message.
711#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)]
712#[serde(deny_unknown_fields)]
713pub struct AdapterProtocol {
714    pub request: AdapterRequest,
715    pub response: AdapterResponse,
716    #[serde(default, skip_serializing_if = "Option::is_none")]
717    pub eval_client_request: Option<EvalClientRequest>,
718    #[serde(default, skip_serializing_if = "Option::is_none")]
719    pub eval_client_result: Option<EvalClientResult>,
720    #[serde(default, skip_serializing_if = "Option::is_none")]
721    pub bench_client_request: Option<BenchClientRequest>,
722    #[serde(default, skip_serializing_if = "Option::is_none")]
723    pub bench_client_result: Option<BenchClientResult>,
724}
inferlab_protocol/wire.rs

inferlab_protocol/
wire.rs
