sie_sdk/types/
cluster.rs

1//! Model catalogue, health and capacity payloads.
2
3// Wire-mirror types: field names are the API contract itself, and the ones whose
4// meaning is not obvious carry their own doc comment.
5#![allow(missing_docs)]
6
7use std::collections::HashMap;
8
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11
12use super::ModelState;
13
14/// Dimensionality of each output a model can produce.
15#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
16pub struct ModelDims {
17    #[serde(default, skip_serializing_if = "Option::is_none")]
18    pub dense: Option<u32>,
19    #[serde(default, skip_serializing_if = "Option::is_none")]
20    pub sparse: Option<u32>,
21    #[serde(default, skip_serializing_if = "Option::is_none")]
22    pub multivector: Option<u32>,
23}
24
25/// Optional generation features a model supports.
26#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
27pub struct ModelCapabilities {
28    /// Grammar kinds accepted by `generate`: any of `json_schema`, `regex`, `ebnf`.
29    #[serde(default, deserialize_with = "crate::types::null_as_default")]
30    pub grammar: Vec<String>,
31    #[serde(default, deserialize_with = "crate::types::null_as_default")]
32    pub tools: bool,
33    #[serde(default, deserialize_with = "crate::types::null_as_default")]
34    pub lora_adapters: Vec<String>,
35    #[serde(default, deserialize_with = "crate::types::null_as_default")]
36    pub profile_lora_adapters: HashMap<String, Vec<String>>,
37    #[serde(default, deserialize_with = "crate::types::null_as_default")]
38    pub code: bool,
39    #[serde(default, deserialize_with = "crate::types::null_as_default")]
40    pub sql: bool,
41    #[serde(default, deserialize_with = "crate::types::null_as_default")]
42    pub guard: bool,
43}
44
45/// One entry of the model catalogue.
46#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
47pub struct ModelInfo {
48    #[serde(default, deserialize_with = "crate::types::null_as_default")]
49    pub name: String,
50    #[serde(default, deserialize_with = "crate::types::null_as_default")]
51    pub loaded: bool,
52    #[serde(default, deserialize_with = "crate::types::null_as_default")]
53    pub inputs: Vec<String>,
54    #[serde(default, deserialize_with = "crate::types::null_as_default")]
55    pub outputs: Vec<String>,
56    #[serde(default, deserialize_with = "crate::types::null_as_default")]
57    pub dims: ModelDims,
58    /// Lifecycle state on the worker that holds it.
59    #[serde(default, skip_serializing_if = "Option::is_none")]
60    pub state: Option<ModelState>,
61    /// Named profile variants, when the model advertises more than one.
62    #[serde(default, deserialize_with = "crate::types::null_as_default")]
63    pub profiles: HashMap<String, Value>,
64    /// Why the last load attempt failed, when one did.
65    #[serde(default, skip_serializing_if = "Option::is_none")]
66    pub last_error: Option<String>,
67    #[serde(default, skip_serializing_if = "Option::is_none")]
68    pub max_sequence_length: Option<u32>,
69    #[serde(default, skip_serializing_if = "Option::is_none")]
70    pub revision: Option<String>,
71    #[serde(default, deserialize_with = "crate::types::null_as_default")]
72    pub capabilities: ModelCapabilities,
73}
74
75/// Cluster-wide rollup of a model's availability.
76#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
77pub struct ModelSummary {
78    #[serde(default, deserialize_with = "crate::types::null_as_default")]
79    pub name: String,
80    #[serde(default, skip_serializing_if = "Option::is_none")]
81    pub state: Option<ModelState>,
82    #[serde(default, deserialize_with = "crate::types::null_as_default")]
83    pub worker_count: u32,
84    #[serde(default, deserialize_with = "crate::types::null_as_default")]
85    pub gpu_types: Vec<String>,
86    #[serde(default, deserialize_with = "crate::types::null_as_default")]
87    pub total_queue_depth: u64,
88}
89
90/// One worker as the gateway sees it.
91#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
92pub struct WorkerInfo {
93    #[serde(default, deserialize_with = "crate::types::null_as_default")]
94    pub name: String,
95    #[serde(default, deserialize_with = "crate::types::null_as_default")]
96    pub url: String,
97    #[serde(default, deserialize_with = "crate::types::null_as_default")]
98    pub gpu: String,
99    #[serde(default, deserialize_with = "crate::types::null_as_default")]
100    pub gpu_count: u32,
101    #[serde(default, deserialize_with = "crate::types::null_as_default")]
102    pub ready_gpu_slots: u32,
103    #[serde(default, deserialize_with = "crate::types::null_as_default")]
104    pub healthy: bool,
105    #[serde(default, deserialize_with = "crate::types::null_as_default")]
106    pub queue_depth: u64,
107    #[serde(default, deserialize_with = "crate::types::null_as_default")]
108    pub pending_cost: u64,
109    #[serde(default, deserialize_with = "crate::types::null_as_default")]
110    pub inflight_batches: u64,
111    #[serde(default, deserialize_with = "crate::types::null_as_default")]
112    pub loaded_models: Vec<String>,
113    #[serde(default, deserialize_with = "crate::types::null_as_default")]
114    pub memory_used_bytes: u64,
115    #[serde(default, deserialize_with = "crate::types::null_as_default")]
116    pub memory_total_bytes: u64,
117    #[serde(default, skip_serializing_if = "Option::is_none")]
118    pub bundle: Option<String>,
119    #[serde(default, skip_serializing_if = "Option::is_none")]
120    pub bundle_config_hash: Option<String>,
121}
122
123/// Cluster totals.
124#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
125pub struct ClusterSummary {
126    #[serde(default, deserialize_with = "crate::types::null_as_default")]
127    pub worker_count: u32,
128    #[serde(default, deserialize_with = "crate::types::null_as_default")]
129    pub gpu_count: u32,
130    #[serde(default, deserialize_with = "crate::types::null_as_default")]
131    pub models_loaded: u32,
132    #[serde(default, deserialize_with = "crate::types::null_as_default")]
133    pub total_qps: f64,
134}
135
136/// `GET /health` on a gateway or a worker.
137#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
138pub struct HealthResponse {
139    #[serde(default, deserialize_with = "crate::types::null_as_default")]
140    pub status: String,
141    /// `gateway` or `worker`.
142    #[serde(
143        default,
144        deserialize_with = "crate::types::null_as_default",
145        rename = "type"
146    )]
147    pub kind: String,
148    #[serde(default, deserialize_with = "crate::types::null_as_default")]
149    pub cluster: ClusterSummary,
150    #[serde(default, deserialize_with = "crate::types::null_as_default")]
151    pub configured_gpu_types: Vec<String>,
152    #[serde(default, deserialize_with = "crate::types::null_as_default")]
153    pub live_gpu_types: Vec<String>,
154    #[serde(default, deserialize_with = "crate::types::null_as_default")]
155    pub workers: Vec<WorkerInfo>,
156    #[serde(default, deserialize_with = "crate::types::null_as_default")]
157    pub models: Vec<ModelSummary>,
158}
159
160/// Capacity as reported for the whole cluster, or for one GPU type.
161#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
162pub struct CapacityInfo {
163    /// `healthy`, `degraded` or `no_workers`.
164    pub status: String,
165    pub worker_count: u32,
166    pub gpu_count: u32,
167    pub models_loaded: u32,
168    pub configured_gpu_types: Vec<String>,
169    pub live_gpu_types: Vec<String>,
170    pub workers: Vec<WorkerInfo>,
171}
172
173/// Process-level facts about a worker.
174#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
175pub struct ServerInfo {
176    #[serde(default, deserialize_with = "crate::types::null_as_default")]
177    pub version: String,
178    #[serde(default, deserialize_with = "crate::types::null_as_default")]
179    pub uptime_seconds: u64,
180    #[serde(default, deserialize_with = "crate::types::null_as_default")]
181    pub user: String,
182    #[serde(default, deserialize_with = "crate::types::null_as_default")]
183    pub working_dir: String,
184    #[serde(default, deserialize_with = "crate::types::null_as_default")]
185    pub pid: i64,
186}
187
188/// One GPU's live utilization.
189#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
190pub struct GpuMetrics {
191    /// Device handle, such as `cuda:0`.
192    #[serde(default, deserialize_with = "crate::types::null_as_default")]
193    pub device: String,
194    #[serde(default, deserialize_with = "crate::types::null_as_default")]
195    pub name: String,
196    #[serde(default, deserialize_with = "crate::types::null_as_default")]
197    pub gpu_type: String,
198    /// 0 to 100.
199    #[serde(default, deserialize_with = "crate::types::null_as_default")]
200    pub utilization_pct: u32,
201    #[serde(default, deserialize_with = "crate::types::null_as_default")]
202    pub memory_used_bytes: u64,
203    #[serde(default, deserialize_with = "crate::types::null_as_default")]
204    pub memory_total_bytes: u64,
205    #[serde(default, deserialize_with = "crate::types::null_as_default")]
206    pub memory_threshold_pct: f64,
207}
208
209/// How a model was configured on the worker that loaded it.
210#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
211pub struct ModelConfig {
212    #[serde(default)]
213    pub hf_id: Option<String>,
214    #[serde(default)]
215    pub adapter: Option<String>,
216    #[serde(default, deserialize_with = "crate::types::null_as_default")]
217    pub inputs: Vec<String>,
218    #[serde(default, deserialize_with = "crate::types::null_as_default")]
219    pub outputs: Vec<String>,
220    #[serde(default)]
221    pub dims: Option<Value>,
222    #[serde(default)]
223    pub max_sequence_length: Option<u32>,
224    #[serde(default)]
225    pub pooling: Option<String>,
226    #[serde(default, deserialize_with = "crate::types::null_as_default")]
227    pub normalize: bool,
228    #[serde(default)]
229    pub adapter_options_loadtime: Option<Value>,
230    #[serde(default)]
231    pub adapter_options_runtime: Option<Value>,
232}
233
234/// Adaptive batching state. Absent entirely when adaptive batching is off.
235#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
236pub struct AdaptiveBatchingStatus {
237    #[serde(default, deserialize_with = "crate::types::null_as_default")]
238    pub calibrated: bool,
239    #[serde(default)]
240    pub target_p50_ms: Option<f64>,
241    #[serde(default, deserialize_with = "crate::types::null_as_default")]
242    pub wait_ms: f64,
243    #[serde(default, deserialize_with = "crate::types::null_as_default")]
244    pub batch_cost: u64,
245    #[serde(default)]
246    pub p50_ms: Option<f64>,
247    #[serde(default)]
248    pub headroom_ms: Option<f64>,
249    #[serde(default)]
250    pub fill_ratio: Option<f64>,
251}
252
253/// One model's state on one worker.
254#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
255pub struct ModelStatus {
256    #[serde(default, deserialize_with = "crate::types::null_as_default")]
257    pub name: String,
258    #[serde(default, skip_serializing_if = "Option::is_none")]
259    pub state: Option<ModelState>,
260    #[serde(default)]
261    pub device: Option<String>,
262    #[serde(default, deserialize_with = "crate::types::null_as_default")]
263    pub memory_bytes: u64,
264    #[serde(default, deserialize_with = "crate::types::null_as_default")]
265    pub config: ModelConfig,
266    #[serde(default, deserialize_with = "crate::types::null_as_default")]
267    pub queue_depth: u64,
268    #[serde(default, deserialize_with = "crate::types::null_as_default")]
269    pub queue_pending_items: u64,
270    #[serde(default, skip_serializing_if = "Option::is_none")]
271    pub adaptive_batching: Option<AdaptiveBatchingStatus>,
272}
273
274/// A worker's periodic status broadcast.
275#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
276pub struct WorkerStatusMessage {
277    #[serde(default, deserialize_with = "crate::types::null_as_default")]
278    pub timestamp: f64,
279    #[serde(default, deserialize_with = "crate::types::null_as_default")]
280    pub ready: bool,
281    #[serde(default, deserialize_with = "crate::types::null_as_default")]
282    pub name: String,
283    #[serde(default, deserialize_with = "crate::types::null_as_default")]
284    pub machine_profile: String,
285    /// Empty when the worker is not running in queue mode.
286    #[serde(default, deserialize_with = "crate::types::null_as_default")]
287    pub pool_name: String,
288    #[serde(default, deserialize_with = "crate::types::null_as_default")]
289    pub gpu_count: u32,
290    #[serde(default, deserialize_with = "crate::types::null_as_default")]
291    pub bundle: String,
292    /// Empty when the worker has not computed one.
293    #[serde(default, deserialize_with = "crate::types::null_as_default")]
294    pub bundle_config_hash: String,
295    #[serde(default, deserialize_with = "crate::types::null_as_default")]
296    pub loaded_models: Vec<String>,
297    #[serde(default, deserialize_with = "crate::types::null_as_default")]
298    pub server: ServerInfo,
299    #[serde(default, deserialize_with = "crate::types::null_as_default")]
300    pub gpus: Vec<GpuMetrics>,
301    #[serde(default, deserialize_with = "crate::types::null_as_default")]
302    pub models: Vec<ModelStatus>,
303    #[serde(default, deserialize_with = "crate::types::null_as_default")]
304    pub max_batch_requests: u32,
305    /// The worker owns this hysteresis; do not re-derive it from queue depth.
306    #[serde(default, deserialize_with = "crate::types::null_as_default")]
307    pub saturated: bool,
308}
309
310/// A gateway's periodic status broadcast.
311#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
312pub struct ClusterStatusMessage {
313    #[serde(default, deserialize_with = "crate::types::null_as_default")]
314    pub timestamp: f64,
315    #[serde(default, deserialize_with = "crate::types::null_as_default")]
316    pub cluster: ClusterSummary,
317    #[serde(default, deserialize_with = "crate::types::null_as_default")]
318    pub workers: Vec<WorkerInfo>,
319    #[serde(default, deserialize_with = "crate::types::null_as_default")]
320    pub models: Vec<ModelSummary>,
321}
322
323/// One status broadcast, from whichever endpoint the client watched.
324///
325/// The two shapes share no required field, so which one a payload is comes from the
326/// endpoint it arrived on rather than from the payload itself.
327#[derive(Debug, Clone, PartialEq, Serialize)]
328#[serde(untagged)]
329pub enum StatusMessage {
330    /// A gateway reporting on its whole cluster.
331    Cluster(Box<ClusterStatusMessage>),
332    /// A single worker reporting on itself.
333    Worker(Box<WorkerStatusMessage>),
334}
335
336impl StatusMessage {
337    /// The cluster summary, when this message came from a gateway.
338    pub fn cluster(&self) -> Option<&ClusterStatusMessage> {
339        match self {
340            Self::Cluster(message) => Some(message),
341            Self::Worker(_) => None,
342        }
343    }
344
345    /// The worker report, when this message came from a worker.
346    pub fn worker(&self) -> Option<&WorkerStatusMessage> {
347        match self {
348            Self::Worker(message) => Some(message),
349            Self::Cluster(_) => None,
350        }
351    }
352}
sie_sdk/types/cluster.rs

sie_sdk/types/
cluster.rs
