1use bytes::Bytes;
2use ml_runtime_model::ModelReference;
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::{collections::BTreeMap, path::PathBuf, time::Duration};
6
7#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
9pub enum ExecutionPolicy {
10 #[default]
11 LocalOnly,
12 RemoteOnly,
13 PreferLocal,
14 PreferRemote,
15 LocalThenRemote,
16 RemoteThenLocal,
17}
18
19#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
20pub struct Tensor {
21 pub shape: Vec<usize>,
22 pub values: Vec<f32>,
23}
24
25impl Tensor {
26 pub fn new(shape: impl Into<Vec<usize>>, values: impl Into<Vec<f32>>) -> Self {
27 Self {
28 shape: shape.into(),
29 values: values.into(),
30 }
31 }
32}
33
34#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
35pub struct Image {
36 pub mime_type: String,
37 pub data: Bytes,
38}
39
40#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
41pub struct Audio {
42 pub mime_type: String,
43 pub data: Bytes,
44}
45
46#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
47pub struct InferenceOptions {
48 pub timeout_ms: Option<u64>,
49 pub max_output_tokens: Option<u64>,
50 pub batch_size: Option<usize>,
51 pub require_local: bool,
52 pub require_remote: bool,
53 pub stream: bool,
54 #[serde(default)]
55 pub execution: ExecutionPolicy,
56}
57
58impl Default for InferenceOptions {
59 fn default() -> Self {
60 Self {
61 timeout_ms: None,
62 max_output_tokens: None,
63 batch_size: None,
64 require_local: false,
65 require_remote: false,
66 stream: false,
67 execution: ExecutionPolicy::LocalOnly,
68 }
69 }
70}
71
72#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
73pub enum Input {
74 Text(String),
75 Tokens(Vec<u32>),
76 Tensor(Tensor),
77 Image(Image),
78 Audio(Audio),
79 Binary(Bytes),
80 Structured(Value),
81}
82
83impl Input {
84 pub fn batch_size(&self) -> usize {
85 match self {
86 Self::Tokens(_) => 1,
87 Self::Tensor(tensor) => tensor.shape.first().copied().unwrap_or(1),
88 _ => 1,
89 }
90 }
91}
92
93impl From<String> for Input {
94 fn from(value: String) -> Self {
95 Self::Text(value)
96 }
97}
98
99impl From<&str> for Input {
100 fn from(value: &str) -> Self {
101 Self::Text(value.to_owned())
102 }
103}
104
105#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
106pub enum Output {
107 Text(String),
108 Tokens(Vec<u32>),
109 Tensor(Tensor),
110 Embedding(Vec<f32>),
111 Structured(Value),
112 Binary(Bytes),
113}
114
115#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
116pub struct ExecutionMetadata {
117 #[serde(default)]
118 pub request_id: Option<String>,
119 pub model: String,
120 pub model_version: Option<String>,
121 pub provider: String,
123 pub backend: String,
125 pub execution_target: String,
128 pub routing_policy: ExecutionPolicy,
129 pub fallback: bool,
130 pub fallback_from: Option<String>,
131 pub fallback_reason: Option<String>,
132 pub latency_ms: Option<f64>,
133 pub input_tokens: Option<u64>,
134 pub output_tokens: Option<u64>,
135 pub hardware: Option<String>,
136 #[serde(default)]
137 pub cache_hit: bool,
138 #[serde(default)]
139 pub batch_size: usize,
140 #[serde(default)]
141 pub queue_wait_ms: Option<f64>,
142 #[serde(default)]
143 pub model_load_ms: Option<f64>,
144 #[serde(default)]
145 pub execution_ms: Option<f64>,
146 #[serde(default)]
147 pub streaming_requested: bool,
148 #[serde(default)]
149 pub streaming_supported: bool,
150 #[serde(default)]
151 pub output_event_count: usize,
152 #[serde(default)]
153 pub completion_state: Option<String>,
154}
155
156#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
157pub struct InferenceRequest {
158 pub model: ModelReference,
159 pub input: Input,
160 pub options: InferenceOptions,
161}
162
163impl InferenceRequest {
164 pub fn new(model: impl Into<ModelReference>, input: impl Into<Input>) -> Self {
166 Self {
167 model: model.into(),
168 input: input.into(),
169 options: InferenceOptions::default(),
170 }
171 }
172
173 pub fn with_options(mut self, options: InferenceOptions) -> Self {
175 self.options = options;
176 self
177 }
178}
179
180#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
181pub struct InferenceResult {
182 pub output: Output,
183 pub metadata: ExecutionMetadata,
184}
185
186impl InferenceResult {
187 pub fn output(&self) -> &Output {
188 &self.output
189 }
190
191 pub fn metadata(&self) -> &ExecutionMetadata {
192 &self.metadata
193 }
194
195 pub fn into_output(self) -> Output {
196 self.output
197 }
198}
199
200#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
201pub struct InferenceChunk {
202 pub output: Output,
203 pub done: bool,
204 pub metadata: Option<ExecutionMetadata>,
205}
206
207pub type InferenceOutput = Output;
208
209#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
211pub struct DecisionRequest {
212 pub state: Value,
213 pub decisions: Vec<DecisionQuestion>,
214}
215
216#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
218pub struct DecisionModelCapabilities {
219 pub backend: String,
220 pub device: DeviceKind,
221 pub model_architecture: ModelArchitecture,
222 pub supports_batching: bool,
223 pub max_batch_size: Option<usize>,
224 pub batch_size: Option<usize>,
225 pub supports_async: bool,
226 pub supports_cancellation: bool,
228 pub supports_parallel_execution: bool,
229 pub recommended_parallelism: Option<usize>,
230 pub supports_structured_decisions: bool,
231}
232
233impl DecisionModelCapabilities {
234 pub fn conservative(backend: impl Into<String>) -> Self {
235 Self {
236 backend: backend.into(),
237 device: DeviceKind::Unknown,
238 model_architecture: ModelArchitecture::StructuredDecision,
239 supports_batching: false,
240 max_batch_size: Some(1),
241 batch_size: Some(1),
242 supports_async: false,
243 supports_cancellation: false,
244 supports_parallel_execution: false,
245 recommended_parallelism: Some(1),
246 supports_structured_decisions: true,
247 }
248 }
249}
250
251#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
252#[serde(rename_all = "snake_case")]
253pub enum DeviceKind {
254 Cpu,
255 Gpu,
256 NeuralEngine,
257 WebAssembly,
258 Unknown,
259 Other(String),
260}
261
262#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
263#[serde(rename_all = "snake_case")]
264pub enum ModelArchitecture {
265 StructuredDecision,
266 Tensor,
267 Generative,
268 Other(String),
269}
270
271#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
273pub struct DecisionExecutionPolicy {
274 #[serde(default)]
275 pub latency_target: Option<Duration>,
276 #[serde(default)]
277 pub max_parallelism: Option<usize>,
278 #[serde(default)]
279 pub max_batch_size: Option<usize>,
280 #[serde(default = "default_true")]
281 pub allow_parallel: bool,
282 #[serde(default = "default_true")]
283 pub allow_batching: bool,
284 #[serde(default = "default_true")]
285 pub allow_selective_execution: bool,
286}
287
288fn default_true() -> bool {
289 true
290}
291
292impl Default for DecisionExecutionPolicy {
293 fn default() -> Self {
294 Self {
295 latency_target: None,
296 max_parallelism: None,
297 max_batch_size: None,
298 allow_parallel: true,
299 allow_batching: true,
300 allow_selective_execution: true,
301 }
302 }
303}
304
305#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
307pub struct DecisionNode {
308 pub question: DecisionQuestion,
309 #[serde(default)]
310 pub dependencies: Vec<DecisionDependency>,
311}
312
313#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
314pub struct DecisionDependency {
315 pub decision: String,
316 #[serde(default)]
317 pub condition: DecisionDependencyCondition,
318}
319
320#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
321#[serde(tag = "type", content = "value", rename_all = "snake_case")]
322pub enum DecisionDependencyCondition {
323 #[default]
324 Completed,
325 ChoiceEquals(String),
326 NoulEquals(bool),
327 ScoreAtLeast(f64),
328 ScoreAtMost(f64),
329}
330
331#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
332pub struct DecisionGraphRequest {
333 pub state: Value,
334 pub nodes: Vec<DecisionNode>,
335 #[serde(default)]
336 pub policy: DecisionExecutionPolicy,
337}
338
339impl DecisionGraphRequest {
340 pub fn independent(request: DecisionRequest, policy: DecisionExecutionPolicy) -> Self {
341 Self {
342 state: request.state,
343 nodes: request
344 .decisions
345 .into_iter()
346 .map(|question| DecisionNode {
347 question,
348 dependencies: Vec::new(),
349 })
350 .collect(),
351 policy,
352 }
353 }
354}
355
356#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
357#[serde(tag = "type", rename_all = "snake_case")]
358pub enum DecisionExecutionStrategy {
359 Single,
360 Sequential,
361 Parallel { concurrency: usize },
362 Batched { batch_size: usize },
363 Selective,
364}
365
366#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
367pub struct DecisionPlanStage {
368 pub nodes: Vec<String>,
369 pub batches: Vec<Vec<String>>,
370}
371
372#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
373pub struct DecisionExecutionPlan {
374 pub strategy: DecisionExecutionStrategy,
375 pub reason: String,
376 pub stages: Vec<DecisionPlanStage>,
377}
378
379#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
380pub struct DecisionExecutionDiagnostics {
381 pub capabilities: DecisionModelCapabilities,
382 pub strategy: DecisionExecutionStrategy,
383 pub reason: String,
384 pub requested_nodes: usize,
385 pub executed_nodes: usize,
386 pub skipped_nodes: usize,
387 pub batches: Vec<Vec<String>>,
388 pub max_concurrency: usize,
389 pub cancelled: bool,
390}
391
392impl DecisionRequest {
393 pub fn new(state: impl Into<Value>, decisions: Vec<DecisionQuestion>) -> Self {
394 Self {
395 state: state.into(),
396 decisions,
397 }
398 }
399}
400
401#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
402pub struct DecisionQuestion {
403 pub name: String,
404 pub instructions: String,
405 pub kind: DecisionType,
406}
407
408#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
409#[serde(tag = "type", rename_all = "snake_case")]
410pub enum DecisionType {
411 Choice {
412 options: Vec<DecisionOption>,
413 },
414 Score {
415 levels: Vec<Value>,
416 },
417 Noul {
419 #[serde(default)]
420 false_description: Option<Value>,
421 #[serde(default)]
422 true_description: Option<Value>,
423 },
424}
425
426#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
427pub struct DecisionOption {
428 pub label: String,
429 #[serde(default)]
430 pub description: Option<Value>,
431}
432
433#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
434pub struct DecisionResult {
435 pub model: ModelIdentity,
436 pub backend: String,
437 pub decisions: Vec<TypedDecision>,
438 pub execution: DecisionExecution,
439 pub provenance: DecisionProvenance,
440}
441
442#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
445pub struct PlannedDecisionResult {
446 #[serde(flatten)]
447 pub result: DecisionResult,
448 pub planning: DecisionExecutionDiagnostics,
449}
450
451impl std::ops::Deref for PlannedDecisionResult {
452 type Target = DecisionResult;
453
454 fn deref(&self) -> &Self::Target {
455 &self.result
456 }
457}
458
459#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
460pub struct ModelIdentity {
461 pub identifier: String,
462 pub revision: Option<String>,
463}
464
465#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
466pub struct TypedDecision {
467 pub name: String,
468 pub kind: String,
469 pub value: DecisionValue,
470 pub probabilities: BTreeMap<String, f64>,
471 pub confidence: f64,
472 pub action_probability: f64,
473}
474
475#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
476#[serde(tag = "type", content = "value", rename_all = "snake_case")]
477pub enum DecisionValue {
478 Choice(String),
479 Score(f64),
480 Noul(bool),
481}
482
483#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
484pub struct DecisionExecution {
485 pub latency: Duration,
486 pub input_tokens: u64,
487 pub output_tokens: u64,
488}
489
490#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
491pub struct DecisionProvenance {
492 pub artifact_path: PathBuf,
493 pub artifact_sha256: String,
494 pub runtime_version: String,
495}
496
497#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
498pub struct ModelDescription {
499 pub identifier: String,
500 pub revision: Option<String>,
501 pub backend: String,
502 pub artifact_path: PathBuf,
503 pub artifact_sha256: String,
504 pub decision_types: Vec<String>,
505}
506
507#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
508pub enum InferenceStreamEvent {
509 Started(ExecutionMetadata),
510 Output(InferenceOutput),
511 Completed(ExecutionMetadata),
512}