Skip to main content

ml_runtime_inference/
lib.rs

1use bytes::Bytes;
2use ml_runtime_model::ModelReference;
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::{collections::BTreeMap, path::PathBuf, time::Duration};
6
7/// Determines where the runtime attempts to execute a request and whether it may fall back.
8#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
9pub enum ExecutionPolicy {
10    #[default]
11    LocalOnly,
12    RemoteOnly,
13    PreferLocal,
14    PreferRemote,
15    LocalThenRemote,
16    RemoteThenLocal,
17}
18
19#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
20pub struct Tensor {
21    pub shape: Vec<usize>,
22    pub values: Vec<f32>,
23}
24
25impl Tensor {
26    pub fn new(shape: impl Into<Vec<usize>>, values: impl Into<Vec<f32>>) -> Self {
27        Self {
28            shape: shape.into(),
29            values: values.into(),
30        }
31    }
32}
33
34#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
35pub struct Image {
36    pub mime_type: String,
37    pub data: Bytes,
38}
39
40#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
41pub struct Audio {
42    pub mime_type: String,
43    pub data: Bytes,
44}
45
46#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
47pub struct InferenceOptions {
48    pub timeout_ms: Option<u64>,
49    pub max_output_tokens: Option<u64>,
50    pub batch_size: Option<usize>,
51    pub require_local: bool,
52    pub require_remote: bool,
53    pub stream: bool,
54    #[serde(default)]
55    pub execution: ExecutionPolicy,
56}
57
58impl Default for InferenceOptions {
59    fn default() -> Self {
60        Self {
61            timeout_ms: None,
62            max_output_tokens: None,
63            batch_size: None,
64            require_local: false,
65            require_remote: false,
66            stream: false,
67            execution: ExecutionPolicy::LocalOnly,
68        }
69    }
70}
71
72#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
73pub enum Input {
74    Text(String),
75    Tokens(Vec<u32>),
76    Tensor(Tensor),
77    Image(Image),
78    Audio(Audio),
79    Binary(Bytes),
80    Structured(Value),
81}
82
83impl Input {
84    pub fn batch_size(&self) -> usize {
85        match self {
86            Self::Tokens(_) => 1,
87            Self::Tensor(tensor) => tensor.shape.first().copied().unwrap_or(1),
88            _ => 1,
89        }
90    }
91}
92
93impl From<String> for Input {
94    fn from(value: String) -> Self {
95        Self::Text(value)
96    }
97}
98
99impl From<&str> for Input {
100    fn from(value: &str) -> Self {
101        Self::Text(value.to_owned())
102    }
103}
104
105#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
106pub enum Output {
107    Text(String),
108    Tokens(Vec<u32>),
109    Tensor(Tensor),
110    Embedding(Vec<f32>),
111    Structured(Value),
112    Binary(Bytes),
113}
114
115#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
116pub struct ExecutionMetadata {
117    #[serde(default)]
118    pub request_id: Option<String>,
119    pub model: String,
120    pub model_version: Option<String>,
121    /// Execution location or transport selected by this runtime (`local`, `remote`, etc.).
122    pub provider: String,
123    /// Computation implementation reported by the execution target (`onnx`, `cpu`, etc.).
124    pub backend: String,
125    /// Observable target selected by this runtime. For a client this may be remote even when
126    /// `backend` reports the server's local computation backend.
127    pub execution_target: String,
128    pub routing_policy: ExecutionPolicy,
129    pub fallback: bool,
130    pub fallback_from: Option<String>,
131    pub fallback_reason: Option<String>,
132    pub latency_ms: Option<f64>,
133    pub input_tokens: Option<u64>,
134    pub output_tokens: Option<u64>,
135    pub hardware: Option<String>,
136    #[serde(default)]
137    pub cache_hit: bool,
138    #[serde(default)]
139    pub batch_size: usize,
140    #[serde(default)]
141    pub queue_wait_ms: Option<f64>,
142    #[serde(default)]
143    pub model_load_ms: Option<f64>,
144    #[serde(default)]
145    pub execution_ms: Option<f64>,
146    #[serde(default)]
147    pub streaming_requested: bool,
148    #[serde(default)]
149    pub streaming_supported: bool,
150    #[serde(default)]
151    pub output_event_count: usize,
152    #[serde(default)]
153    pub completion_state: Option<String>,
154}
155
156#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
157pub struct InferenceRequest {
158    pub model: ModelReference,
159    pub input: Input,
160    pub options: InferenceOptions,
161}
162
163impl InferenceRequest {
164    /// Creates a request with default local-only execution options.
165    pub fn new(model: impl Into<ModelReference>, input: impl Into<Input>) -> Self {
166        Self {
167            model: model.into(),
168            input: input.into(),
169            options: InferenceOptions::default(),
170        }
171    }
172
173    /// Replaces the request's execution options.
174    pub fn with_options(mut self, options: InferenceOptions) -> Self {
175        self.options = options;
176        self
177    }
178}
179
180#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
181pub struct InferenceResult {
182    pub output: Output,
183    pub metadata: ExecutionMetadata,
184}
185
186impl InferenceResult {
187    pub fn output(&self) -> &Output {
188        &self.output
189    }
190
191    pub fn metadata(&self) -> &ExecutionMetadata {
192        &self.metadata
193    }
194
195    pub fn into_output(self) -> Output {
196        self.output
197    }
198}
199
200#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
201pub struct InferenceChunk {
202    pub output: Output,
203    pub done: bool,
204    pub metadata: Option<ExecutionMetadata>,
205}
206
207pub type InferenceOutput = Output;
208
209/// A model-neutral request for one or more typed decisions about immutable input state.
210#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
211pub struct DecisionRequest {
212    pub state: Value,
213    pub decisions: Vec<DecisionQuestion>,
214}
215
216/// Backend/model execution facts used by the structured-decision planner.
217#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
218pub struct DecisionModelCapabilities {
219    pub backend: String,
220    pub device: DeviceKind,
221    pub model_architecture: ModelArchitecture,
222    pub supports_batching: bool,
223    pub max_batch_size: Option<usize>,
224    pub batch_size: Option<usize>,
225    pub supports_async: bool,
226    /// True only when an in-flight native operation can actually be interrupted.
227    pub supports_cancellation: bool,
228    pub supports_parallel_execution: bool,
229    pub recommended_parallelism: Option<usize>,
230    pub supports_structured_decisions: bool,
231}
232
233impl DecisionModelCapabilities {
234    pub fn conservative(backend: impl Into<String>) -> Self {
235        Self {
236            backend: backend.into(),
237            device: DeviceKind::Unknown,
238            model_architecture: ModelArchitecture::StructuredDecision,
239            supports_batching: false,
240            max_batch_size: Some(1),
241            batch_size: Some(1),
242            supports_async: false,
243            supports_cancellation: false,
244            supports_parallel_execution: false,
245            recommended_parallelism: Some(1),
246            supports_structured_decisions: true,
247        }
248    }
249}
250
251#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
252#[serde(rename_all = "snake_case")]
253pub enum DeviceKind {
254    Cpu,
255    Gpu,
256    NeuralEngine,
257    WebAssembly,
258    Unknown,
259    Other(String),
260}
261
262#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
263#[serde(rename_all = "snake_case")]
264pub enum ModelArchitecture {
265    StructuredDecision,
266    Tensor,
267    Generative,
268    Other(String),
269}
270
271/// Application constraints. These bound execution without prescribing a backend strategy.
272#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
273pub struct DecisionExecutionPolicy {
274    #[serde(default)]
275    pub latency_target: Option<Duration>,
276    #[serde(default)]
277    pub max_parallelism: Option<usize>,
278    #[serde(default)]
279    pub max_batch_size: Option<usize>,
280    #[serde(default = "default_true")]
281    pub allow_parallel: bool,
282    #[serde(default = "default_true")]
283    pub allow_batching: bool,
284    #[serde(default = "default_true")]
285    pub allow_selective_execution: bool,
286}
287
288fn default_true() -> bool {
289    true
290}
291
292impl Default for DecisionExecutionPolicy {
293    fn default() -> Self {
294        Self {
295            latency_target: None,
296            max_parallelism: None,
297            max_batch_size: None,
298            allow_parallel: true,
299            allow_batching: true,
300            allow_selective_execution: true,
301        }
302    }
303}
304
305/// One decision node and the predicates that control whether it is required.
306#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
307pub struct DecisionNode {
308    pub question: DecisionQuestion,
309    #[serde(default)]
310    pub dependencies: Vec<DecisionDependency>,
311}
312
313#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
314pub struct DecisionDependency {
315    pub decision: String,
316    #[serde(default)]
317    pub condition: DecisionDependencyCondition,
318}
319
320#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
321#[serde(tag = "type", content = "value", rename_all = "snake_case")]
322pub enum DecisionDependencyCondition {
323    #[default]
324    Completed,
325    ChoiceEquals(String),
326    NoulEquals(bool),
327    ScoreAtLeast(f64),
328    ScoreAtMost(f64),
329}
330
331#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
332pub struct DecisionGraphRequest {
333    pub state: Value,
334    pub nodes: Vec<DecisionNode>,
335    #[serde(default)]
336    pub policy: DecisionExecutionPolicy,
337}
338
339impl DecisionGraphRequest {
340    pub fn independent(request: DecisionRequest, policy: DecisionExecutionPolicy) -> Self {
341        Self {
342            state: request.state,
343            nodes: request
344                .decisions
345                .into_iter()
346                .map(|question| DecisionNode {
347                    question,
348                    dependencies: Vec::new(),
349                })
350                .collect(),
351            policy,
352        }
353    }
354}
355
356#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
357#[serde(tag = "type", rename_all = "snake_case")]
358pub enum DecisionExecutionStrategy {
359    Single,
360    Sequential,
361    Parallel { concurrency: usize },
362    Batched { batch_size: usize },
363    Selective,
364}
365
366#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
367pub struct DecisionPlanStage {
368    pub nodes: Vec<String>,
369    pub batches: Vec<Vec<String>>,
370}
371
372#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
373pub struct DecisionExecutionPlan {
374    pub strategy: DecisionExecutionStrategy,
375    pub reason: String,
376    pub stages: Vec<DecisionPlanStage>,
377}
378
379#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
380pub struct DecisionExecutionDiagnostics {
381    pub capabilities: DecisionModelCapabilities,
382    pub strategy: DecisionExecutionStrategy,
383    pub reason: String,
384    pub requested_nodes: usize,
385    pub executed_nodes: usize,
386    pub skipped_nodes: usize,
387    pub batches: Vec<Vec<String>>,
388    pub max_concurrency: usize,
389    pub cancelled: bool,
390}
391
392impl DecisionRequest {
393    pub fn new(state: impl Into<Value>, decisions: Vec<DecisionQuestion>) -> Self {
394        Self {
395            state: state.into(),
396            decisions,
397        }
398    }
399}
400
401#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
402pub struct DecisionQuestion {
403    pub name: String,
404    pub instructions: String,
405    pub kind: DecisionType,
406}
407
408#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
409#[serde(tag = "type", rename_all = "snake_case")]
410pub enum DecisionType {
411    Choice {
412        options: Vec<DecisionOption>,
413    },
414    Score {
415        levels: Vec<Value>,
416    },
417    /// Laya's binary yes/no decision head ("noul" in the model contract).
418    Noul {
419        #[serde(default)]
420        false_description: Option<Value>,
421        #[serde(default)]
422        true_description: Option<Value>,
423    },
424}
425
426#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
427pub struct DecisionOption {
428    pub label: String,
429    #[serde(default)]
430    pub description: Option<Value>,
431}
432
433#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
434pub struct DecisionResult {
435    pub model: ModelIdentity,
436    pub backend: String,
437    pub decisions: Vec<TypedDecision>,
438    pub execution: DecisionExecution,
439    pub provenance: DecisionProvenance,
440}
441
442/// Result returned by the capability-aware executor. Flattening preserves the
443/// existing JSON shape while keeping `DecisionResult` source-compatible.
444#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
445pub struct PlannedDecisionResult {
446    #[serde(flatten)]
447    pub result: DecisionResult,
448    pub planning: DecisionExecutionDiagnostics,
449}
450
451impl std::ops::Deref for PlannedDecisionResult {
452    type Target = DecisionResult;
453
454    fn deref(&self) -> &Self::Target {
455        &self.result
456    }
457}
458
459#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
460pub struct ModelIdentity {
461    pub identifier: String,
462    pub revision: Option<String>,
463}
464
465#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
466pub struct TypedDecision {
467    pub name: String,
468    pub kind: String,
469    pub value: DecisionValue,
470    pub probabilities: BTreeMap<String, f64>,
471    pub confidence: f64,
472    pub action_probability: f64,
473}
474
475#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
476#[serde(tag = "type", content = "value", rename_all = "snake_case")]
477pub enum DecisionValue {
478    Choice(String),
479    Score(f64),
480    Noul(bool),
481}
482
483#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
484pub struct DecisionExecution {
485    pub latency: Duration,
486    pub input_tokens: u64,
487    pub output_tokens: u64,
488}
489
490#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
491pub struct DecisionProvenance {
492    pub artifact_path: PathBuf,
493    pub artifact_sha256: String,
494    pub runtime_version: String,
495}
496
497#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
498pub struct ModelDescription {
499    pub identifier: String,
500    pub revision: Option<String>,
501    pub backend: String,
502    pub artifact_path: PathBuf,
503    pub artifact_sha256: String,
504    pub decision_types: Vec<String>,
505}
506
507#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
508pub enum InferenceStreamEvent {
509    Started(ExecutionMetadata),
510    Output(InferenceOutput),
511    Completed(ExecutionMetadata),
512}