Skip to main content

ml_runtime_inference/
lib.rs

1use bytes::Bytes;
2use ml_runtime_model::ModelReference;
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::{collections::BTreeMap, path::PathBuf, time::Duration};
6
7/// Determines where the runtime attempts to execute a request and whether it may fall back.
8#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
9pub enum ExecutionPolicy {
10    #[default]
11    LocalOnly,
12    RemoteOnly,
13    PreferLocal,
14    PreferRemote,
15    LocalThenRemote,
16    RemoteThenLocal,
17}
18
19#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
20pub struct Tensor {
21    pub shape: Vec<usize>,
22    pub values: Vec<f32>,
23}
24
25impl Tensor {
26    pub fn new(shape: impl Into<Vec<usize>>, values: impl Into<Vec<f32>>) -> Self {
27        Self {
28            shape: shape.into(),
29            values: values.into(),
30        }
31    }
32}
33
34#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
35pub struct Image {
36    pub mime_type: String,
37    pub data: Bytes,
38}
39
40#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
41pub struct Audio {
42    pub mime_type: String,
43    pub data: Bytes,
44}
45
46#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
47pub struct InferenceOptions {
48    pub timeout_ms: Option<u64>,
49    pub max_output_tokens: Option<u64>,
50    pub batch_size: Option<usize>,
51    pub require_local: bool,
52    pub require_remote: bool,
53    pub stream: bool,
54    #[serde(default)]
55    pub execution: ExecutionPolicy,
56}
57
58impl Default for InferenceOptions {
59    fn default() -> Self {
60        Self {
61            timeout_ms: None,
62            max_output_tokens: None,
63            batch_size: None,
64            require_local: false,
65            require_remote: false,
66            stream: false,
67            execution: ExecutionPolicy::LocalOnly,
68        }
69    }
70}
71
72#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
73pub enum Input {
74    Text(String),
75    Tokens(Vec<u32>),
76    Tensor(Tensor),
77    Image(Image),
78    Audio(Audio),
79    Binary(Bytes),
80    Structured(Value),
81}
82
83impl Input {
84    pub fn batch_size(&self) -> usize {
85        match self {
86            Self::Tokens(_) => 1,
87            Self::Tensor(tensor) => tensor.shape.first().copied().unwrap_or(1),
88            _ => 1,
89        }
90    }
91}
92
93impl From<String> for Input {
94    fn from(value: String) -> Self {
95        Self::Text(value)
96    }
97}
98
99impl From<&str> for Input {
100    fn from(value: &str) -> Self {
101        Self::Text(value.to_owned())
102    }
103}
104
105#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
106pub enum Output {
107    Text(String),
108    Tokens(Vec<u32>),
109    Tensor(Tensor),
110    Embedding(Vec<f32>),
111    Structured(Value),
112    Binary(Bytes),
113}
114
115#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
116pub struct ExecutionMetadata {
117    #[serde(default)]
118    pub request_id: Option<String>,
119    pub model: String,
120    pub model_version: Option<String>,
121    /// Execution location or transport selected by this runtime (`local`, `remote`, etc.).
122    pub provider: String,
123    /// Computation implementation reported by the execution target (`onnx`, `cpu`, etc.).
124    pub backend: String,
125    /// Observable target selected by this runtime. For a client this may be remote even when
126    /// `backend` reports the server's local computation backend.
127    pub execution_target: String,
128    pub routing_policy: ExecutionPolicy,
129    pub fallback: bool,
130    pub fallback_from: Option<String>,
131    pub fallback_reason: Option<String>,
132    pub latency_ms: Option<f64>,
133    pub input_tokens: Option<u64>,
134    pub output_tokens: Option<u64>,
135    pub hardware: Option<String>,
136    #[serde(default)]
137    pub cache_hit: bool,
138    #[serde(default)]
139    pub batch_size: usize,
140    #[serde(default)]
141    pub queue_wait_ms: Option<f64>,
142    #[serde(default)]
143    pub model_load_ms: Option<f64>,
144    #[serde(default)]
145    pub execution_ms: Option<f64>,
146    #[serde(default)]
147    pub streaming_requested: bool,
148    #[serde(default)]
149    pub streaming_supported: bool,
150    #[serde(default)]
151    pub output_event_count: usize,
152    #[serde(default)]
153    pub completion_state: Option<String>,
154}
155
156#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
157pub struct InferenceRequest {
158    pub model: ModelReference,
159    pub input: Input,
160    pub options: InferenceOptions,
161}
162
163impl InferenceRequest {
164    /// Creates a request with default local-only execution options.
165    pub fn new(model: impl Into<ModelReference>, input: impl Into<Input>) -> Self {
166        Self {
167            model: model.into(),
168            input: input.into(),
169            options: InferenceOptions::default(),
170        }
171    }
172
173    /// Replaces the request's execution options.
174    pub fn with_options(mut self, options: InferenceOptions) -> Self {
175        self.options = options;
176        self
177    }
178}
179
180#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
181pub struct InferenceResult {
182    pub output: Output,
183    pub metadata: ExecutionMetadata,
184}
185
186impl InferenceResult {
187    pub fn output(&self) -> &Output {
188        &self.output
189    }
190
191    pub fn metadata(&self) -> &ExecutionMetadata {
192        &self.metadata
193    }
194
195    pub fn into_output(self) -> Output {
196        self.output
197    }
198}
199
200#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
201pub struct InferenceChunk {
202    pub output: Output,
203    pub done: bool,
204    pub metadata: Option<ExecutionMetadata>,
205}
206
207pub type InferenceOutput = Output;
208
209/// A model-neutral request for one or more typed decisions about immutable input state.
210#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
211pub struct DecisionRequest {
212    pub state: Value,
213    pub decisions: Vec<DecisionQuestion>,
214}
215
216impl DecisionRequest {
217    pub fn new(state: impl Into<Value>, decisions: Vec<DecisionQuestion>) -> Self {
218        Self {
219            state: state.into(),
220            decisions,
221        }
222    }
223}
224
225#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
226pub struct DecisionQuestion {
227    pub name: String,
228    pub instructions: String,
229    pub kind: DecisionType,
230}
231
232#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
233#[serde(tag = "type", rename_all = "snake_case")]
234pub enum DecisionType {
235    Choice {
236        options: Vec<DecisionOption>,
237    },
238    Score {
239        levels: Vec<Value>,
240    },
241    /// Laya's binary yes/no decision head ("noul" in the model contract).
242    Noul {
243        #[serde(default)]
244        false_description: Option<Value>,
245        #[serde(default)]
246        true_description: Option<Value>,
247    },
248}
249
250#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
251pub struct DecisionOption {
252    pub label: String,
253    #[serde(default)]
254    pub description: Option<Value>,
255}
256
257#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
258pub struct DecisionResult {
259    pub model: ModelIdentity,
260    pub backend: String,
261    pub decisions: Vec<TypedDecision>,
262    pub execution: DecisionExecution,
263    pub provenance: DecisionProvenance,
264}
265
266#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
267pub struct ModelIdentity {
268    pub identifier: String,
269    pub revision: Option<String>,
270}
271
272#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
273pub struct TypedDecision {
274    pub name: String,
275    pub kind: String,
276    pub value: DecisionValue,
277    pub probabilities: BTreeMap<String, f64>,
278    pub confidence: f64,
279    pub action_probability: f64,
280}
281
282#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
283#[serde(tag = "type", content = "value", rename_all = "snake_case")]
284pub enum DecisionValue {
285    Choice(String),
286    Score(f64),
287    Noul(bool),
288}
289
290#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
291pub struct DecisionExecution {
292    pub latency: Duration,
293    pub input_tokens: u64,
294    pub output_tokens: u64,
295}
296
297#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
298pub struct DecisionProvenance {
299    pub artifact_path: PathBuf,
300    pub artifact_sha256: String,
301    pub runtime_version: String,
302}
303
304#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
305pub struct ModelDescription {
306    pub identifier: String,
307    pub revision: Option<String>,
308    pub backend: String,
309    pub artifact_path: PathBuf,
310    pub artifact_sha256: String,
311    pub decision_types: Vec<String>,
312}
313
314#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
315pub enum InferenceStreamEvent {
316    Started(ExecutionMetadata),
317    Output(InferenceOutput),
318    Completed(ExecutionMetadata),
319}