1use bytes::Bytes;
2use ml_runtime_model::ModelReference;
3use serde::{Deserialize, Serialize};
4use serde_json::Value;
5use std::{collections::BTreeMap, path::PathBuf, time::Duration};
6
7#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
9pub enum ExecutionPolicy {
10 #[default]
11 LocalOnly,
12 RemoteOnly,
13 PreferLocal,
14 PreferRemote,
15 LocalThenRemote,
16 RemoteThenLocal,
17}
18
19#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
20pub struct Tensor {
21 pub shape: Vec<usize>,
22 pub values: Vec<f32>,
23}
24
25impl Tensor {
26 pub fn new(shape: impl Into<Vec<usize>>, values: impl Into<Vec<f32>>) -> Self {
27 Self {
28 shape: shape.into(),
29 values: values.into(),
30 }
31 }
32}
33
34#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
35pub struct Image {
36 pub mime_type: String,
37 pub data: Bytes,
38}
39
40#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
41pub struct Audio {
42 pub mime_type: String,
43 pub data: Bytes,
44}
45
46#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
47pub struct InferenceOptions {
48 pub timeout_ms: Option<u64>,
49 pub max_output_tokens: Option<u64>,
50 pub batch_size: Option<usize>,
51 pub require_local: bool,
52 pub require_remote: bool,
53 pub stream: bool,
54 #[serde(default)]
55 pub execution: ExecutionPolicy,
56}
57
58impl Default for InferenceOptions {
59 fn default() -> Self {
60 Self {
61 timeout_ms: None,
62 max_output_tokens: None,
63 batch_size: None,
64 require_local: false,
65 require_remote: false,
66 stream: false,
67 execution: ExecutionPolicy::LocalOnly,
68 }
69 }
70}
71
72#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
73pub enum Input {
74 Text(String),
75 Tokens(Vec<u32>),
76 Tensor(Tensor),
77 Image(Image),
78 Audio(Audio),
79 Binary(Bytes),
80 Structured(Value),
81}
82
83impl Input {
84 pub fn batch_size(&self) -> usize {
85 match self {
86 Self::Tokens(_) => 1,
87 Self::Tensor(tensor) => tensor.shape.first().copied().unwrap_or(1),
88 _ => 1,
89 }
90 }
91}
92
93impl From<String> for Input {
94 fn from(value: String) -> Self {
95 Self::Text(value)
96 }
97}
98
99impl From<&str> for Input {
100 fn from(value: &str) -> Self {
101 Self::Text(value.to_owned())
102 }
103}
104
105#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
106pub enum Output {
107 Text(String),
108 Tokens(Vec<u32>),
109 Tensor(Tensor),
110 Embedding(Vec<f32>),
111 Structured(Value),
112 Binary(Bytes),
113}
114
115#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
116pub struct ExecutionMetadata {
117 #[serde(default)]
118 pub request_id: Option<String>,
119 pub model: String,
120 pub model_version: Option<String>,
121 pub provider: String,
123 pub backend: String,
125 pub execution_target: String,
128 pub routing_policy: ExecutionPolicy,
129 pub fallback: bool,
130 pub fallback_from: Option<String>,
131 pub fallback_reason: Option<String>,
132 pub latency_ms: Option<f64>,
133 pub input_tokens: Option<u64>,
134 pub output_tokens: Option<u64>,
135 pub hardware: Option<String>,
136 #[serde(default)]
137 pub cache_hit: bool,
138 #[serde(default)]
139 pub batch_size: usize,
140 #[serde(default)]
141 pub queue_wait_ms: Option<f64>,
142 #[serde(default)]
143 pub model_load_ms: Option<f64>,
144 #[serde(default)]
145 pub execution_ms: Option<f64>,
146 #[serde(default)]
147 pub streaming_requested: bool,
148 #[serde(default)]
149 pub streaming_supported: bool,
150 #[serde(default)]
151 pub output_event_count: usize,
152 #[serde(default)]
153 pub completion_state: Option<String>,
154}
155
156#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
157pub struct InferenceRequest {
158 pub model: ModelReference,
159 pub input: Input,
160 pub options: InferenceOptions,
161}
162
163impl InferenceRequest {
164 pub fn new(model: impl Into<ModelReference>, input: impl Into<Input>) -> Self {
166 Self {
167 model: model.into(),
168 input: input.into(),
169 options: InferenceOptions::default(),
170 }
171 }
172
173 pub fn with_options(mut self, options: InferenceOptions) -> Self {
175 self.options = options;
176 self
177 }
178}
179
180#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
181pub struct InferenceResult {
182 pub output: Output,
183 pub metadata: ExecutionMetadata,
184}
185
186impl InferenceResult {
187 pub fn output(&self) -> &Output {
188 &self.output
189 }
190
191 pub fn metadata(&self) -> &ExecutionMetadata {
192 &self.metadata
193 }
194
195 pub fn into_output(self) -> Output {
196 self.output
197 }
198}
199
200#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
201pub struct InferenceChunk {
202 pub output: Output,
203 pub done: bool,
204 pub metadata: Option<ExecutionMetadata>,
205}
206
207pub type InferenceOutput = Output;
208
209#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
211pub struct DecisionRequest {
212 pub state: Value,
213 pub decisions: Vec<DecisionQuestion>,
214}
215
216impl DecisionRequest {
217 pub fn new(state: impl Into<Value>, decisions: Vec<DecisionQuestion>) -> Self {
218 Self {
219 state: state.into(),
220 decisions,
221 }
222 }
223}
224
225#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
226pub struct DecisionQuestion {
227 pub name: String,
228 pub instructions: String,
229 pub kind: DecisionType,
230}
231
232#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
233#[serde(tag = "type", rename_all = "snake_case")]
234pub enum DecisionType {
235 Choice {
236 options: Vec<DecisionOption>,
237 },
238 Score {
239 levels: Vec<Value>,
240 },
241 Noul {
243 #[serde(default)]
244 false_description: Option<Value>,
245 #[serde(default)]
246 true_description: Option<Value>,
247 },
248}
249
250#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
251pub struct DecisionOption {
252 pub label: String,
253 #[serde(default)]
254 pub description: Option<Value>,
255}
256
257#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
258pub struct DecisionResult {
259 pub model: ModelIdentity,
260 pub backend: String,
261 pub decisions: Vec<TypedDecision>,
262 pub execution: DecisionExecution,
263 pub provenance: DecisionProvenance,
264}
265
266#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
267pub struct ModelIdentity {
268 pub identifier: String,
269 pub revision: Option<String>,
270}
271
272#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
273pub struct TypedDecision {
274 pub name: String,
275 pub kind: String,
276 pub value: DecisionValue,
277 pub probabilities: BTreeMap<String, f64>,
278 pub confidence: f64,
279 pub action_probability: f64,
280}
281
282#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
283#[serde(tag = "type", content = "value", rename_all = "snake_case")]
284pub enum DecisionValue {
285 Choice(String),
286 Score(f64),
287 Noul(bool),
288}
289
290#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
291pub struct DecisionExecution {
292 pub latency: Duration,
293 pub input_tokens: u64,
294 pub output_tokens: u64,
295}
296
297#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
298pub struct DecisionProvenance {
299 pub artifact_path: PathBuf,
300 pub artifact_sha256: String,
301 pub runtime_version: String,
302}
303
304#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
305pub struct ModelDescription {
306 pub identifier: String,
307 pub revision: Option<String>,
308 pub backend: String,
309 pub artifact_path: PathBuf,
310 pub artifact_sha256: String,
311 pub decision_types: Vec<String>,
312}
313
314#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
315pub enum InferenceStreamEvent {
316 Started(ExecutionMetadata),
317 Output(InferenceOutput),
318 Completed(ExecutionMetadata),
319}