1#![cfg_attr(test, allow(clippy::default_constructed_unit_structs))]
2
3use futures_util::StreamExt;
4use ml_runtime_backend::{Backend, BackendCapability};
5use ml_runtime_common::BoxStream;
6use ml_runtime_cpu_backend::CpuBackend;
7use ml_runtime_http_provider::HttpProvider;
8use ml_runtime_inference::InferenceChunk;
9use ml_runtime_model::{CatalogError, ModelCatalog, ModelHandle};
10use ml_runtime_onnx_backend::OnnxBackend;
11use ml_runtime_provider::{Provider, ProviderCapability};
12use serde::{Deserialize, Serialize};
13use std::{
14 collections::{BTreeMap, HashMap, VecDeque},
15 hash::{Hash, Hasher},
16 path::{Path, PathBuf},
17 sync::{
18 atomic::{AtomicU64, AtomicUsize, Ordering},
19 Arc, Mutex, RwLock,
20 },
21 time::{Duration, Instant},
22};
23use tokio::{
24 sync::{mpsc, Semaphore},
25 time::timeout,
26};
27use tokio_stream::wrappers::ReceiverStream;
28
29mod capability;
30mod decision;
31
32pub use capability::{
33 AvailabilityState, BuiltinCapabilityProvider, Capability, CapabilityAuthorizer,
34 CapabilityAvailability, CapabilityExecutionMetadata, CapabilityLimits,
35 CapabilityPackageManifest, CapabilityPolicy, CapabilityProvider, CapabilityRegistry,
36 CapabilityResolution, ExecutionRequest, ExecutionResult, LocalProviderPackage,
37 ManifestCapabilityProvider, PackageCapability, PackageManifest, ProviderDescriptor,
38 ProviderManifest, ResolutionCandidate, ResolutionConstraints,
39};
40pub use decision::DecisionPlanner;
41
42pub use ml_runtime_backend;
43pub use ml_runtime_backend::{DecisionModel, DecisionModelProvider};
44pub use ml_runtime_common;
45pub use ml_runtime_common::{CancellationToken, RuntimeError, RuntimeResult};
46pub use ml_runtime_inference;
47pub use ml_runtime_inference::{
48 DecisionDependency, DecisionDependencyCondition, DecisionExecution,
49 DecisionExecutionDiagnostics, DecisionExecutionPlan, DecisionExecutionPolicy,
50 DecisionExecutionStrategy, DecisionGraphRequest, DecisionModelCapabilities, DecisionNode,
51 DecisionOption, DecisionPlanStage, DecisionProvenance, DecisionQuestion, DecisionRequest,
52 DecisionResult, DecisionType, DecisionValue, DeviceKind, ExecutionMetadata, ExecutionPolicy,
53 InferenceOptions, InferenceRequest, InferenceResult, InferenceStreamEvent, Input,
54 ModelArchitecture, ModelDescription, ModelIdentity, Output, PlannedDecisionResult, Tensor,
55 TypedDecision,
56};
57pub use ml_runtime_model;
58pub use ml_runtime_model::{
59 default_model_root, BuiltinModelRegistry, CompiledArtifactManifest, CompiledStatus,
60 FilesystemModelCatalog, InstallationManifest, ModelDescriptor, ModelFormat, ModelId,
61 ModelLocation, ModelReference, ModelSpec, RegisteredModelInstaller,
62};
63pub use ml_runtime_provider;
64
65pub const VERSION: &str = env!("CARGO_PKG_VERSION");
66
67#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
68#[serde(rename_all = "snake_case")]
69pub enum InstalledModelStatus {
70 NotInstalled,
71 InstalledNotCompiled,
72 Ready,
73 Corrupt,
74 Incompatible,
75 Unsupported,
76}
77
78impl std::fmt::Display for InstalledModelStatus {
79 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
80 formatter.write_str(match self {
81 Self::NotInstalled => "not_installed",
82 Self::InstalledNotCompiled => "installed_not_compiled",
83 Self::Ready => "ready",
84 Self::Corrupt => "corrupt",
85 Self::Incompatible => "incompatible",
86 Self::Unsupported => "unsupported",
87 })
88 }
89}
90
91#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
92pub struct InstalledModelRecord {
93 pub model: String,
94 pub revision: String,
95 pub backend: String,
96 pub platform: String,
97 pub status: InstalledModelStatus,
98 pub compiled: bool,
99 pub package_path: PathBuf,
100}
101
102#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
103pub struct ModelDiagnostic {
104 pub check: String,
105 pub ok: bool,
106 pub message: String,
107}
108
109#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
110pub struct ModelDoctorReport {
111 pub model: InstalledModelRecord,
112 pub diagnostics: Vec<ModelDiagnostic>,
113 pub manifest: Option<InstallationManifest>,
114}
115
116#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
117pub struct ModelLifecycleResult {
118 pub model: InstalledModelRecord,
119 pub already_installed: bool,
120 pub duration_ms: u128,
121}
122
123pub type InferenceStream = BoxStream<RuntimeResult<InferenceStreamEvent>>;
125
126#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
127pub enum Platform {
128 Linux,
129 Macos,
130 Windows,
131 Unknown,
132}
133
134impl Platform {
135 fn current() -> Self {
136 if cfg!(target_os = "linux") {
137 Self::Linux
138 } else if cfg!(target_os = "macos") {
139 Self::Macos
140 } else if cfg!(target_os = "windows") {
141 Self::Windows
142 } else {
143 Self::Unknown
144 }
145 }
146}
147
148#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
149pub struct ModelCapability {
150 pub id: String,
151 pub format: ModelFormat,
152 pub location: ModelLocation,
153 pub loaded: bool,
154 pub provider: Option<String>,
155 pub backend: Option<String>,
156}
157
158#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
159pub struct RuntimeCapabilities {
160 pub environment: RuntimeEnvironment,
161 pub platforms: Vec<Platform>,
162 pub providers: Vec<ProviderCapability>,
163 pub backends: Vec<BackendCapability>,
164 pub models: Vec<ModelCapability>,
165 pub capabilities: Vec<Capability>,
166}
167
168#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
169pub struct RuntimeEnvironment {
170 pub platform: String,
171 pub architecture: String,
172 pub operating_system: String,
173 pub cpu: String,
174 pub memory_bytes: Option<u64>,
175 pub accelerators: Vec<String>,
176 pub filesystem: String,
177 pub network: String,
178 pub developer_tools: Vec<String>,
179 pub installed_providers: Vec<String>,
180}
181
182impl RuntimeEnvironment {
183 pub fn discover(provider_ids: Vec<String>) -> Self {
184 Self {
185 platform: std::env::consts::OS.to_owned(),
186 architecture: std::env::consts::ARCH.to_owned(),
187 operating_system: std::env::consts::OS.to_owned(),
188 cpu: std::thread::available_parallelism()
189 .map(|count| format!("{} logical cores", count.get()))
190 .unwrap_or_else(|_| "unknown".to_owned()),
191 memory_bytes: available_memory_bytes(),
192 accelerators: Vec::new(),
193 filesystem: std::env::current_dir()
194 .map(|path| path.display().to_string())
195 .unwrap_or_else(|_| "unavailable".to_owned()),
196 network: "available by explicit policy".to_owned(),
197 developer_tools: ["git"]
198 .iter()
199 .filter(|tool| command_available(tool))
200 .map(|tool| (*tool).to_owned())
201 .collect(),
202 installed_providers: provider_ids,
203 }
204 }
205}
206
207fn command_available(command: &str) -> bool {
208 std::process::Command::new(command)
209 .arg("--version")
210 .output()
211 .map(|output| output.status.success())
212 .unwrap_or(false)
213}
214
215fn available_memory_bytes() -> Option<u64> {
216 let contents = std::fs::read_to_string("/proc/meminfo").ok()?;
217 contents.lines().find_map(|line| {
218 let value = line
219 .strip_prefix("MemTotal:")?
220 .split_whitespace()
221 .next()?
222 .parse::<u64>()
223 .ok()?;
224 Some(value * 1024)
225 })
226}
227
228#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
229pub struct RuntimeStatus {
230 pub loaded_models: usize,
231 pub max_models: usize,
232 pub max_concurrent_inferences: usize,
233 pub max_concurrent_model_loads: usize,
234}
235
236pub type ExecutionId = String;
237pub type RequestId = String;
238
239#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
240pub enum ExecutionCompletion {
241 Succeeded,
242 Failed,
243 Cancelled,
244}
245
246#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
247pub struct ExecutionRecord {
248 pub execution_id: ExecutionId,
249 pub request_id: RequestId,
250 pub model_id: String,
251 pub model_version: String,
252 pub provider: String,
253 pub backend: String,
254 pub target: String,
255 pub routing_policy: ExecutionPolicy,
256 pub fallback: Option<String>,
257 pub cache_hit: bool,
258 pub batch_size: usize,
259 pub queued_duration: Duration,
260 pub model_load_duration: Duration,
261 pub execution_duration: Duration,
262 pub total_duration: Duration,
263 pub streaming: bool,
264 pub output_event_count: usize,
265 pub completion: ExecutionCompletion,
266}
267
268#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
269pub struct RuntimeMetricsSnapshot {
270 pub total_requests: u64,
271 pub successful_requests: u64,
272 pub failed_requests: u64,
273 pub cancelled_requests: u64,
274 pub local_executions: u64,
275 pub remote_executions: u64,
276 pub fallback_executions: u64,
277 pub fallback_failures: u64,
278 pub model_loads: u64,
279 pub model_load_failures: u64,
280 pub cache_hits: u64,
281 pub cache_misses: u64,
282 pub evictions: u64,
283 pub queued_requests: u64,
284 pub active_inferences: usize,
285 pub active_model_loads: usize,
286 pub batch_executions: u64,
287 pub individual_executions: u64,
288 pub total_requests_batched: u64,
289 pub streaming_requests: u64,
290 pub streamed_output_events: u64,
291 pub cancelled_streams: u64,
292 pub compatibility_streams: u64,
293 pub queue_wait_micros: Vec<u64>,
294 pub model_load_micros: Vec<u64>,
295 pub execution_micros: Vec<u64>,
296 pub total_duration_micros: Vec<u64>,
297 pub batch_sizes: Vec<usize>,
298}
299
300#[derive(Default)]
301pub struct RuntimeMetrics {
302 total_requests: AtomicU64,
303 successful_requests: AtomicU64,
304 failed_requests: AtomicU64,
305 cancelled_requests: AtomicU64,
306 local_executions: AtomicU64,
307 remote_executions: AtomicU64,
308 fallback_executions: AtomicU64,
309 fallback_failures: AtomicU64,
310 model_loads: AtomicU64,
311 model_load_failures: AtomicU64,
312 cache_hits: AtomicU64,
313 cache_misses: AtomicU64,
314 evictions: AtomicU64,
315 queued_requests: AtomicU64,
316 active_inferences: AtomicUsize,
317 active_model_loads: AtomicUsize,
318 batch_executions: AtomicU64,
319 individual_executions: AtomicU64,
320 total_requests_batched: AtomicU64,
321 streaming_requests: AtomicU64,
322 streamed_output_events: AtomicU64,
323 cancelled_streams: AtomicU64,
324 compatibility_streams: AtomicU64,
325 queue_wait_micros: Mutex<Vec<u64>>,
326 model_load_micros: Mutex<Vec<u64>>,
327 execution_micros: Mutex<Vec<u64>>,
328 total_duration_micros: Mutex<Vec<u64>>,
329 batch_sizes: Mutex<Vec<usize>>,
330}
331
332impl RuntimeMetrics {
333 pub fn snapshot(&self) -> RuntimeMetricsSnapshot {
334 let values = |values: &Mutex<Vec<u64>>| values.lock().expect("metrics poisoned").clone();
335 RuntimeMetricsSnapshot {
336 total_requests: self.total_requests.load(Ordering::Relaxed),
337 successful_requests: self.successful_requests.load(Ordering::Relaxed),
338 failed_requests: self.failed_requests.load(Ordering::Relaxed),
339 cancelled_requests: self.cancelled_requests.load(Ordering::Relaxed),
340 local_executions: self.local_executions.load(Ordering::Relaxed),
341 remote_executions: self.remote_executions.load(Ordering::Relaxed),
342 fallback_executions: self.fallback_executions.load(Ordering::Relaxed),
343 fallback_failures: self.fallback_failures.load(Ordering::Relaxed),
344 model_loads: self.model_loads.load(Ordering::Relaxed),
345 model_load_failures: self.model_load_failures.load(Ordering::Relaxed),
346 cache_hits: self.cache_hits.load(Ordering::Relaxed),
347 cache_misses: self.cache_misses.load(Ordering::Relaxed),
348 evictions: self.evictions.load(Ordering::Relaxed),
349 queued_requests: self.queued_requests.load(Ordering::Relaxed),
350 active_inferences: self.active_inferences.load(Ordering::Relaxed),
351 active_model_loads: self.active_model_loads.load(Ordering::Relaxed),
352 batch_executions: self.batch_executions.load(Ordering::Relaxed),
353 individual_executions: self.individual_executions.load(Ordering::Relaxed),
354 total_requests_batched: self.total_requests_batched.load(Ordering::Relaxed),
355 streaming_requests: self.streaming_requests.load(Ordering::Relaxed),
356 streamed_output_events: self.streamed_output_events.load(Ordering::Relaxed),
357 cancelled_streams: self.cancelled_streams.load(Ordering::Relaxed),
358 compatibility_streams: self.compatibility_streams.load(Ordering::Relaxed),
359 queue_wait_micros: values(&self.queue_wait_micros),
360 model_load_micros: values(&self.model_load_micros),
361 execution_micros: values(&self.execution_micros),
362 total_duration_micros: values(&self.total_duration_micros),
363 batch_sizes: self.batch_sizes.lock().expect("metrics poisoned").clone(),
364 }
365 }
366}
367
368#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
369pub struct ModelResource {
370 pub id: String,
371 pub version: Option<String>,
372 pub target: String,
373 pub state: ModelLifecycleState,
374 pub memory_bytes: u64,
375 pub last_used: u64,
376}
377
378#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
379pub struct RuntimeSnapshot {
380 pub active_inferences: usize,
381 pub active_model_loads: usize,
382 pub queued_inferences: usize,
383 pub loaded_models: usize,
384 pub cache_entries: usize,
385 pub cache_memory_bytes: u64,
386 pub metrics: RuntimeMetricsSnapshot,
387}
388
389#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
390pub enum ModelLifecycleState {
391 Discovered,
392 Validated,
393 Loaded,
394 Ready,
395 InUse,
396 Evicted,
397}
398
399#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
400pub struct RuntimeConfig {
401 pub selected_backend: Option<String>,
402 pub selected_provider: Option<String>,
403 pub prefer_acceleration: bool,
404 pub allow_remote_fallback: bool,
405 pub timeout_ms: Option<u64>,
406 pub max_concurrency: usize,
407 pub max_batch_size: Option<usize>,
408 pub max_concurrent_inferences: usize,
409 pub max_concurrent_model_loads: usize,
410 pub max_models: usize,
411 pub max_memory_bytes: Option<u64>,
412 pub max_stream_buffer: usize,
413}
414
415impl Default for RuntimeConfig {
416 fn default() -> Self {
417 Self {
418 selected_backend: None,
419 selected_provider: Some("local".to_owned()),
420 prefer_acceleration: false,
421 allow_remote_fallback: false,
422 timeout_ms: None,
423 max_concurrency: 4,
424 max_batch_size: Some(16),
425 max_concurrent_inferences: 4,
426 max_concurrent_model_loads: 1,
427 max_models: 8,
428 max_memory_bytes: None,
429 max_stream_buffer: 32,
430 }
431 }
432}
433
434#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
435pub struct RuntimeSelection {
436 pub model: String,
437 pub provider: String,
438 pub backend: Option<String>,
439 pub hardware: Option<String>,
440 pub fallback: bool,
441 pub fallback_from: Option<String>,
442 pub reason: String,
443 pub policy: ExecutionPolicy,
444}
445
446#[derive(Clone, Debug)]
447pub struct ModelRegistry {
448 models: Vec<ModelCapability>,
449}
450
451impl ModelRegistry {
452 pub fn list(&self) -> &[ModelCapability] {
453 &self.models
454 }
455}
456
457#[derive(Clone, Debug)]
458pub struct ProviderRegistry {
459 providers: Vec<ProviderCapability>,
460}
461
462impl ProviderRegistry {
463 pub fn list(&self) -> &[ProviderCapability] {
464 &self.providers
465 }
466}
467
468#[derive(Clone, Debug)]
469pub struct BackendRegistry {
470 backends: Vec<BackendCapability>,
471}
472
473impl BackendRegistry {
474 pub fn list(&self) -> &[BackendCapability] {
475 &self.backends
476 }
477}
478
479#[derive(Clone)]
480struct LoadedModel {
481 spec: ModelSpec,
482 handle: ModelHandle,
483 backend_name: String,
484 provider_name: String,
485 last_used: u64,
486 memory_bytes: u64,
487}
488
489#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
490pub struct RuntimeModelHandle {
492 model_id: String,
493 version: Option<String>,
494 provider: String,
495 backend: String,
496 state: ModelLifecycleState,
497}
498
499impl RuntimeModelHandle {
500 pub fn id(&self) -> &str {
501 &self.model_id
502 }
503 pub fn version(&self) -> Option<&str> {
504 self.version.as_deref()
505 }
506 pub fn provider(&self) -> &str {
507 &self.provider
508 }
509 pub fn backend(&self) -> &str {
510 &self.backend
511 }
512 pub fn state(&self) -> &ModelLifecycleState {
513 &self.state
514 }
515}
516
517#[derive(Clone, Debug, Eq)]
518struct ModelCacheKey {
519 id: String,
520 version: Option<String>,
521 provider: String,
522 backend: String,
523}
524
525impl PartialEq for ModelCacheKey {
526 fn eq(&self, other: &Self) -> bool {
527 self.id == other.id
528 && self.version == other.version
529 && self.provider == other.provider
530 && self.backend == other.backend
531 }
532}
533
534impl Hash for ModelCacheKey {
535 fn hash<H: Hasher>(&self, state: &mut H) {
536 self.id.hash(state);
537 self.version.hash(state);
538 self.provider.hash(state);
539 self.backend.hash(state);
540 }
541}
542
543pub struct RuntimeBuilder {
545 config: RuntimeConfig,
546 backends: BTreeMap<String, Arc<dyn Backend>>,
547 providers: BTreeMap<String, Arc<dyn Provider>>,
548 decision_providers: BTreeMap<String, Arc<dyn DecisionModelProvider>>,
549 catalog: Option<Arc<dyn ModelCatalog>>,
550 capability_registry: CapabilityRegistry,
551 model_root: Option<std::path::PathBuf>,
552}
553
554impl Default for RuntimeBuilder {
555 fn default() -> Self {
556 let mut builder = Self {
557 config: RuntimeConfig::default(),
558 backends: BTreeMap::new(),
559 providers: BTreeMap::new(),
560 decision_providers: BTreeMap::new(),
561 catalog: None,
562 capability_registry: CapabilityRegistry::with_builtins(),
563 model_root: default_model_root().ok(),
564 };
565 builder
566 .backends
567 .insert("cpu".to_owned(), Arc::new(CpuBackend));
568 builder
569 .backends
570 .insert("onnx".to_owned(), Arc::new(OnnxBackend));
571 builder
572 .decision_providers
573 .insert("onnx".to_owned(), Arc::new(OnnxBackend));
574 #[cfg(feature = "coreml")]
575 builder.decision_providers.insert(
576 "coreml".to_owned(),
577 Arc::new(ml_runtime_coreml_backend::CoreMlBackend),
578 );
579 builder
580 }
581}
582
583impl RuntimeBuilder {
584 pub fn backend(mut self, name: impl Into<String>) -> Self {
585 self.config.selected_backend = Some(name.into());
586 self
587 }
588
589 pub fn provider(mut self, name: impl Into<String>) -> Self {
590 self.config.selected_provider = Some(name.into());
591 self
592 }
593
594 pub fn remote(mut self, endpoint: impl Into<String>) -> Self {
596 let provider = HttpProvider::new(endpoint);
597 self.providers
598 .insert(provider.name().to_owned(), Arc::new(provider));
599 self
600 }
601
602 pub fn catalog<C>(mut self, catalog: C) -> Self
603 where
604 C: ModelCatalog + 'static,
605 {
606 self.catalog = Some(Arc::new(catalog));
607 self
608 }
609
610 pub fn model_root(mut self, root: impl Into<std::path::PathBuf>) -> Self {
611 self.model_root = Some(root.into());
612 self
613 }
614
615 pub fn prefer_acceleration(mut self, enabled: bool) -> Self {
616 self.config.prefer_acceleration = enabled;
617 self
618 }
619
620 pub fn allow_remote_fallback(mut self, enabled: bool) -> Self {
621 self.config.allow_remote_fallback = enabled;
622 self
623 }
624
625 pub fn timeout_ms(mut self, timeout_ms: u64) -> Self {
626 self.config.timeout_ms = Some(timeout_ms);
627 self
628 }
629
630 pub fn concurrency(mut self, max_concurrency: usize) -> Self {
631 self.config.max_concurrency = max_concurrency.max(1);
632 self.config.max_concurrent_inferences = max_concurrency.max(1);
633 self
634 }
635
636 pub fn max_concurrent_inferences(mut self, limit: usize) -> Self {
637 self.config.max_concurrent_inferences = limit.max(1);
638 self.config.max_concurrency = limit.max(1);
639 self
640 }
641
642 pub fn max_concurrent_model_loads(mut self, limit: usize) -> Self {
643 self.config.max_concurrent_model_loads = limit.max(1);
644 self
645 }
646
647 pub fn max_models(mut self, limit: usize) -> Self {
648 self.config.max_models = limit.max(1);
649 self
650 }
651
652 pub fn max_memory_bytes(mut self, limit: u64) -> Self {
653 self.config.max_memory_bytes = Some(limit);
654 self
655 }
656
657 pub fn max_batch_size(mut self, max_batch_size: usize) -> Self {
658 self.config.max_batch_size = Some(max_batch_size.max(1));
659 self
660 }
661
662 pub fn register_backend<B>(mut self, backend: B) -> Self
664 where
665 B: Backend + 'static,
666 {
667 let name = backend.name().to_owned();
668 self.backends.insert(name, Arc::new(backend));
669 self
670 }
671
672 pub fn register_backend_arc(mut self, backend: Arc<dyn Backend>) -> Self {
673 self.backends.insert(backend.name().to_owned(), backend);
674 self
675 }
676
677 pub fn register_provider<P>(mut self, provider: P) -> Self
679 where
680 P: Provider + 'static,
681 {
682 let name = provider.name().to_owned();
683 self.providers.insert(name, Arc::new(provider));
684 self
685 }
686
687 pub fn register_provider_arc(mut self, provider: Arc<dyn Provider>) -> Self {
688 self.providers.insert(provider.name().to_owned(), provider);
689 self
690 }
691
692 pub fn register_decision_provider<P>(mut self, provider: P) -> Self
693 where
694 P: DecisionModelProvider + 'static,
695 {
696 self.decision_providers
697 .insert(provider.name().to_owned(), Arc::new(provider));
698 self
699 }
700
701 pub fn register_capability_provider<P>(mut self, provider: P) -> Self
702 where
703 P: CapabilityProvider + 'static,
704 {
705 self.capability_registry.register(provider);
706 self
707 }
708
709 pub fn build(self) -> Runtime {
710 Runtime {
711 config: self.config.clone(),
712 backends: self.backends,
713 providers: self.providers,
714 decision_providers: self.decision_providers,
715 catalog: self.catalog,
716 capability_registry: self.capability_registry,
717 model_root: self.model_root,
718 loaded_models: RwLock::new(HashMap::new()),
719 inference_semaphore: Arc::new(Semaphore::new(
720 self.config.max_concurrent_inferences.max(1),
721 )),
722 load_semaphore: Arc::new(Semaphore::new(
723 self.config.max_concurrent_model_loads.max(1),
724 )),
725 cache_clock: AtomicU64::new(0),
726 metrics: Arc::new(RuntimeMetrics::default()),
727 execution_clock: AtomicU64::new(0),
728 execution_records: Mutex::new(VecDeque::new()),
729 }
730 }
731}
732
733pub struct Runtime {
739 config: RuntimeConfig,
740 backends: BTreeMap<String, Arc<dyn Backend>>,
741 providers: BTreeMap<String, Arc<dyn Provider>>,
742 decision_providers: BTreeMap<String, Arc<dyn DecisionModelProvider>>,
743 catalog: Option<Arc<dyn ModelCatalog>>,
744 capability_registry: CapabilityRegistry,
745 model_root: Option<std::path::PathBuf>,
746 loaded_models: RwLock<HashMap<ModelCacheKey, LoadedModel>>,
747 inference_semaphore: Arc<Semaphore>,
748 load_semaphore: Arc<Semaphore>,
749 cache_clock: AtomicU64,
750 metrics: Arc<RuntimeMetrics>,
751 execution_clock: AtomicU64,
752 execution_records: Mutex<VecDeque<ExecutionRecord>>,
753}
754
755impl Default for Runtime {
756 fn default() -> Self {
757 Self::new()
758 }
759}
760
761impl Runtime {
762 pub fn new() -> Self {
763 Self::builder().build()
764 }
765
766 pub fn builder() -> RuntimeBuilder {
767 RuntimeBuilder::default()
768 }
769
770 pub fn config(&self) -> &RuntimeConfig {
771 &self.config
772 }
773
774 pub fn load_decision_model(
777 &self,
778 artifact: impl AsRef<Path>,
779 ) -> RuntimeResult<Box<dyn DecisionModel>> {
780 let artifact = artifact.as_ref();
781 if !artifact.exists() {
782 return Err(RuntimeError::ModelNotFound {
783 model: artifact.display().to_string(),
784 });
785 }
786 if !artifact.is_dir() {
787 return Err(RuntimeError::ModelUnavailable {
788 model: artifact.display().to_string(),
789 provider: "local".to_owned(),
790 reason: "typed-decision artifact must be a directory".to_owned(),
791 });
792 }
793 let candidates: Vec<_> = self
794 .decision_providers
795 .values()
796 .filter(|provider| provider.supports_artifact(artifact))
797 .collect();
798 match candidates.as_slice() {
799 [] => Err(RuntimeError::UnsupportedModel {
800 model: artifact.display().to_string(),
801 backend: "none".to_owned(),
802 reason: "no registered typed-decision provider accepts this artifact".to_owned(),
803 }),
804 [provider] => provider.load_decision_model(artifact),
805 _ => Err(RuntimeError::CapabilityMismatch {
806 reason: format!(
807 "multiple typed-decision providers accept {}",
808 artifact.display()
809 ),
810 }),
811 }
812 }
813
814 pub fn decision_capabilities(&self, model: &dyn DecisionModel) -> DecisionModelCapabilities {
817 model.capabilities()
818 }
819
820 pub fn explain_decision(
822 &self,
823 model: &dyn DecisionModel,
824 request: &DecisionGraphRequest,
825 ) -> RuntimeResult<DecisionExecutionPlan> {
826 DecisionPlanner.plan(request, &model.capabilities())
827 }
828
829 pub async fn execute_decision_graph(
832 &self,
833 model: Arc<dyn DecisionModel>,
834 request: DecisionGraphRequest,
835 cancellation: Option<CancellationToken>,
836 ) -> RuntimeResult<PlannedDecisionResult> {
837 decision::execute_graph(model, request, cancellation).await
838 }
839
840 pub async fn decide_async(
843 &self,
844 model: Arc<dyn DecisionModel>,
845 request: DecisionRequest,
846 policy: DecisionExecutionPolicy,
847 cancellation: Option<CancellationToken>,
848 ) -> RuntimeResult<PlannedDecisionResult> {
849 self.execute_decision_graph(
850 model,
851 DecisionGraphRequest::independent(request, policy),
852 cancellation,
853 )
854 .await
855 }
856
857 fn registered_model_root(&self) -> RuntimeResult<&Path> {
858 self.model_root
859 .as_deref()
860 .ok_or_else(|| RuntimeError::ModelUnavailable {
861 model: "registered models".to_owned(),
862 provider: "local".to_owned(),
863 reason: "model directory unavailable; set ML_RUNTIME_MODEL_DIR".to_owned(),
864 })
865 }
866
867 pub fn installed_models(&self) -> RuntimeResult<Vec<InstalledModelRecord>> {
868 let root = self.registered_model_root()?;
869 let installer = RegisteredModelInstaller::new(root)
870 .map_err(|error| RuntimeError::execution("model list", error.to_string()))?;
871 Ok(BuiltinModelRegistry
872 .list()
873 .into_iter()
874 .map(|model| {
875 let package = installer.installed_path(&model);
876 let (status, compiled) = if !package.exists() {
877 (InstalledModelStatus::NotInstalled, false)
878 } else if model.backend == "coreml" && !cfg!(target_os = "macos") {
879 (InstalledModelStatus::Unsupported, false)
880 } else {
881 match InstallationManifest::read(&package) {
882 Ok(manifest)
883 if manifest.model == model.name
884 && manifest.model_revision == model.revision
885 && manifest.runtime_requirement == model.runtime_requirement
886 && ml_runtime_model::runtime_requirement_satisfied(
887 &manifest.runtime_requirement,
888 VERSION,
889 )
890 && manifest.os == std::env::consts::OS
891 && manifest.architecture == std::env::consts::ARCH
892 && manifest.compiled_status == CompiledStatus::Ready
893 && manifest
894 .compiled_artifact
895 .as_ref()
896 .is_some_and(|artifact| artifact.path.exists()) =>
897 {
898 (InstalledModelStatus::Ready, true)
899 }
900 Ok(manifest) if manifest.compiled_status != CompiledStatus::Ready => {
901 (InstalledModelStatus::InstalledNotCompiled, false)
902 }
903 Ok(_) => (InstalledModelStatus::Incompatible, false),
904 Err(_) => (InstalledModelStatus::Corrupt, false),
905 }
906 };
907 InstalledModelRecord {
908 model: model.name,
909 revision: model.revision,
910 backend: model.backend,
911 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
912 status,
913 compiled,
914 package_path: package,
915 }
916 })
917 .collect())
918 }
919
920 pub async fn install_registered_model(
921 &self,
922 name: &str,
923 source_override: Option<&str>,
924 replace: bool,
925 ) -> RuntimeResult<ModelLifecycleResult> {
926 let started = Instant::now();
927 let model =
928 BuiltinModelRegistry
929 .resolve(name)
930 .map_err(|error| RuntimeError::ModelNotFound {
931 model: error.to_string(),
932 })?;
933 if model.backend == "coreml" && !cfg!(target_os = "macos") {
934 return Err(RuntimeError::backend_unavailable(
935 "coreml",
936 "Laya installation requires macOS; Core ML is unavailable on this platform",
937 ));
938 }
939 let root = self.registered_model_root()?;
940 let installer = RegisteredModelInstaller::new(root)
941 .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
942 let installed = installer
943 .install(
944 &model,
945 source_override,
946 if replace {
947 ml_runtime_model::InstallMode::Replace
948 } else {
949 ml_runtime_model::InstallMode::KeepExisting
950 },
951 None,
952 )
953 .await
954 .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
955
956 let (prepared_path, model_identity, prepared_identity) = match model.backend.as_str() {
957 "onnx" => {
958 let identity = OnnxBackend::validate_laya(&installed.package_path)?;
959 (
960 installed.package_path.join(&model.artifact_path),
961 identity.clone(),
962 identity,
963 )
964 }
965 "coreml" => {
966 #[cfg(feature = "coreml")]
967 {
968 let prepared = if installed.already_installed && !replace {
969 ml_runtime_coreml_backend::CoreMlBackend::validate_prepared_laya(
970 &installed.package_path,
971 )
972 .or_else(|_| {
973 ml_runtime_coreml_backend::CoreMlBackend::prepare_laya(
974 &installed.package_path,
975 )
976 })?
977 } else {
978 ml_runtime_coreml_backend::CoreMlBackend::prepare_laya(
979 &installed.package_path,
980 )?
981 };
982 (
983 prepared.path,
984 prepared.model_identity,
985 prepared.compiled_identity,
986 )
987 }
988 #[cfg(not(feature = "coreml"))]
989 {
990 return Err(RuntimeError::backend_unavailable(
991 "coreml",
992 "this runtime was built without Core ML support",
993 ));
994 }
995 }
996 backend => {
997 return Err(RuntimeError::UnsupportedModel {
998 model: model.name.clone(),
999 backend: backend.to_owned(),
1000 reason: "registered backend has no preparation lifecycle".to_owned(),
1001 })
1002 }
1003 };
1004
1005 {
1006 let manifest = InstallationManifest {
1007 schema_version: 1,
1008 model: model.name.clone(),
1009 model_revision: model.revision.clone(),
1010 model_checksum: model_identity,
1011 runtime_version: VERSION.to_owned(),
1012 runtime_requirement: model.runtime_requirement.clone(),
1013 backend: model.backend.clone(),
1014 os: std::env::consts::OS.to_owned(),
1015 architecture: std::env::consts::ARCH.to_owned(),
1016 installed_at_unix_seconds: std::time::SystemTime::now()
1017 .duration_since(std::time::UNIX_EPOCH)
1018 .unwrap_or_default()
1019 .as_secs(),
1020 compiled_status: CompiledStatus::Ready,
1021 compiled_artifact: Some(CompiledArtifactManifest {
1022 path: prepared_path,
1023 identity: prepared_identity,
1024 }),
1025 };
1026 manifest
1027 .write_atomic(&installed.package_path)
1028 .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
1029 Ok(ModelLifecycleResult {
1030 model: InstalledModelRecord {
1031 model: model.name,
1032 revision: model.revision,
1033 backend: model.backend,
1034 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1035 status: InstalledModelStatus::Ready,
1036 compiled: true,
1037 package_path: installed.package_path,
1038 },
1039 already_installed: installed.already_installed,
1040 duration_ms: started.elapsed().as_millis(),
1041 })
1042 }
1043 }
1044
1045 pub fn doctor_registered_model(&self, name: &str) -> RuntimeResult<ModelDoctorReport> {
1046 let model =
1047 BuiltinModelRegistry
1048 .resolve(name)
1049 .map_err(|error| RuntimeError::ModelNotFound {
1050 model: error.to_string(),
1051 })?;
1052 let root = self.registered_model_root()?;
1053 let installer = RegisteredModelInstaller::new(root)
1054 .map_err(|error| RuntimeError::execution("model doctor", error.to_string()))?;
1055 let package = installer.installed_path(&model);
1056 let mut diagnostics = Vec::new();
1057 if !package.is_dir() {
1058 diagnostics.push(ModelDiagnostic {
1059 check: "installation".to_owned(),
1060 ok: false,
1061 message: format!(
1062 "model is not installed; run `ml-runtime model install {}`",
1063 model.name
1064 ),
1065 });
1066 return Ok(ModelDoctorReport {
1067 model: InstalledModelRecord {
1068 model: model.name,
1069 revision: model.revision,
1070 backend: model.backend,
1071 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1072 status: InstalledModelStatus::NotInstalled,
1073 compiled: false,
1074 package_path: package,
1075 },
1076 diagnostics,
1077 manifest: None,
1078 });
1079 }
1080 let artifact_valid = match installer.validate(&model, &package) {
1081 Ok(()) => {
1082 diagnostics.push(ModelDiagnostic {
1083 check: "artifact".to_owned(),
1084 ok: true,
1085 message: format!(
1086 "registered files, {} artifact, tokenizer, and SHA-256 checksums are valid",
1087 model.backend
1088 ),
1089 });
1090 true
1091 }
1092 Err(error) => {
1093 diagnostics.push(ModelDiagnostic {
1094 check: "artifact".to_owned(),
1095 ok: false,
1096 message: format!(
1097 "{error}; reinstall with `ml-runtime model install {} --replace`",
1098 model.name
1099 ),
1100 });
1101 false
1102 }
1103 };
1104 let manifest = InstallationManifest::read(&package).ok();
1105 let compatible = manifest.as_ref().is_some_and(|manifest| {
1106 manifest.schema_version == 1
1107 && manifest.model == model.name
1108 && manifest.model_revision == model.revision
1109 && manifest.backend == model.backend
1110 && ml_runtime_model::runtime_requirement_satisfied(
1111 &manifest.runtime_requirement,
1112 VERSION,
1113 )
1114 && manifest.os == std::env::consts::OS
1115 && manifest.architecture == std::env::consts::ARCH
1116 });
1117 diagnostics.push(ModelDiagnostic {
1118 check: "metadata".to_owned(),
1119 ok: compatible,
1120 message: if compatible {
1121 "installation metadata and runtime/backend compatibility are valid".to_owned()
1122 } else {
1123 format!(
1124 "installation metadata is missing or incompatible; run `ml-runtime model install {} --replace`",
1125 model.name
1126 )
1127 },
1128 });
1129 let supported = model.supports_platform(std::env::consts::OS, std::env::consts::ARCH);
1130 diagnostics.push(ModelDiagnostic {
1131 check: "platform".to_owned(),
1132 ok: supported,
1133 message: if supported {
1134 format!("{} is supported on this platform", model.backend)
1135 } else {
1136 "Core ML is only supported on macOS".to_owned()
1137 },
1138 });
1139 let compiled_valid = supported
1140 && compatible
1141 && match model.backend.as_str() {
1142 "onnx" => OnnxBackend::validate_laya(&package).is_ok_and(|identity| {
1143 manifest.as_ref().is_some_and(|manifest| {
1144 manifest.model_checksum == identity
1145 && manifest.compiled_artifact.as_ref().is_some_and(|artifact| {
1146 artifact.path == package.join(&model.artifact_path)
1147 && artifact.identity == identity
1148 })
1149 })
1150 }),
1151 "coreml" => {
1152 #[cfg(feature = "coreml")]
1153 {
1154 ml_runtime_coreml_backend::CoreMlBackend::validate_prepared_laya(&package)
1155 .is_ok()
1156 }
1157 #[cfg(not(feature = "coreml"))]
1158 {
1159 false
1160 }
1161 }
1162 _ => false,
1163 };
1164 diagnostics.push(ModelDiagnostic {
1165 check: "compiled_artifact".to_owned(),
1166 ok: compiled_valid,
1167 message: if compiled_valid {
1168 format!("prepared {} artifact identity and schema are valid", model.backend)
1169 } else {
1170 format!(
1171 "compiled artifact is missing or invalid; run `ml-runtime model install {} --replace`",
1172 model.name
1173 )
1174 },
1175 });
1176 let status = if !supported {
1177 InstalledModelStatus::Unsupported
1178 } else if !artifact_valid {
1179 InstalledModelStatus::Corrupt
1180 } else if !compatible {
1181 InstalledModelStatus::Incompatible
1182 } else if !compiled_valid {
1183 InstalledModelStatus::InstalledNotCompiled
1184 } else {
1185 InstalledModelStatus::Ready
1186 };
1187 Ok(ModelDoctorReport {
1188 model: InstalledModelRecord {
1189 model: model.name,
1190 revision: model.revision,
1191 backend: model.backend,
1192 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1193 compiled: compiled_valid,
1194 package_path: package,
1195 status,
1196 },
1197 diagnostics,
1198 manifest,
1199 })
1200 }
1201
1202 pub fn remove_registered_model(&self, name: &str) -> RuntimeResult<InstalledModelRecord> {
1203 let model =
1204 BuiltinModelRegistry
1205 .resolve(name)
1206 .map_err(|error| RuntimeError::ModelNotFound {
1207 model: error.to_string(),
1208 })?;
1209 let root = self.registered_model_root()?;
1210 let installer = RegisteredModelInstaller::new(root)
1211 .map_err(|error| RuntimeError::execution("model remove", error.to_string()))?;
1212 let package = installer.installed_path(&model);
1213 if package.exists() {
1214 #[cfg(feature = "coreml")]
1215 if model.backend == "coreml" {
1216 ml_runtime_coreml_backend::CoreMlBackend::remove_prepared_laya(&package)?;
1217 }
1218 std::fs::remove_dir_all(&package)
1219 .map_err(|error| RuntimeError::execution("model remove", error.to_string()))?;
1220 }
1221 Ok(InstalledModelRecord {
1222 model: model.name,
1223 revision: model.revision,
1224 backend: model.backend,
1225 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1226 status: InstalledModelStatus::NotInstalled,
1227 compiled: false,
1228 package_path: package,
1229 })
1230 }
1231
1232 pub fn load_model(&self, name: &str) -> RuntimeResult<Box<dyn DecisionModel>> {
1235 let registered =
1236 BuiltinModelRegistry
1237 .resolve(name)
1238 .map_err(|error| RuntimeError::ModelNotFound {
1239 model: error.to_string(),
1240 })?;
1241 let root = self
1242 .model_root
1243 .as_ref()
1244 .ok_or_else(|| RuntimeError::ModelUnavailable {
1245 model: name.to_owned(),
1246 provider: "local".to_owned(),
1247 reason: "model directory unavailable; set ML_RUNTIME_MODEL_DIR".to_owned(),
1248 })?;
1249 let installer = RegisteredModelInstaller::new(root).map_err(|error| {
1250 RuntimeError::execution("installed model validation", error.to_string())
1251 })?;
1252 let package = installer.installed_path(®istered);
1253 if !package.exists() {
1254 return Err(RuntimeError::ModelNotFound {
1255 model: format!("{name}; install it with `ml-runtime model install {name}`"),
1256 });
1257 }
1258 installer.validate(®istered, &package).map_err(|error| {
1259 RuntimeError::ModelIntegrity {
1260 model: name.to_owned(),
1261 reason: format!(
1262 "{error}; reinstall with `ml-runtime model install {name} --replace`"
1263 ),
1264 }
1265 })?;
1266 let manifest =
1267 InstallationManifest::read(&package).map_err(|error| RuntimeError::ModelIntegrity {
1268 model: name.to_owned(),
1269 reason: format!("{error}; run `ml-runtime model doctor {name}`"),
1270 })?;
1271 let compatible = manifest.schema_version == 1
1272 && manifest.model == registered.name
1273 && manifest.model_revision == registered.revision
1274 && manifest.backend == registered.backend
1275 && ml_runtime_model::runtime_requirement_satisfied(
1276 &manifest.runtime_requirement,
1277 VERSION,
1278 )
1279 && manifest.os == std::env::consts::OS
1280 && manifest.architecture == std::env::consts::ARCH
1281 && manifest.compiled_status == CompiledStatus::Ready
1282 && manifest.compiled_artifact.as_ref().is_some_and(|artifact| {
1283 artifact.path.exists()
1284 && (registered.backend != "onnx"
1285 || (artifact.path == package.join(®istered.artifact_path)
1286 && artifact.identity == manifest.model_checksum))
1287 });
1288 if !compatible {
1289 return Err(RuntimeError::ModelUnavailable {
1290 model: name.to_owned(),
1291 provider: "local".to_owned(),
1292 reason: format!(
1293 "installation is incomplete, incompatible, or not compiled; run `ml-runtime model doctor {name}`"
1294 ),
1295 });
1296 }
1297 self.load_decision_model(package)
1298 }
1299
1300 pub fn set_catalog<C>(&mut self, catalog: C)
1301 where
1302 C: ModelCatalog + 'static,
1303 {
1304 self.catalog = Some(Arc::new(catalog));
1305 }
1306
1307 pub async fn catalog_models(&self) -> RuntimeResult<Vec<ModelDescriptor>> {
1308 self.catalog
1309 .as_ref()
1310 .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1311 .list()
1312 .await
1313 .map_err(catalog_error)
1314 }
1315
1316 pub async fn resolve_model(
1317 &self,
1318 reference: &ModelReference,
1319 ) -> RuntimeResult<ModelDescriptor> {
1320 self.catalog
1321 .as_ref()
1322 .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1323 .resolve(reference)
1324 .await
1325 .map_err(catalog_error)
1326 }
1327
1328 pub async fn refresh_catalog(&self) -> RuntimeResult<()> {
1329 self.catalog
1330 .as_ref()
1331 .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1332 .refresh()
1333 .await
1334 .map_err(catalog_error)
1335 }
1336
1337 pub fn models(&self) -> ModelRegistry {
1338 let models = self
1339 .loaded_models
1340 .read()
1341 .expect("loaded model registry poisoned")
1342 .values()
1343 .map(|entry| ModelCapability {
1344 id: entry.spec.id.clone(),
1345 format: entry.spec.format.clone(),
1346 location: entry.spec.location.clone(),
1347 loaded: true,
1348 provider: Some(entry.provider_name.clone()),
1349 backend: Some(entry.backend_name.clone()),
1350 })
1351 .collect();
1352 ModelRegistry { models }
1353 }
1354
1355 pub fn providers(&self) -> ProviderRegistry {
1356 let providers = self
1357 .providers
1358 .iter()
1359 .map(|(name, provider)| {
1360 ProviderCapability::from_parts(name.clone(), provider.capabilities())
1361 })
1362 .collect();
1363 ProviderRegistry { providers }
1364 }
1365
1366 pub fn backends(&self) -> BackendRegistry {
1367 let backends = self
1368 .backends
1369 .iter()
1370 .map(|(name, backend)| {
1371 BackendCapability::from_parts(name.clone(), backend.capabilities())
1372 })
1373 .collect();
1374 BackendRegistry { backends }
1375 }
1376
1377 pub fn capabilities(&self) -> RuntimeCapabilities {
1378 RuntimeCapabilities {
1379 environment: self.environment(),
1380 platforms: vec![Platform::current()],
1381 providers: self.providers().providers,
1382 backends: self.backends().backends,
1383 models: self.models().models,
1384 capabilities: self.capability_registry.list(),
1385 }
1386 }
1387
1388 pub fn capability(&self, id: &str) -> Option<Capability> {
1389 self.capability_registry.get(id)
1390 }
1391
1392 pub fn capability_registry(&self) -> &CapabilityRegistry {
1393 &self.capability_registry
1394 }
1395
1396 pub fn load_capability_packages(
1400 &mut self,
1401 root: impl AsRef<std::path::Path>,
1402 ) -> RuntimeResult<Vec<String>> {
1403 self.capability_registry.load_local_packages(root)
1404 }
1405
1406 pub fn environment(&self) -> RuntimeEnvironment {
1407 RuntimeEnvironment::discover(
1408 self.capability_registry
1409 .providers()
1410 .iter()
1411 .map(|provider| provider.id().to_owned())
1412 .collect(),
1413 )
1414 }
1415
1416 pub fn resolve_capability(
1417 &self,
1418 request: &ExecutionRequest,
1419 ) -> RuntimeResult<CapabilityResolution> {
1420 self.resolve_capability_with_constraints(request, &ResolutionConstraints::default())
1421 }
1422
1423 pub fn resolve_capability_with_constraints(
1424 &self,
1425 request: &ExecutionRequest,
1426 constraints: &ResolutionConstraints,
1427 ) -> RuntimeResult<CapabilityResolution> {
1428 self.capability_registry
1429 .resolve(&request.capability, constraints)
1430 }
1431
1432 pub async fn execute_capability(
1433 &self,
1434 request: ExecutionRequest,
1435 authorizer: Option<&dyn CapabilityAuthorizer>,
1436 ) -> RuntimeResult<ExecutionResult> {
1437 let resolution = self
1438 .capability_registry
1439 .resolve(&request.capability, &ResolutionConstraints::default())?;
1440 let provider = self
1441 .capability_registry
1442 .provider(&resolution.provider)
1443 .ok_or_else(|| {
1444 RuntimeError::provider_unavailable(&resolution.provider, "not registered")
1445 })?;
1446 if resolution.provider == "builtin" {
1447 capability::execute(request, authorizer).await
1448 } else {
1449 if let Some(authorizer) = authorizer {
1450 if request.policy.require_authorization {
1451 let capability = self
1452 .capability_registry
1453 .get(&request.capability)
1454 .ok_or_else(|| RuntimeError::CapabilityMismatch {
1455 reason: format!("unknown capability {}", request.capability),
1456 })?;
1457 authorizer.authorize(&request, &capability).await?;
1458 }
1459 } else if self
1460 .capability_registry
1461 .get(&request.capability)
1462 .is_some_and(|capability| !capability.requirements.is_empty())
1463 {
1464 return Err(RuntimeError::Execution {
1465 operation: request.capability,
1466 reason: "authorization required".to_owned(),
1467 });
1468 }
1469 provider.execute(request).await
1470 }
1471 }
1472
1473 pub fn status(&self) -> RuntimeStatus {
1474 RuntimeStatus {
1475 loaded_models: self
1476 .loaded_models
1477 .read()
1478 .expect("loaded model registry poisoned")
1479 .len(),
1480 max_models: self.config.max_models,
1481 max_concurrent_inferences: self.config.max_concurrent_inferences,
1482 max_concurrent_model_loads: self.config.max_concurrent_model_loads,
1483 }
1484 }
1485
1486 pub fn metrics(&self) -> RuntimeMetricsSnapshot {
1487 self.metrics.snapshot()
1488 }
1489
1490 pub fn executions(&self) -> Vec<ExecutionRecord> {
1491 self.execution_records
1492 .lock()
1493 .expect("execution records poisoned")
1494 .iter()
1495 .cloned()
1496 .collect()
1497 }
1498
1499 pub fn resources(&self) -> Vec<ModelResource> {
1500 self.loaded_models
1501 .read()
1502 .expect("loaded model registry poisoned")
1503 .values()
1504 .map(|model| ModelResource {
1505 id: model.spec.id.clone(),
1506 version: model.spec.version.clone(),
1507 target: format!("{}/{}", model.provider_name, model.backend_name),
1508 state: ModelLifecycleState::Ready,
1509 memory_bytes: model.memory_bytes,
1510 last_used: model.last_used,
1511 })
1512 .collect()
1513 }
1514
1515 pub fn snapshot(&self) -> RuntimeSnapshot {
1516 let resources = self.resources();
1517 let metrics = self.metrics();
1518 RuntimeSnapshot {
1519 active_inferences: metrics.active_inferences,
1520 active_model_loads: metrics.active_model_loads,
1521 queued_inferences: (metrics.queued_requests as usize)
1522 .saturating_sub(metrics.active_inferences),
1523 loaded_models: resources.len(),
1524 cache_entries: resources.len(),
1525 cache_memory_bytes: resources.iter().map(|resource| resource.memory_bytes).sum(),
1526 metrics,
1527 }
1528 }
1529
1530 pub fn selection_for(&self, model: &ModelSpec) -> RuntimeResult<RuntimeSelection> {
1531 let backend_name = self.select_backend(model)?;
1532 let backend = self
1533 .backends
1534 .get(&backend_name)
1535 .ok_or_else(|| RuntimeError::backend_unavailable(&backend_name, "not registered"))?;
1536 Ok(RuntimeSelection {
1537 model: model.id.clone(),
1538 provider: "local".to_owned(),
1539 backend: Some(backend_name),
1540 hardware: backend.capabilities().hardware,
1541 fallback: false,
1542 fallback_from: None,
1543 reason: "preferred local execution".to_owned(),
1544 policy: ExecutionPolicy::LocalOnly,
1545 })
1546 }
1547
1548 pub async fn load(&self, model: ModelSpec) -> RuntimeResult<RuntimeModelHandle> {
1551 if self
1552 .config
1553 .selected_provider
1554 .as_deref()
1555 .is_some_and(|provider| provider != "local")
1556 {
1557 return Err(RuntimeError::provider_unavailable(
1558 self.config
1559 .selected_provider
1560 .clone()
1561 .unwrap_or_else(|| "remote".to_owned()),
1562 "load is only supported for local execution paths",
1563 ));
1564 }
1565
1566 let backend_name = self.select_backend(&model)?;
1567 let key = ModelCacheKey {
1568 id: model.id.clone(),
1569 version: model.version.clone(),
1570 provider: "local".to_owned(),
1571 backend: backend_name.clone(),
1572 };
1573 if let Some(existing) = self
1574 .loaded_models
1575 .read()
1576 .expect("loaded model registry poisoned")
1577 .get(&key)
1578 {
1579 self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1580 return Ok(runtime_handle(existing));
1581 }
1582 self.metrics.cache_misses.fetch_add(1, Ordering::Relaxed);
1583 self.metrics
1584 .active_model_loads
1585 .fetch_add(1, Ordering::Relaxed);
1586 let _permit = self
1587 .load_semaphore
1588 .acquire()
1589 .await
1590 .map_err(|_| RuntimeError::execution("load", "runtime load semaphore closed"))?;
1591 if let Some(existing) = self
1592 .loaded_models
1593 .read()
1594 .expect("loaded model registry poisoned")
1595 .get(&key)
1596 {
1597 self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1598 return Ok(runtime_handle(existing));
1599 }
1600 let backend = self
1601 .backends
1602 .get(&backend_name)
1603 .ok_or_else(|| RuntimeError::backend_unavailable(&backend_name, "not registered"))?
1604 .clone();
1605 let timeout_ms = self.config.timeout_ms;
1606 let loaded = run_with_timeout(timeout_ms, "load", backend.load(&model)).await;
1607 self.metrics
1608 .active_model_loads
1609 .fetch_sub(1, Ordering::Relaxed);
1610 let handle = match loaded {
1611 Ok(handle) => handle,
1612 Err(error) => {
1613 self.metrics
1614 .model_load_failures
1615 .fetch_add(1, Ordering::Relaxed);
1616 return Err(error);
1617 }
1618 };
1619
1620 let mut cache = self
1621 .loaded_models
1622 .write()
1623 .expect("loaded model registry poisoned");
1624 if cache.len() >= self.config.max_models {
1625 if let Some(oldest) = cache
1626 .iter()
1627 .min_by_key(|(_, model)| model.last_used)
1628 .map(|(key, _)| key.clone())
1629 {
1630 cache.remove(&oldest);
1631 self.metrics.evictions.fetch_add(1, Ordering::Relaxed);
1632 }
1633 }
1634 self.metrics.model_loads.fetch_add(1, Ordering::Relaxed);
1635 let loaded = LoadedModel {
1636 memory_bytes: model_memory_bytes(&model),
1637 spec: model,
1638 handle,
1639 backend_name,
1640 provider_name: "local".to_owned(),
1641 last_used: self.cache_clock.fetch_add(1, Ordering::Relaxed),
1642 };
1643 let runtime_handle = runtime_handle(&loaded);
1644 cache.insert(key, loaded);
1645 Ok(runtime_handle)
1646 }
1647
1648 pub async fn load_reference(
1649 &self,
1650 reference: ModelReference,
1651 ) -> RuntimeResult<RuntimeModelHandle> {
1652 match reference {
1653 ModelReference::Spec(spec) => self.load(spec).await,
1654 ModelReference::Id { id, version } => self
1655 .find_loaded(&id, version.as_ref())
1656 .map(|loaded| runtime_handle(&loaded)),
1657 }
1658 }
1659
1660 pub async fn infer(&self, request: InferenceRequest) -> RuntimeResult<InferenceResult> {
1662 self.infer_with_cancellation(request, CancellationToken::new())
1663 .await
1664 }
1665
1666 pub async fn infer_loaded(
1668 &self,
1669 model: &RuntimeModelHandle,
1670 input: Input,
1671 options: InferenceOptions,
1672 ) -> RuntimeResult<InferenceResult> {
1673 self.infer(InferenceRequest {
1674 model: ModelReference::id(model.id().to_owned(), model.version().map(str::to_owned)),
1675 input,
1676 options,
1677 })
1678 .await
1679 }
1680
1681 pub async fn infer_loaded_stream(
1682 &self,
1683 model: &RuntimeModelHandle,
1684 input: Input,
1685 mut options: InferenceOptions,
1686 ) -> RuntimeResult<InferenceStream> {
1687 options.stream = true;
1688 self.infer_stream(InferenceRequest {
1689 model: ModelReference::id(model.id().to_owned(), model.version().map(str::to_owned)),
1690 input,
1691 options,
1692 })
1693 .await
1694 }
1695
1696 pub async fn infer_batch(
1701 &self,
1702 mut requests: Vec<InferenceRequest>,
1703 ) -> RuntimeResult<Vec<InferenceResult>> {
1704 if requests.is_empty() {
1705 return Ok(Vec::new());
1706 }
1707 if let Some(limit) = self.config.max_batch_size {
1708 if requests.len() > limit {
1709 return Err(RuntimeError::ResourceLimit {
1710 reason: format!(
1711 "batch size {} exceeds configured limit {limit}",
1712 requests.len()
1713 ),
1714 });
1715 }
1716 }
1717 for request in &requests {
1718 self.validate_request(request)?;
1719 }
1720 for request in &mut requests {
1721 self.resolve_catalog_reference_for_local(request).await?;
1722 }
1723 let selection = self.select_for_request(&requests[0])?;
1724 let same_model = requests
1725 .iter()
1726 .all(|request| model_id(&request.model) == model_id(&requests[0].model));
1727 if selection.provider != "local" || !same_model {
1728 return futures_util::future::try_join_all(
1729 requests.into_iter().map(|request| self.infer(request)),
1730 )
1731 .await;
1732 }
1733 let loaded = self.resolve_loaded_model(&requests[0].model).await?;
1734 let backend = self
1735 .backends
1736 .get(&loaded.backend_name)
1737 .ok_or_else(|| {
1738 RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
1739 })?
1740 .clone();
1741 if !backend.capabilities().batching {
1742 return futures_util::future::try_join_all(
1743 requests.into_iter().map(|request| self.infer(request)),
1744 )
1745 .await;
1746 }
1747 let batch_size = requests.len();
1748 self.metrics
1749 .total_requests
1750 .fetch_add(batch_size as u64, Ordering::Relaxed);
1751 self.metrics
1752 .batch_executions
1753 .fetch_add(1, Ordering::Relaxed);
1754 self.metrics
1755 .total_requests_batched
1756 .fetch_add(batch_size as u64, Ordering::Relaxed);
1757 self.metrics
1758 .batch_sizes
1759 .lock()
1760 .expect("metrics poisoned")
1761 .push(batch_size);
1762 let _permit = self.inference_semaphore.acquire().await.map_err(|_| {
1763 RuntimeError::execution("infer_batch", "runtime inference semaphore closed")
1764 })?;
1765 let mut results = run_with_timeout(
1766 self.config.timeout_ms,
1767 "infer_batch",
1768 backend.infer_batch(&loaded.handle, &requests, Some(CancellationToken::new())),
1769 )
1770 .await?;
1771 if results.len() != batch_size {
1772 return Err(RuntimeError::execution(
1773 "infer_batch",
1774 format!(
1775 "backend returned {} results for {batch_size} requests",
1776 results.len()
1777 ),
1778 ));
1779 }
1780 for (result, request) in results.iter_mut().zip(requests.iter()) {
1781 let output = result.output.clone();
1782 decorate_metadata(
1783 &mut result.metadata,
1784 &output,
1785 &loaded.spec.id,
1786 "local",
1787 &loaded.backend_name,
1788 backend.capabilities().hardware.clone(),
1789 Duration::ZERO,
1790 estimate_input_tokens(&request.input),
1791 loaded.spec.version.clone(),
1792 selection.policy.clone(),
1793 false,
1794 None,
1795 Some(selection.reason.clone()),
1796 );
1797 result.metadata.batch_size = batch_size;
1798 result.metadata.cache_hit = true;
1799 }
1800 self.metrics
1801 .successful_requests
1802 .fetch_add(results.len() as u64, Ordering::Relaxed);
1803 Ok(results)
1804 }
1805
1806 pub async fn infer_with_cancellation(
1809 &self,
1810 request: InferenceRequest,
1811 cancellation: CancellationToken,
1812 ) -> RuntimeResult<InferenceResult> {
1813 let started = Instant::now();
1814 let request_id = format!(
1815 "request-{}",
1816 self.execution_clock.fetch_add(1, Ordering::Relaxed) + 1
1817 );
1818 self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
1819 self.metrics
1820 .active_inferences
1821 .fetch_add(1, Ordering::Relaxed);
1822 let model = model_details(&request.model);
1823 let result = self.infer_inner(request.clone(), cancellation).await;
1824 self.metrics
1825 .active_inferences
1826 .fetch_sub(1, Ordering::Relaxed);
1827 match result {
1828 Ok(mut result) => {
1829 result.metadata.request_id = Some(request_id.clone());
1830 self.record_execution(
1831 &request_id,
1832 started.elapsed(),
1833 &result.metadata,
1834 ExecutionCompletion::Succeeded,
1835 );
1836 self.metrics
1837 .successful_requests
1838 .fetch_add(1, Ordering::Relaxed);
1839 Ok(result)
1840 }
1841 Err(error) => {
1842 let completion = if matches!(error, RuntimeError::Cancelled) {
1843 self.metrics
1844 .cancelled_requests
1845 .fetch_add(1, Ordering::Relaxed);
1846 ExecutionCompletion::Cancelled
1847 } else {
1848 self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
1849 ExecutionCompletion::Failed
1850 };
1851 let metadata = ExecutionMetadata {
1852 request_id: Some(request_id.clone()),
1853 model: model.0,
1854 model_version: model.1,
1855 routing_policy: request.options.execution.clone(),
1856 streaming_requested: request.options.stream,
1857 ..ExecutionMetadata::default()
1858 };
1859 self.record_execution(&request_id, started.elapsed(), &metadata, completion);
1860 Err(error)
1861 }
1862 }
1863 }
1864
1865 async fn infer_inner(
1866 &self,
1867 mut request: InferenceRequest,
1868 cancellation: CancellationToken,
1869 ) -> RuntimeResult<InferenceResult> {
1870 self.validate_request(&request)?;
1871 if cancellation.is_cancelled() {
1872 return Err(RuntimeError::Cancelled);
1873 }
1874
1875 let _permit = tokio::select! {
1876 permit = self.inference_semaphore.acquire() => permit
1877 .map_err(|_| RuntimeError::execution("infer", "runtime inference semaphore closed"))?,
1878 _ = cancellation.cancelled() => return Err(RuntimeError::Cancelled),
1879 };
1880
1881 self.resolve_catalog_reference_for_local(&mut request)
1882 .await?;
1883
1884 let mut selection = self.select_for_request(&request)?;
1885 let timeout_ms = request.options.timeout_ms.or(self.config.timeout_ms);
1886 let input_tokens = estimate_input_tokens(&request.input);
1887
1888 match selection.provider.as_str() {
1889 "local" => {
1890 let cache_hit = self.is_loaded(&request.model);
1891 let model_load_started = Instant::now();
1892 if cache_hit {
1893 self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1894 } else {
1895 self.metrics.cache_misses.fetch_add(1, Ordering::Relaxed);
1896 }
1897
1898 let loaded = match self.resolve_loaded_model(&request.model).await {
1899 Ok(loaded) => loaded,
1900 Err(error) if self.can_fallback(&request, &selection, &error) => {
1901 selection = self.remote_selection(
1902 model_id(&request.model),
1903 true,
1904 Some("local model unavailable".to_owned()),
1905 &request.options,
1906 )?;
1907 return self
1908 .infer_remote(
1909 request,
1910 cancellation,
1911 timeout_ms,
1912 input_tokens,
1913 selection,
1914 )
1915 .await;
1916 }
1917 Err(error) => return Err(error),
1918 };
1919 let backend = self
1920 .backends
1921 .get(&loaded.backend_name)
1922 .ok_or_else(|| {
1923 RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
1924 })?
1925 .clone();
1926 let started = Instant::now();
1927 let result = run_with_timeout(
1928 timeout_ms,
1929 "infer",
1930 backend.infer(&loaded.handle, &request, Some(cancellation.clone())),
1931 )
1932 .await;
1933 let mut result = match result {
1934 Ok(result) => result,
1935 Err(error) if self.can_fallback(&request, &selection, &error) => {
1936 selection = self.remote_selection(
1937 model_id(&request.model),
1938 true,
1939 Some(error.to_string()),
1940 &request.options,
1941 )?;
1942 return self
1943 .infer_remote(
1944 request,
1945 cancellation,
1946 timeout_ms,
1947 input_tokens,
1948 selection,
1949 )
1950 .await;
1951 }
1952 Err(error) => return Err(error),
1953 };
1954 result.metadata.model_load_ms = Some(if cache_hit {
1955 0.0
1956 } else {
1957 model_load_started.elapsed().as_secs_f64() * 1000.0
1958 });
1959 let capabilities = backend.capabilities();
1960 let output = result.output.clone();
1961 decorate_metadata(
1962 &mut result.metadata,
1963 &output,
1964 &loaded.spec.id,
1965 "local",
1966 &loaded.backend_name,
1967 capabilities.hardware,
1968 started.elapsed(),
1969 input_tokens,
1970 loaded.spec.version.clone(),
1971 selection.policy.clone(),
1972 selection.fallback,
1973 selection.fallback_from.clone(),
1974 Some(selection.reason.clone()),
1975 );
1976 result.metadata.cache_hit = cache_hit;
1977 Ok(result)
1978 }
1979 _ => {
1980 self.infer_remote(request, cancellation, timeout_ms, input_tokens, selection)
1981 .await
1982 }
1983 }
1984 }
1985
1986 fn record_execution(
1987 &self,
1988 request_id: &str,
1989 total_duration: Duration,
1990 metadata: &ExecutionMetadata,
1991 completion: ExecutionCompletion,
1992 ) {
1993 let execution_id = format!(
1994 "execution-{}",
1995 self.execution_clock.fetch_add(1, Ordering::Relaxed) + 1
1996 );
1997 let execution_duration =
1998 Duration::from_secs_f64(metadata.execution_ms.unwrap_or(0.0) / 1000.0);
1999 let queued_duration =
2000 Duration::from_secs_f64(metadata.queue_wait_ms.unwrap_or(0.0) / 1000.0);
2001 let model_load_duration =
2002 Duration::from_secs_f64(metadata.model_load_ms.unwrap_or(0.0) / 1000.0);
2003 self.metrics
2004 .queue_wait_micros
2005 .lock()
2006 .expect("metrics poisoned")
2007 .push(queued_duration.as_micros() as u64);
2008 self.metrics
2009 .model_load_micros
2010 .lock()
2011 .expect("metrics poisoned")
2012 .push(model_load_duration.as_micros() as u64);
2013 self.metrics
2014 .execution_micros
2015 .lock()
2016 .expect("metrics poisoned")
2017 .push(execution_duration.as_micros() as u64);
2018 self.metrics
2019 .total_duration_micros
2020 .lock()
2021 .expect("metrics poisoned")
2022 .push(total_duration.as_micros() as u64);
2023 if metadata.provider == "local" {
2024 self.metrics
2025 .local_executions
2026 .fetch_add(1, Ordering::Relaxed);
2027 } else {
2028 self.metrics
2029 .remote_executions
2030 .fetch_add(1, Ordering::Relaxed);
2031 }
2032 if metadata.fallback {
2033 self.metrics
2034 .fallback_executions
2035 .fetch_add(1, Ordering::Relaxed);
2036 }
2037 if metadata.streaming_requested {
2038 self.metrics
2039 .streaming_requests
2040 .fetch_add(1, Ordering::Relaxed);
2041 }
2042 let record = ExecutionRecord {
2043 execution_id,
2044 request_id: request_id.to_owned(),
2045 model_id: metadata.model.clone(),
2046 model_version: metadata.model_version.clone().unwrap_or_default(),
2047 provider: metadata.provider.clone(),
2048 backend: metadata.backend.clone(),
2049 target: metadata.execution_target.clone(),
2050 routing_policy: metadata.routing_policy.clone(),
2051 fallback: metadata.fallback_from.clone(),
2052 cache_hit: metadata.cache_hit,
2053 batch_size: metadata.batch_size.max(1),
2054 queued_duration,
2055 model_load_duration,
2056 execution_duration,
2057 total_duration,
2058 streaming: metadata.streaming_requested,
2059 output_event_count: metadata.output_event_count,
2060 completion,
2061 };
2062 let mut records = self
2063 .execution_records
2064 .lock()
2065 .expect("execution records poisoned");
2066 records.push_back(record);
2067 while records.len() > 256 {
2068 records.pop_front();
2069 }
2070 }
2071
2072 async fn infer_remote(
2073 &self,
2074 request: InferenceRequest,
2075 cancellation: CancellationToken,
2076 timeout_ms: Option<u64>,
2077 input_tokens: Option<u64>,
2078 selection: RuntimeSelection,
2079 ) -> RuntimeResult<InferenceResult> {
2080 let provider_name = selection.provider.as_str();
2081 let provider = self
2082 .providers
2083 .get(provider_name)
2084 .ok_or_else(|| RuntimeError::provider_unavailable(provider_name, "not registered"))?
2085 .clone();
2086 let started = Instant::now();
2087 let mut result = run_with_timeout(
2088 timeout_ms,
2089 "infer",
2090 provider.infer(request.clone(), Some(cancellation)),
2091 )
2092 .await?;
2093 let backend_name = result.metadata.backend.clone();
2094 let hardware = result.metadata.hardware.clone();
2095 let output = result.output.clone();
2096 let (model, version) = model_details(&request.model);
2097 decorate_metadata(
2098 &mut result.metadata,
2099 &output,
2100 &model,
2101 provider_name,
2102 backend_name.as_str(),
2103 hardware,
2104 started.elapsed(),
2105 input_tokens,
2106 version,
2107 selection.policy,
2108 selection.fallback,
2109 selection.fallback_from,
2110 Some(selection.reason),
2111 );
2112 Ok(result)
2113 }
2114
2115 pub async fn infer_stream(&self, request: InferenceRequest) -> RuntimeResult<InferenceStream> {
2117 self.infer_stream_with_cancellation(request, CancellationToken::new())
2118 .await
2119 }
2120
2121 pub async fn infer_stream_with_cancellation(
2123 &self,
2124 mut request: InferenceRequest,
2125 cancellation: CancellationToken,
2126 ) -> RuntimeResult<InferenceStream> {
2127 self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
2128 self.metrics
2129 .streaming_requests
2130 .fetch_add(1, Ordering::Relaxed);
2131 self.validate_request(&request)?;
2132 if !request.options.stream {
2133 return Err(RuntimeError::CapabilityMismatch {
2134 reason: "infer_stream requires InferenceOptions::stream to be true".to_owned(),
2135 });
2136 }
2137 self.resolve_catalog_reference_for_local(&mut request)
2138 .await?;
2139 let selection = self.select_for_request(&request)?;
2140 let buffer = self.config.max_stream_buffer;
2141 if buffer == 0 {
2142 return Err(RuntimeError::ResourceLimit {
2143 reason: "max_stream_buffer must be greater than zero".to_owned(),
2144 });
2145 }
2146 match selection.provider.as_str() {
2147 "local" => {
2148 let loaded = self.resolve_loaded_model(&request.model).await?;
2149 let backend = self
2150 .backends
2151 .get(&loaded.backend_name)
2152 .ok_or_else(|| {
2153 RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
2154 })?
2155 .clone();
2156 let capabilities = backend.capabilities();
2157 let stream = if capabilities.streaming {
2158 backend
2159 .infer_stream(&loaded.handle, &request, Some(cancellation.clone()))
2160 .await?
2161 } else {
2162 self.metrics
2163 .compatibility_streams
2164 .fetch_add(1, Ordering::Relaxed);
2165 let result = backend
2166 .infer(&loaded.handle, &request, Some(cancellation.clone()))
2167 .await?;
2168 Box::pin(futures_util::stream::iter(vec![Ok(InferenceChunk {
2169 output: result.output,
2170 done: true,
2171 metadata: Some(result.metadata),
2172 })])) as BoxStream<_>
2173 };
2174 Ok(normalize_stream(
2175 stream,
2176 request,
2177 selection,
2178 capabilities.streaming,
2179 true,
2180 buffer,
2181 cancellation,
2182 self.metrics.clone(),
2183 ))
2184 }
2185 provider_name => {
2186 let provider = self
2187 .providers
2188 .get(provider_name)
2189 .ok_or_else(|| {
2190 RuntimeError::provider_unavailable(provider_name, "not registered")
2191 })?
2192 .clone();
2193 let capabilities = provider.capabilities();
2194 let stream = if capabilities.streaming {
2195 provider
2196 .infer_stream(request.clone(), Some(cancellation.clone()))
2197 .await?
2198 } else {
2199 self.metrics
2200 .compatibility_streams
2201 .fetch_add(1, Ordering::Relaxed);
2202 let result = provider
2203 .infer(request.clone(), Some(cancellation.clone()))
2204 .await?;
2205 Box::pin(futures_util::stream::iter(vec![Ok(InferenceChunk {
2206 output: result.output,
2207 done: true,
2208 metadata: Some(result.metadata),
2209 })])) as BoxStream<_>
2210 };
2211 Ok(normalize_stream(
2212 stream,
2213 request,
2214 selection,
2215 capabilities.streaming,
2216 false,
2217 buffer,
2218 cancellation,
2219 self.metrics.clone(),
2220 ))
2221 }
2222 }
2223 }
2224
2225 fn validate_request(&self, request: &InferenceRequest) -> RuntimeResult<()> {
2226 if request.options.require_local && request.options.require_remote {
2227 return Err(RuntimeError::InvalidRequest {
2228 reason: "request cannot require both local and remote execution".to_owned(),
2229 });
2230 }
2231 if let Some(limit) = self.config.max_batch_size {
2232 let batch_size = request
2233 .options
2234 .batch_size
2235 .unwrap_or_else(|| request.input.batch_size());
2236 if batch_size > limit {
2237 return Err(RuntimeError::ResourceLimit {
2238 reason: format!("batch size {batch_size} exceeds configured limit {limit}"),
2239 });
2240 }
2241 }
2242 Ok(())
2243 }
2244
2245 async fn resolve_catalog_reference_for_local(
2246 &self,
2247 request: &mut InferenceRequest,
2248 ) -> RuntimeResult<()> {
2249 let local_resolution = request.options.require_local
2250 || (!request.options.require_remote
2251 && matches!(
2252 request.options.execution,
2253 ExecutionPolicy::LocalOnly
2254 | ExecutionPolicy::PreferLocal
2255 | ExecutionPolicy::LocalThenRemote
2256 ));
2257 if local_resolution && !self.is_loaded(&request.model) {
2258 if let (Some(catalog), ModelReference::Id { .. }) = (&self.catalog, &request.model) {
2259 let descriptor = catalog
2260 .resolve(&request.model)
2261 .await
2262 .map_err(catalog_error)?;
2263 catalog
2264 .validate(&descriptor.id)
2265 .await
2266 .map_err(catalog_error)?;
2267 request.model = ModelReference::Spec(descriptor.spec());
2268 }
2269 }
2270 Ok(())
2271 }
2272
2273 async fn resolve_loaded_model(&self, reference: &ModelReference) -> RuntimeResult<LoadedModel> {
2274 match reference {
2275 ModelReference::Id { id, version } => self.find_loaded(id, version.as_ref()),
2276 ModelReference::Spec(spec) => {
2277 if let Ok(existing) = self.find_loaded(&spec.id, spec.version.as_ref()) {
2278 return Ok(existing);
2279 }
2280 self.load(spec.clone()).await?;
2281 self.find_loaded(&spec.id, spec.version.as_ref())
2282 }
2283 }
2284 }
2285
2286 fn find_loaded(&self, id: &str, version: Option<&String>) -> RuntimeResult<LoadedModel> {
2287 let mut cache = self
2288 .loaded_models
2289 .write()
2290 .expect("loaded model registry poisoned");
2291 let found = cache.iter_mut().find(|(key, loaded)| {
2292 key.id == id
2293 && version.is_none_or(|version| loaded.spec.version.as_ref() == Some(version))
2294 });
2295 if let Some((_, loaded)) = found {
2296 loaded.last_used = self.cache_clock.fetch_add(1, Ordering::Relaxed);
2297 return Ok(loaded.clone());
2298 }
2299
2300 Err(RuntimeError::ModelNotFound {
2301 model: id.to_owned(),
2302 })
2303 }
2304
2305 fn is_loaded(&self, reference: &ModelReference) -> bool {
2306 let (id, version) = model_details(reference);
2307 self.loaded_models
2308 .read()
2309 .expect("loaded model registry poisoned")
2310 .keys()
2311 .any(|key| {
2312 key.id == id
2313 && version
2314 .as_ref()
2315 .is_none_or(|v| key.version.as_ref() == Some(v))
2316 })
2317 }
2318
2319 fn select_for_request(&self, request: &InferenceRequest) -> RuntimeResult<RuntimeSelection> {
2320 let model_id = model_id(&request.model);
2321 let policy = if request.options.require_remote {
2322 ExecutionPolicy::RemoteOnly
2323 } else if request.options.require_local {
2324 ExecutionPolicy::LocalOnly
2325 } else if self.config.allow_remote_fallback {
2326 ExecutionPolicy::LocalThenRemote
2327 } else {
2328 request.options.execution.clone()
2329 };
2330 match policy {
2331 ExecutionPolicy::RemoteOnly => self.remote_selection(
2332 model_id,
2333 false,
2334 Some("remote-only policy".to_owned()),
2335 &request.options,
2336 ),
2337 ExecutionPolicy::LocalOnly => self.local_selection(&request.model, policy),
2338 ExecutionPolicy::PreferLocal => match self
2339 .local_selection(&request.model, policy.clone())
2340 {
2341 Ok(selection) => Ok(selection),
2342 Err(error) => {
2343 self.remote_selection(model_id, true, Some(error.to_string()), &request.options)
2344 }
2345 },
2346 ExecutionPolicy::LocalThenRemote => match self
2347 .local_selection(&request.model, policy.clone())
2348 {
2349 Ok(selection) => Ok(selection),
2350 Err(error) => {
2351 self.remote_selection(model_id, true, Some(error.to_string()), &request.options)
2352 }
2353 },
2354 ExecutionPolicy::PreferRemote | ExecutionPolicy::RemoteThenLocal => match self
2355 .remote_selection(
2356 model_id.clone(),
2357 false,
2358 Some("preferred remote execution".to_owned()),
2359 &request.options,
2360 ) {
2361 Ok(selection) => Ok(selection),
2362 Err(_) => self.local_selection(&request.model, policy),
2363 },
2364 }
2365 }
2366
2367 fn local_selection(
2368 &self,
2369 reference: &ModelReference,
2370 policy: ExecutionPolicy,
2371 ) -> RuntimeResult<RuntimeSelection> {
2372 match reference {
2373 ModelReference::Id { id, version } => {
2374 let loaded = self.find_loaded(id, version.as_ref())?;
2375 Ok(RuntimeSelection {
2376 model: loaded.spec.id,
2377 provider: "local".to_owned(),
2378 backend: Some(loaded.backend_name.clone()),
2379 hardware: self
2380 .backends
2381 .get(&loaded.backend_name)
2382 .and_then(|backend| backend.capabilities().hardware),
2383 fallback: false,
2384 fallback_from: None,
2385 reason: "preferred local execution".to_owned(),
2386 policy,
2387 })
2388 }
2389 ModelReference::Spec(spec) => {
2390 let backend = self.select_backend(spec)?;
2391 let hardware = self
2392 .backends
2393 .get(&backend)
2394 .and_then(|b| b.capabilities().hardware);
2395 Ok(RuntimeSelection {
2396 model: spec.id.clone(),
2397 provider: "local".to_owned(),
2398 backend: Some(backend),
2399 hardware,
2400 fallback: false,
2401 fallback_from: None,
2402 reason: "preferred local execution".to_owned(),
2403 policy,
2404 })
2405 }
2406 }
2407 }
2408
2409 fn remote_selection(
2410 &self,
2411 model: String,
2412 fallback: bool,
2413 reason: Option<String>,
2414 options: &InferenceOptions,
2415 ) -> RuntimeResult<RuntimeSelection> {
2416 let selected = if let Some(provider_name) = self.config.selected_provider.as_deref() {
2417 if provider_name != "local" {
2418 Some(provider_name.to_owned())
2419 } else {
2420 None
2421 }
2422 } else {
2423 None
2424 }
2425 .or_else(|| {
2426 self.providers.iter().find_map(|(name, provider)| {
2427 let capabilities = provider.capabilities();
2428 if capabilities.available && capabilities.remote {
2429 Some(name.clone())
2430 } else {
2431 None
2432 }
2433 })
2434 })
2435 .ok_or_else(|| {
2436 RuntimeError::provider_unavailable("remote", "no remote provider registered")
2437 })?;
2438
2439 Ok(RuntimeSelection {
2440 model,
2441 provider: selected,
2442 backend: None,
2443 hardware: None,
2444 fallback,
2445 fallback_from: fallback.then(|| "local".to_owned()),
2446 reason: reason.unwrap_or_else(|| "remote execution selected by policy".to_owned()),
2447 policy: if fallback && options.execution == ExecutionPolicy::LocalOnly {
2448 ExecutionPolicy::LocalThenRemote
2449 } else {
2450 options.execution.clone()
2451 },
2452 })
2453 }
2454
2455 fn can_fallback(
2456 &self,
2457 request: &InferenceRequest,
2458 selection: &RuntimeSelection,
2459 error: &RuntimeError,
2460 ) -> bool {
2461 matches!(
2462 selection.policy,
2463 ExecutionPolicy::LocalThenRemote | ExecutionPolicy::PreferLocal
2464 ) && !request.options.require_local
2465 && matches!(
2466 error,
2467 RuntimeError::ProviderUnavailable { .. }
2468 | RuntimeError::ModelUnavailable { .. }
2469 | RuntimeError::BackendUnavailable { .. }
2470 | RuntimeError::ModelNotFound { .. }
2471 | RuntimeError::CapabilityMismatch { .. }
2472 | RuntimeError::Timeout { .. }
2473 | RuntimeError::Transport { .. }
2474 | RuntimeError::Protocol { .. }
2475 )
2476 }
2477
2478 fn select_backend(&self, model: &ModelSpec) -> RuntimeResult<String> {
2479 if let Some(selected) = self.config.selected_backend.as_deref() {
2480 let backend = self
2481 .backends
2482 .get(selected)
2483 .ok_or_else(|| RuntimeError::backend_unavailable(selected, "not registered"))?;
2484 let capabilities = backend.capabilities();
2485 if !capabilities.available {
2486 return Err(RuntimeError::backend_unavailable(
2487 selected,
2488 "backend is not available on this platform",
2489 ));
2490 }
2491 if !backend.supports(model) {
2492 return Err(RuntimeError::UnsupportedModel {
2493 model: model.id.clone(),
2494 backend: selected.to_owned(),
2495 reason: "model format is not supported".to_owned(),
2496 });
2497 }
2498 return Ok(selected.to_owned());
2499 }
2500
2501 let mut candidates: Vec<_> = self
2502 .backends
2503 .iter()
2504 .filter_map(|(name, backend)| {
2505 let capabilities = backend.capabilities();
2506 if capabilities.available && backend.supports(model) {
2507 Some((
2508 name.clone(),
2509 !capabilities.accelerators.is_empty(),
2510 capabilities.hardware.clone(),
2511 ))
2512 } else {
2513 None
2514 }
2515 })
2516 .collect();
2517
2518 candidates.sort_by(|left, right| left.0.cmp(&right.0));
2519 if self.config.prefer_acceleration {
2520 candidates
2521 .sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0)));
2522 }
2523
2524 candidates
2525 .into_iter()
2526 .next()
2527 .map(|candidate| candidate.0)
2528 .ok_or_else(|| {
2529 RuntimeError::backend_unavailable("local", "no compatible backend registered")
2530 })
2531 }
2532}
2533
2534async fn run_with_timeout<F, T>(
2535 timeout_ms: Option<u64>,
2536 operation: &str,
2537 future: F,
2538) -> RuntimeResult<T>
2539where
2540 F: std::future::Future<Output = RuntimeResult<T>>,
2541{
2542 if let Some(timeout_ms) = timeout_ms {
2543 timeout(Duration::from_millis(timeout_ms), future)
2544 .await
2545 .map_err(|_| RuntimeError::Timeout {
2546 operation: operation.to_owned(),
2547 })?
2548 } else {
2549 future.await
2550 }
2551}
2552
2553fn model_id(reference: &ModelReference) -> String {
2554 match reference {
2555 ModelReference::Id { id, .. } => id.clone(),
2556 ModelReference::Spec(spec) => spec.id.clone(),
2557 }
2558}
2559
2560fn runtime_handle(loaded: &LoadedModel) -> RuntimeModelHandle {
2561 RuntimeModelHandle {
2562 model_id: loaded.spec.id.clone(),
2563 version: loaded.spec.version.clone(),
2564 provider: loaded.provider_name.clone(),
2565 backend: loaded.backend_name.clone(),
2566 state: ModelLifecycleState::Ready,
2567 }
2568}
2569
2570fn model_details(reference: &ModelReference) -> (String, Option<String>) {
2571 match reference {
2572 ModelReference::Id { id, version } => (id.clone(), version.clone()),
2573 ModelReference::Spec(spec) => (spec.id.clone(), spec.version.clone()),
2574 }
2575}
2576
2577fn catalog_error(error: CatalogError) -> RuntimeError {
2578 match error {
2579 CatalogError::ModelNotFound(model) => RuntimeError::ModelNotFound { model },
2580 CatalogError::InvalidIdentity(model) => RuntimeError::InvalidRequest {
2581 reason: format!("invalid model identity: {model}"),
2582 },
2583 CatalogError::ArtifactIntegrity { model, reason } => {
2584 RuntimeError::ModelIntegrity { model, reason }
2585 }
2586 other => RuntimeError::Execution {
2587 operation: "catalog".to_owned(),
2588 reason: other.to_string(),
2589 },
2590 }
2591}
2592
2593fn estimate_input_tokens(input: &Input) -> Option<u64> {
2594 match input {
2595 Input::Text(text) => Some(text.split_whitespace().count() as u64),
2596 Input::Tokens(tokens) => Some(tokens.len() as u64),
2597 _ => None,
2598 }
2599}
2600
2601fn estimate_output_tokens(output: &Output) -> Option<u64> {
2602 match output {
2603 Output::Text(text) => Some(text.split_whitespace().count() as u64),
2604 Output::Tokens(tokens) => Some(tokens.len() as u64),
2605 _ => None,
2606 }
2607}
2608
2609fn model_memory_bytes(model: &ModelSpec) -> u64 {
2610 match &model.location {
2611 ModelLocation::Path(path) => std::fs::metadata(path)
2612 .map(|metadata| metadata.len())
2613 .unwrap_or(0),
2614 _ => 0,
2615 }
2616}
2617
2618#[allow(clippy::too_many_arguments)]
2619fn decorate_metadata(
2620 metadata: &mut ExecutionMetadata,
2621 output: &Output,
2622 model: &str,
2623 provider: &str,
2624 backend: &str,
2625 hardware: Option<String>,
2626 elapsed: Duration,
2627 input_tokens: Option<u64>,
2628 model_version: Option<String>,
2629 routing_policy: ExecutionPolicy,
2630 fallback: bool,
2631 fallback_from: Option<String>,
2632 fallback_reason: Option<String>,
2633) {
2634 metadata.model = model.to_owned();
2635 metadata.model_version = model_version;
2636 metadata.provider = provider.to_owned();
2637 metadata.backend = backend.to_owned();
2638 metadata.execution_target = provider.to_owned();
2639 metadata.routing_policy = routing_policy;
2640 metadata.fallback = fallback;
2641 metadata.fallback_from = fallback_from;
2642 metadata.fallback_reason = fallback_reason;
2643 metadata.hardware = hardware;
2644 metadata.latency_ms = Some(elapsed.as_secs_f64() * 1000.0);
2645 metadata.execution_ms = metadata.latency_ms;
2646 metadata.input_tokens = input_tokens;
2647 metadata.output_tokens = estimate_output_tokens(output);
2648}
2649
2650#[allow(clippy::too_many_arguments)]
2651fn normalize_stream(
2652 mut source: BoxStream<RuntimeResult<InferenceChunk>>,
2653 request: InferenceRequest,
2654 selection: RuntimeSelection,
2655 streaming_supported: bool,
2656 cache_hit: bool,
2657 buffer: usize,
2658 cancellation: CancellationToken,
2659 metrics: Arc<RuntimeMetrics>,
2660) -> InferenceStream {
2661 let (sender, receiver) = mpsc::channel(buffer);
2662 tokio::spawn(async move {
2663 let (model, version) = model_details(&request.model);
2664 let mut metadata = ExecutionMetadata {
2665 model,
2666 model_version: version,
2667 provider: selection.provider.clone(),
2668 execution_target: selection.provider.clone(),
2669 routing_policy: selection.policy.clone(),
2670 fallback: selection.fallback,
2671 fallback_from: selection.fallback_from.clone(),
2672 fallback_reason: Some(selection.reason.clone()),
2673 streaming_requested: true,
2674 streaming_supported,
2675 cache_hit,
2676 completion_state: Some("started".to_owned()),
2677 ..ExecutionMetadata::default()
2678 };
2679 if sender
2680 .send(Ok(InferenceStreamEvent::Started(metadata.clone())))
2681 .await
2682 .is_err()
2683 {
2684 return;
2685 }
2686
2687 let mut output_count = 0;
2688 while let Some(item) = tokio::select! {
2689 item = source.next() => item,
2690 _ = cancellation.cancelled() => {
2691 metrics.cancelled_streams.fetch_add(1, Ordering::Relaxed);
2692 let _ = sender.send(Err(RuntimeError::Cancelled)).await;
2693 return;
2694 }
2695 } {
2696 match item {
2697 Ok(chunk) => {
2698 if let Some(chunk_metadata) = chunk.metadata {
2699 metadata = chunk_metadata;
2700 }
2701 metadata.streaming_requested = true;
2702 metadata.streaming_supported = streaming_supported;
2703 metadata.cache_hit = cache_hit;
2704 metadata.output_event_count = output_count + 1;
2705 output_count += 1;
2706 if sender
2707 .send(Ok(InferenceStreamEvent::Output(chunk.output)))
2708 .await
2709 .is_err()
2710 {
2711 return;
2712 }
2713 metrics
2714 .streamed_output_events
2715 .fetch_add(1, Ordering::Relaxed);
2716 if chunk.done {
2717 metadata.output_event_count = output_count;
2718 metadata.completion_state = Some("completed".to_owned());
2719 let _ = sender
2720 .send(Ok(InferenceStreamEvent::Completed(metadata)))
2721 .await;
2722 return;
2723 }
2724 }
2725 Err(error) => {
2726 let _ = sender.send(Err(error)).await;
2727 return;
2728 }
2729 }
2730 }
2731 metadata.output_event_count = output_count;
2732 metadata.completion_state = Some("completed".to_owned());
2733 let _ = sender
2734 .send(Ok(InferenceStreamEvent::Completed(metadata)))
2735 .await;
2736 });
2737 Box::pin(ReceiverStream::new(receiver))
2738}
2739
2740#[cfg(test)]
2741mod tests {
2742 use super::*;
2743 use async_trait::async_trait;
2744 use futures_util::StreamExt;
2745 use ml_runtime_backend::BackendCapabilities;
2746 use ml_runtime_common::BoxStream;
2747 use ml_runtime_cpu_backend::CpuBackend;
2748 use ml_runtime_inference::InferenceOptions;
2749 use ml_runtime_provider::{Provider, ProviderCapabilities};
2750 use serde_json::json;
2751 use tokio_stream::iter;
2752
2753 #[derive(Clone)]
2754 struct BlockingBackend {
2755 started: Arc<Semaphore>,
2756 }
2757
2758 #[async_trait]
2759 impl Backend for BlockingBackend {
2760 fn name(&self) -> &str {
2761 "blocking"
2762 }
2763
2764 fn capabilities(&self) -> BackendCapabilities {
2765 BackendCapabilities {
2766 available: true,
2767 local: true,
2768 streaming: true,
2769 cancellation: true,
2770 supported_formats: vec![ModelFormat::Unknown],
2771 ..BackendCapabilities::default()
2772 }
2773 }
2774
2775 fn supports(&self, model: &ModelSpec) -> bool {
2776 model.format == ModelFormat::Unknown
2777 }
2778
2779 async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle> {
2780 Ok(ModelHandle::new(
2781 model.id.clone(),
2782 self.name(),
2783 model.format.clone(),
2784 Arc::new(()),
2785 ))
2786 }
2787
2788 async fn infer(
2789 &self,
2790 _model: &ModelHandle,
2791 _request: &InferenceRequest,
2792 cancellation: Option<CancellationToken>,
2793 ) -> RuntimeResult<InferenceResult> {
2794 self.started.add_permits(1);
2795 cancellation
2796 .expect("runtime supplies cancellation")
2797 .cancelled()
2798 .await;
2799 Err(RuntimeError::Cancelled)
2800 }
2801
2802 async fn infer_stream(
2803 &self,
2804 _model: &ModelHandle,
2805 _request: &InferenceRequest,
2806 _cancellation: Option<CancellationToken>,
2807 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
2808 Ok(Box::pin(futures_util::stream::pending()))
2809 }
2810 }
2811
2812 #[test]
2813 fn runtime_version_is_workspace_version() {
2814 assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
2815 }
2816
2817 #[derive(Clone)]
2818 struct MockRemoteProvider;
2819
2820 #[async_trait]
2821 impl Provider for MockRemoteProvider {
2822 fn name(&self) -> &str {
2823 "mock-remote"
2824 }
2825
2826 fn capabilities(&self) -> ProviderCapabilities {
2827 ProviderCapabilities {
2828 available: true,
2829 local: false,
2830 remote: true,
2831 streaming: true,
2832 cancellation: true,
2833 structured_output: true,
2834 batching: true,
2835 endpoint: Some("mock://remote".to_owned()),
2836 notes: vec!["test remote provider".to_owned()],
2837 }
2838 }
2839
2840 async fn infer(
2841 &self,
2842 request: InferenceRequest,
2843 cancellation: Option<CancellationToken>,
2844 ) -> RuntimeResult<InferenceResult> {
2845 if model_id(&request.model) == "blocking-remote" {
2846 cancellation
2847 .expect("runtime supplies cancellation")
2848 .cancelled()
2849 .await;
2850 return Err(RuntimeError::Cancelled);
2851 }
2852 if cancellation
2853 .as_ref()
2854 .is_some_and(CancellationToken::is_cancelled)
2855 {
2856 return Err(RuntimeError::Cancelled);
2857 }
2858 Ok(InferenceResult {
2859 output: Output::Structured(json!({"remote": model_id(&request.model)})),
2860 metadata: ExecutionMetadata {
2861 backend: "remote-api".to_owned(),
2862 hardware: Some("network".to_owned()),
2863 ..ExecutionMetadata::default()
2864 },
2865 })
2866 }
2867
2868 async fn infer_stream(
2869 &self,
2870 request: InferenceRequest,
2871 _cancellation: Option<CancellationToken>,
2872 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
2873 let stream = iter(vec![Ok(InferenceChunk {
2874 output: Output::Text(model_id(&request.model)),
2875 done: true,
2876 metadata: None,
2877 })]);
2878 Ok(Box::pin(stream))
2879 }
2880 }
2881
2882 #[tokio::test]
2883 async fn loads_and_runs_local_cpu_models() {
2884 let runtime = Runtime::builder()
2885 .register_backend(CpuBackend::default())
2886 .build();
2887 let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
2888 runtime.load(model.clone()).await.unwrap();
2889
2890 let result = runtime
2891 .infer(InferenceRequest {
2892 model: model.into(),
2893 input: Input::Text("hello runtime".to_owned()),
2894 options: InferenceOptions::default(),
2895 })
2896 .await
2897 .unwrap();
2898
2899 assert_eq!(result.output, Output::Text("hello runtime".to_owned()));
2900 assert_eq!(result.metadata.provider, "local");
2901 assert_eq!(result.metadata.backend, "cpu");
2902 }
2903
2904 #[tokio::test]
2905 async fn supports_remote_provider_selection() {
2906 let runtime = Runtime::builder()
2907 .register_backend(CpuBackend::default())
2908 .register_provider(MockRemoteProvider)
2909 .provider("mock-remote")
2910 .build();
2911
2912 let result = runtime
2913 .infer(InferenceRequest {
2914 model: ModelReference::id("remote-model", None),
2915 input: Input::Text("hello".to_owned()),
2916 options: InferenceOptions {
2917 require_remote: true,
2918 ..InferenceOptions::default()
2919 },
2920 })
2921 .await
2922 .unwrap();
2923
2924 assert_eq!(result.metadata.provider, "mock-remote");
2925 assert_eq!(result.metadata.backend, "remote-api");
2926 }
2927
2928 #[tokio::test]
2929 async fn only_falls_back_to_remote_when_enabled() {
2930 let request = InferenceRequest {
2931 model: ModelReference::id("missing", None),
2932 input: Input::Text("hello".to_owned()),
2933 options: InferenceOptions::default(),
2934 };
2935
2936 let local_only = Runtime::builder()
2937 .register_backend(CpuBackend::default())
2938 .register_provider(MockRemoteProvider)
2939 .build();
2940 assert!(matches!(
2941 local_only.infer(request.clone()).await,
2942 Err(RuntimeError::ModelNotFound { .. })
2943 ));
2944
2945 let hybrid = Runtime::builder()
2946 .register_backend(CpuBackend::default())
2947 .register_provider(MockRemoteProvider)
2948 .allow_remote_fallback(true)
2949 .build();
2950 let result = hybrid.infer(request).await.unwrap();
2951 assert_eq!(result.metadata.provider, "mock-remote");
2952 assert!(result.metadata.fallback);
2953 assert_eq!(result.metadata.fallback_from.as_deref(), Some("local"));
2954 }
2955
2956 #[tokio::test]
2957 async fn explicit_policies_choose_deterministic_targets() {
2958 let local = Runtime::builder()
2959 .register_backend(CpuBackend::default())
2960 .register_provider(MockRemoteProvider)
2961 .build();
2962 let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
2963 local.load(model.clone()).await.unwrap();
2964
2965 let local_result = local
2966 .infer(InferenceRequest {
2967 model: model.clone().into(),
2968 input: Input::Text("hello".to_owned()),
2969 options: InferenceOptions {
2970 execution: ExecutionPolicy::PreferLocal,
2971 ..InferenceOptions::default()
2972 },
2973 })
2974 .await
2975 .unwrap();
2976 assert_eq!(local_result.metadata.provider, "local");
2977
2978 let remote_result = local
2979 .infer(InferenceRequest {
2980 model: ModelReference::id("remote-model", None),
2981 input: Input::Text("hello".to_owned()),
2982 options: InferenceOptions {
2983 execution: ExecutionPolicy::PreferRemote,
2984 ..InferenceOptions::default()
2985 },
2986 })
2987 .await
2988 .unwrap();
2989 assert_eq!(remote_result.metadata.provider, "mock-remote");
2990 assert_eq!(
2991 remote_result.metadata.routing_policy,
2992 ExecutionPolicy::PreferRemote
2993 );
2994 }
2995
2996 #[tokio::test]
2997 async fn rejects_requests_above_batch_limit() {
2998 let runtime = Runtime::builder()
2999 .register_backend(CpuBackend::default())
3000 .max_batch_size(2)
3001 .build();
3002
3003 let err = runtime
3004 .infer(InferenceRequest {
3005 model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3006 input: Input::Tokens(vec![1, 2, 3]),
3007 options: InferenceOptions {
3008 batch_size: Some(3),
3009 ..InferenceOptions::default()
3010 },
3011 })
3012 .await
3013 .unwrap_err();
3014
3015 assert!(matches!(err, RuntimeError::ResourceLimit { .. }));
3016 }
3017
3018 #[tokio::test]
3019 async fn propagates_cancellation() {
3020 let runtime = Runtime::builder()
3021 .register_backend(CpuBackend::default())
3022 .build();
3023 let token = CancellationToken::new();
3024 token.cancel();
3025
3026 let err = runtime
3027 .infer_with_cancellation(
3028 InferenceRequest {
3029 model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory)
3030 .into(),
3031 input: Input::Text("hello".to_owned()),
3032 options: InferenceOptions::default(),
3033 },
3034 token,
3035 )
3036 .await
3037 .unwrap_err();
3038
3039 assert!(matches!(err, RuntimeError::Cancelled));
3040 }
3041
3042 #[tokio::test]
3043 async fn cancels_active_and_queued_inference() {
3044 let started = Arc::new(Semaphore::new(0));
3045 let runtime = Arc::new(
3046 Runtime::builder()
3047 .register_backend(BlockingBackend {
3048 started: Arc::clone(&started),
3049 })
3050 .backend("blocking")
3051 .concurrency(1)
3052 .build(),
3053 );
3054 let make_request = || {
3055 InferenceRequest::new(
3056 ModelSpec::new("blocking", ModelFormat::Unknown, ModelLocation::Memory),
3057 "input",
3058 )
3059 };
3060
3061 let active_token = CancellationToken::new();
3062 let active = {
3063 let runtime = Arc::clone(&runtime);
3064 let token = active_token.clone();
3065 tokio::spawn(
3066 async move { runtime.infer_with_cancellation(make_request(), token).await },
3067 )
3068 };
3069 let permit = started.acquire().await.unwrap();
3070 permit.forget();
3071
3072 let queued_token = CancellationToken::new();
3073 let queued = {
3074 let runtime = Arc::clone(&runtime);
3075 let token = queued_token.clone();
3076 tokio::spawn(
3077 async move { runtime.infer_with_cancellation(make_request(), token).await },
3078 )
3079 };
3080 tokio::task::yield_now().await;
3081 queued_token.cancel();
3082 assert!(matches!(
3083 queued.await.unwrap(),
3084 Err(RuntimeError::Cancelled)
3085 ));
3086
3087 active_token.cancel();
3088 assert!(matches!(
3089 active.await.unwrap(),
3090 Err(RuntimeError::Cancelled)
3091 ));
3092 }
3093
3094 #[tokio::test]
3095 async fn cancels_an_active_stream() {
3096 let runtime = Runtime::builder()
3097 .register_backend(BlockingBackend {
3098 started: Arc::new(Semaphore::new(0)),
3099 })
3100 .backend("blocking")
3101 .build();
3102 let token = CancellationToken::new();
3103 let mut stream = runtime
3104 .infer_stream_with_cancellation(
3105 InferenceRequest::new(
3106 ModelSpec::new("blocking", ModelFormat::Unknown, ModelLocation::Memory),
3107 "input",
3108 )
3109 .with_options(InferenceOptions {
3110 stream: true,
3111 ..InferenceOptions::default()
3112 }),
3113 token.clone(),
3114 )
3115 .await
3116 .unwrap();
3117 assert!(matches!(
3118 stream.next().await.unwrap().unwrap(),
3119 InferenceStreamEvent::Started(_)
3120 ));
3121 token.cancel();
3122 assert!(matches!(
3123 stream.next().await.unwrap(),
3124 Err(RuntimeError::Cancelled)
3125 ));
3126 }
3127
3128 #[tokio::test]
3129 async fn propagates_remote_cancellation() {
3130 let runtime = Arc::new(
3131 Runtime::builder()
3132 .register_provider(MockRemoteProvider)
3133 .provider("mock-remote")
3134 .build(),
3135 );
3136 let token = CancellationToken::new();
3137 let task = {
3138 let runtime = Arc::clone(&runtime);
3139 let token = token.clone();
3140 tokio::spawn(async move {
3141 runtime
3142 .infer_with_cancellation(
3143 InferenceRequest::new(ModelReference::latest("blocking-remote"), "input")
3144 .with_options(InferenceOptions {
3145 execution: ExecutionPolicy::RemoteOnly,
3146 ..InferenceOptions::default()
3147 }),
3148 token,
3149 )
3150 .await
3151 })
3152 };
3153 tokio::task::yield_now().await;
3154 token.cancel();
3155 assert!(matches!(task.await.unwrap(), Err(RuntimeError::Cancelled)));
3156 }
3157
3158 #[tokio::test]
3159 async fn streams_from_local_backend() {
3160 let runtime = Runtime::builder()
3161 .register_backend(CpuBackend::default())
3162 .build();
3163
3164 let mut stream = runtime
3165 .infer_stream(InferenceRequest {
3166 model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3167 input: Input::Text("hello runtime".to_owned()),
3168 options: InferenceOptions {
3169 stream: true,
3170 ..InferenceOptions::default()
3171 },
3172 })
3173 .await
3174 .unwrap();
3175
3176 assert!(matches!(
3177 stream.next().await.unwrap().unwrap(),
3178 InferenceStreamEvent::Started(_)
3179 ));
3180 assert!(matches!(
3181 stream.next().await.unwrap().unwrap(),
3182 InferenceStreamEvent::Output(Output::Text(text)) if text == "hello"
3183 ));
3184 assert!(matches!(
3185 stream.next().await.unwrap().unwrap(),
3186 InferenceStreamEvent::Output(Output::Text(text)) if text == "runtime"
3187 ));
3188 assert!(matches!(
3189 stream.next().await.unwrap().unwrap(),
3190 InferenceStreamEvent::Completed(metadata)
3191 if metadata.output_event_count == 2 && metadata.streaming_supported
3192 ));
3193 }
3194
3195 #[tokio::test]
3196 async fn rejects_stream_calls_without_stream_flag() {
3197 let runtime = Runtime::builder()
3198 .register_backend(CpuBackend::default())
3199 .build();
3200 let result = runtime
3201 .infer_stream(InferenceRequest {
3202 model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3203 input: Input::Text("hello".to_owned()),
3204 options: InferenceOptions::default(),
3205 })
3206 .await;
3207
3208 assert!(matches!(
3209 result,
3210 Err(RuntimeError::CapabilityMismatch { .. })
3211 ));
3212 }
3213
3214 #[tokio::test]
3215 async fn streams_from_remote_provider() {
3216 let runtime = Runtime::builder()
3217 .register_backend(CpuBackend::default())
3218 .register_provider(MockRemoteProvider)
3219 .provider("mock-remote")
3220 .build();
3221
3222 let mut stream = runtime
3223 .infer_stream(InferenceRequest {
3224 model: ModelReference::id("remote-model", None),
3225 input: Input::Text("hello".to_owned()),
3226 options: InferenceOptions {
3227 stream: true,
3228 require_remote: true,
3229 ..InferenceOptions::default()
3230 },
3231 })
3232 .await
3233 .unwrap();
3234
3235 assert!(matches!(
3236 stream.next().await.unwrap().unwrap(),
3237 InferenceStreamEvent::Started(_)
3238 ));
3239 assert!(matches!(
3240 stream.next().await.unwrap().unwrap(),
3241 InferenceStreamEvent::Output(Output::Text(text)) if text == "remote-model"
3242 ));
3243 assert!(matches!(
3244 stream.next().await.unwrap().unwrap(),
3245 InferenceStreamEvent::Completed(metadata)
3246 if metadata.output_event_count == 1
3247 ));
3248 }
3249
3250 #[tokio::test]
3251 async fn explicit_handles_reuse_loaded_models_and_report_cache_hits() {
3252 let runtime = Runtime::builder()
3253 .register_backend(CpuBackend::default())
3254 .build();
3255 let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
3256 let handle = runtime.load(model).await.unwrap();
3257 assert_eq!(handle.id(), "echo");
3258 let result = runtime
3259 .infer_loaded(
3260 &handle,
3261 Input::Text("cached".to_owned()),
3262 Default::default(),
3263 )
3264 .await
3265 .unwrap();
3266 assert!(result.metadata.cache_hit);
3267 assert_eq!(runtime.models().list().len(), 1);
3268 }
3269
3270 #[tokio::test]
3271 async fn loaded_stream_reuses_the_cached_model() {
3272 let runtime = Runtime::builder()
3273 .register_backend(CpuBackend::default())
3274 .build();
3275 let handle = runtime
3276 .load(ModelSpec::new(
3277 "echo",
3278 ModelFormat::Unknown,
3279 ModelLocation::Memory,
3280 ))
3281 .await
3282 .unwrap();
3283 let mut stream = runtime
3284 .infer_loaded_stream(
3285 &handle,
3286 Input::Text("cached stream".to_owned()),
3287 InferenceOptions::default(),
3288 )
3289 .await
3290 .unwrap();
3291 let events = futures_util::StreamExt::collect::<Vec<_>>(&mut stream).await;
3292 assert!(matches!(
3293 events.first().and_then(|event| event.as_ref().ok()),
3294 Some(InferenceStreamEvent::Started(_))
3295 ));
3296 assert!(matches!(
3297 events.last().and_then(|event| event.as_ref().ok()),
3298 Some(InferenceStreamEvent::Completed(metadata)) if metadata.cache_hit
3299 ));
3300 assert_eq!(runtime.models().list().len(), 1);
3301 }
3302
3303 #[tokio::test]
3304 async fn cache_evicts_the_least_recently_used_target() {
3305 let runtime = Runtime::builder()
3306 .register_backend(CpuBackend::default())
3307 .max_models(1)
3308 .build();
3309 runtime
3310 .load(ModelSpec::new(
3311 "first",
3312 ModelFormat::Unknown,
3313 ModelLocation::Memory,
3314 ))
3315 .await
3316 .unwrap();
3317 runtime
3318 .load(ModelSpec::new(
3319 "second",
3320 ModelFormat::Unknown,
3321 ModelLocation::Memory,
3322 ))
3323 .await
3324 .unwrap();
3325 assert_eq!(runtime.models().list().len(), 1);
3326 assert_eq!(runtime.models().list()[0].id, "second");
3327 }
3328
3329 #[tokio::test]
3330 async fn batch_inference_preserves_result_order_and_identity() {
3331 let runtime = Runtime::builder()
3332 .register_backend(CpuBackend::default())
3333 .build();
3334 let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
3335 let results = runtime
3336 .infer_batch(vec![
3337 InferenceRequest {
3338 model: model.clone().into(),
3339 input: Input::Text("a".to_owned()),
3340 options: Default::default(),
3341 },
3342 InferenceRequest {
3343 model: model.into(),
3344 input: Input::Text("b".to_owned()),
3345 options: Default::default(),
3346 },
3347 ])
3348 .await
3349 .unwrap();
3350 assert_eq!(results.len(), 2);
3351 assert_eq!(results[0].output, Output::Text("a".to_owned()));
3352 assert_eq!(results[1].output, Output::Text("b".to_owned()));
3353 assert_eq!(results[0].metadata.batch_size, 2);
3354 assert_eq!(results[1].metadata.batch_size, 2);
3355 }
3356
3357 #[tokio::test]
3358 async fn records_execution_metrics_and_snapshot() {
3359 let runtime = Runtime::builder()
3360 .register_backend(CpuBackend::default())
3361 .build();
3362 let model = ModelSpec::new("observed", ModelFormat::Unknown, ModelLocation::Memory);
3363 let result = runtime
3364 .infer(InferenceRequest {
3365 model: model.into(),
3366 input: Input::Text("hello".to_owned()),
3367 options: Default::default(),
3368 })
3369 .await
3370 .unwrap();
3371
3372 let metrics = runtime.metrics();
3373 assert_eq!(metrics.total_requests, 1);
3374 assert_eq!(metrics.successful_requests, 1);
3375 assert_eq!(runtime.executions().len(), 1);
3376 assert_eq!(
3377 runtime.executions()[0].completion,
3378 ExecutionCompletion::Succeeded
3379 );
3380 assert_eq!(
3381 runtime.executions()[0].request_id,
3382 result.metadata.request_id.clone().unwrap()
3383 );
3384 assert_eq!(runtime.snapshot().loaded_models, 1);
3385 }
3386}