1#![cfg_attr(test, allow(clippy::default_constructed_unit_structs))]
2
3use futures_util::StreamExt;
4use ml_runtime_backend::{Backend, BackendCapability};
5use ml_runtime_common::BoxStream;
6use ml_runtime_cpu_backend::CpuBackend;
7use ml_runtime_http_provider::HttpProvider;
8use ml_runtime_inference::InferenceChunk;
9use ml_runtime_model::{CatalogError, ModelCatalog, ModelHandle};
10use ml_runtime_onnx_backend::OnnxBackend;
11use ml_runtime_provider::{Provider, ProviderCapability};
12use serde::{Deserialize, Serialize};
13use std::{
14 collections::{BTreeMap, HashMap, VecDeque},
15 hash::{Hash, Hasher},
16 path::{Path, PathBuf},
17 sync::{
18 atomic::{AtomicU64, AtomicUsize, Ordering},
19 Arc, Mutex, RwLock,
20 },
21 time::{Duration, Instant},
22};
23use tokio::{
24 sync::{mpsc, Semaphore},
25 time::timeout,
26};
27use tokio_stream::wrappers::ReceiverStream;
28
29mod capability;
30mod decision;
31mod generation;
32
33pub use capability::{
34 AvailabilityState, BuiltinCapabilityProvider, Capability, CapabilityAuthorizer,
35 CapabilityAvailability, CapabilityExecutionMetadata, CapabilityLimits,
36 CapabilityPackageManifest, CapabilityPolicy, CapabilityProvider, CapabilityRegistry,
37 CapabilityResolution, ExecutionRequest, ExecutionResult, LocalProviderPackage,
38 ManifestCapabilityProvider, PackageCapability, PackageManifest, ProviderDescriptor,
39 ProviderManifest, ResolutionCandidate, ResolutionConstraints,
40};
41pub use decision::DecisionPlanner;
42
43pub use ml_runtime_backend;
44pub use ml_runtime_backend::{DecisionModel, DecisionModelProvider};
45pub use ml_runtime_common;
46pub use ml_runtime_common::{CancellationToken, RuntimeError, RuntimeResult};
47pub use ml_runtime_http_provider::openai::{
48 ApiKey, OpenAiCompatibleConfig, OpenAiCompatibleProvider,
49};
50pub use ml_runtime_inference;
51pub use ml_runtime_inference::{
52 DecisionDependency, DecisionDependencyCondition, DecisionExecution,
53 DecisionExecutionDiagnostics, DecisionExecutionPlan, DecisionExecutionPolicy,
54 DecisionExecutionStrategy, DecisionGraphRequest, DecisionModelCapabilities, DecisionNode,
55 DecisionOption, DecisionPlanStage, DecisionProvenance, DecisionQuestion, DecisionRequest,
56 DecisionResult, DecisionType, DecisionValue, DeviceKind, ExecutionMetadata, ExecutionPolicy,
57 InferenceOptions, InferenceRequest, InferenceResult, InferenceStreamEvent, Input,
58 ModelArchitecture, ModelDescription, ModelIdentity, Output, PlannedDecisionResult, Tensor,
59 TypedDecision,
60};
61pub use ml_runtime_model;
62pub use ml_runtime_model::{
63 default_model_root, BuiltinModelRegistry, CompiledArtifactManifest, CompiledStatus,
64 FilesystemModelCatalog, InstallationManifest, ModelDescriptor, ModelFormat, ModelId,
65 ModelLocation, ModelReference, ModelSpec, RegisteredModelInstaller,
66};
67pub use ml_runtime_provider;
68pub use ml_runtime_provider::{
69 ChatMessage, ChatMetadata, ChatRequest, ChatResponse, ChatRole, ChatStreamEvent,
70 ChatStreamSummary, ExecutionLocation, FinishReason, GenerationCapabilities,
71 GenerationModelInfo, GenerationOptions, GenerationProvider, ResponseFormat, StructuredOutput,
72 ToolCall, ToolChoice, ToolDefinition, Usage,
73};
74
75pub const VERSION: &str = env!("CARGO_PKG_VERSION");
76
77#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
78#[serde(rename_all = "snake_case")]
79pub enum InstalledModelStatus {
80 NotInstalled,
81 InstalledNotCompiled,
82 Ready,
83 Corrupt,
84 Incompatible,
85 Unsupported,
86}
87
88impl std::fmt::Display for InstalledModelStatus {
89 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
90 formatter.write_str(match self {
91 Self::NotInstalled => "not_installed",
92 Self::InstalledNotCompiled => "installed_not_compiled",
93 Self::Ready => "ready",
94 Self::Corrupt => "corrupt",
95 Self::Incompatible => "incompatible",
96 Self::Unsupported => "unsupported",
97 })
98 }
99}
100
101#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
102pub struct InstalledModelRecord {
103 pub model: String,
104 pub revision: String,
105 pub backend: String,
106 pub platform: String,
107 pub status: InstalledModelStatus,
108 pub compiled: bool,
109 pub package_path: PathBuf,
110}
111
112#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
113pub struct ModelDiagnostic {
114 pub check: String,
115 pub ok: bool,
116 pub message: String,
117}
118
119#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
120pub struct ModelDoctorReport {
121 pub model: InstalledModelRecord,
122 pub diagnostics: Vec<ModelDiagnostic>,
123 pub manifest: Option<InstallationManifest>,
124}
125
126#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
127pub struct ModelLifecycleResult {
128 pub model: InstalledModelRecord,
129 pub already_installed: bool,
130 pub duration_ms: u128,
131}
132
133pub type InferenceStream = BoxStream<RuntimeResult<InferenceStreamEvent>>;
135
136#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
137pub enum Platform {
138 Linux,
139 Macos,
140 Windows,
141 Unknown,
142}
143
144impl Platform {
145 fn current() -> Self {
146 if cfg!(target_os = "linux") {
147 Self::Linux
148 } else if cfg!(target_os = "macos") {
149 Self::Macos
150 } else if cfg!(target_os = "windows") {
151 Self::Windows
152 } else {
153 Self::Unknown
154 }
155 }
156}
157
158#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
159pub struct ModelCapability {
160 pub id: String,
161 pub format: ModelFormat,
162 pub location: ModelLocation,
163 pub loaded: bool,
164 pub provider: Option<String>,
165 pub backend: Option<String>,
166}
167
168#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
169pub struct RuntimeCapabilities {
170 pub environment: RuntimeEnvironment,
171 pub platforms: Vec<Platform>,
172 pub providers: Vec<ProviderCapability>,
173 pub backends: Vec<BackendCapability>,
174 pub models: Vec<ModelCapability>,
175 pub capabilities: Vec<Capability>,
176}
177
178#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
179pub struct RuntimeEnvironment {
180 pub platform: String,
181 pub architecture: String,
182 pub operating_system: String,
183 pub cpu: String,
184 pub memory_bytes: Option<u64>,
185 pub accelerators: Vec<String>,
186 pub filesystem: String,
187 pub network: String,
188 pub developer_tools: Vec<String>,
189 pub installed_providers: Vec<String>,
190}
191
192impl RuntimeEnvironment {
193 pub fn discover(provider_ids: Vec<String>) -> Self {
194 Self {
195 platform: std::env::consts::OS.to_owned(),
196 architecture: std::env::consts::ARCH.to_owned(),
197 operating_system: std::env::consts::OS.to_owned(),
198 cpu: std::thread::available_parallelism()
199 .map(|count| format!("{} logical cores", count.get()))
200 .unwrap_or_else(|_| "unknown".to_owned()),
201 memory_bytes: available_memory_bytes(),
202 accelerators: Vec::new(),
203 filesystem: std::env::current_dir()
204 .map(|path| path.display().to_string())
205 .unwrap_or_else(|_| "unavailable".to_owned()),
206 network: "available by explicit policy".to_owned(),
207 developer_tools: ["git"]
208 .iter()
209 .filter(|tool| command_available(tool))
210 .map(|tool| (*tool).to_owned())
211 .collect(),
212 installed_providers: provider_ids,
213 }
214 }
215}
216
217fn command_available(command: &str) -> bool {
218 std::process::Command::new(command)
219 .arg("--version")
220 .output()
221 .map(|output| output.status.success())
222 .unwrap_or(false)
223}
224
225fn available_memory_bytes() -> Option<u64> {
226 let contents = std::fs::read_to_string("/proc/meminfo").ok()?;
227 contents.lines().find_map(|line| {
228 let value = line
229 .strip_prefix("MemTotal:")?
230 .split_whitespace()
231 .next()?
232 .parse::<u64>()
233 .ok()?;
234 Some(value * 1024)
235 })
236}
237
238#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
239pub struct RuntimeStatus {
240 pub loaded_models: usize,
241 pub max_models: usize,
242 pub max_concurrent_inferences: usize,
243 pub max_concurrent_model_loads: usize,
244}
245
246pub type ExecutionId = String;
247pub type RequestId = String;
248
249#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
250pub enum ExecutionCompletion {
251 Succeeded,
252 Failed,
253 Cancelled,
254}
255
256#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
257pub struct ExecutionRecord {
258 pub execution_id: ExecutionId,
259 pub request_id: RequestId,
260 pub model_id: String,
261 pub model_version: String,
262 pub provider: String,
263 pub backend: String,
264 pub target: String,
265 pub routing_policy: ExecutionPolicy,
266 pub fallback: Option<String>,
267 pub cache_hit: bool,
268 pub batch_size: usize,
269 pub queued_duration: Duration,
270 pub model_load_duration: Duration,
271 pub execution_duration: Duration,
272 pub total_duration: Duration,
273 pub streaming: bool,
274 pub output_event_count: usize,
275 pub completion: ExecutionCompletion,
276}
277
278#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
279pub struct RuntimeMetricsSnapshot {
280 pub total_requests: u64,
281 pub successful_requests: u64,
282 pub failed_requests: u64,
283 pub cancelled_requests: u64,
284 pub local_executions: u64,
285 pub remote_executions: u64,
286 pub fallback_executions: u64,
287 pub fallback_failures: u64,
288 pub model_loads: u64,
289 pub model_load_failures: u64,
290 pub cache_hits: u64,
291 pub cache_misses: u64,
292 pub evictions: u64,
293 pub queued_requests: u64,
294 pub active_inferences: usize,
295 pub active_model_loads: usize,
296 pub batch_executions: u64,
297 pub individual_executions: u64,
298 pub total_requests_batched: u64,
299 pub streaming_requests: u64,
300 pub streamed_output_events: u64,
301 pub cancelled_streams: u64,
302 pub compatibility_streams: u64,
303 pub queue_wait_micros: Vec<u64>,
304 pub model_load_micros: Vec<u64>,
305 pub execution_micros: Vec<u64>,
306 pub total_duration_micros: Vec<u64>,
307 pub batch_sizes: Vec<usize>,
308}
309
310#[derive(Default)]
311pub struct RuntimeMetrics {
312 total_requests: AtomicU64,
313 successful_requests: AtomicU64,
314 failed_requests: AtomicU64,
315 cancelled_requests: AtomicU64,
316 local_executions: AtomicU64,
317 remote_executions: AtomicU64,
318 fallback_executions: AtomicU64,
319 fallback_failures: AtomicU64,
320 model_loads: AtomicU64,
321 model_load_failures: AtomicU64,
322 cache_hits: AtomicU64,
323 cache_misses: AtomicU64,
324 evictions: AtomicU64,
325 queued_requests: AtomicU64,
326 active_inferences: AtomicUsize,
327 active_model_loads: AtomicUsize,
328 batch_executions: AtomicU64,
329 individual_executions: AtomicU64,
330 total_requests_batched: AtomicU64,
331 streaming_requests: AtomicU64,
332 streamed_output_events: AtomicU64,
333 cancelled_streams: AtomicU64,
334 compatibility_streams: AtomicU64,
335 queue_wait_micros: Mutex<Vec<u64>>,
336 model_load_micros: Mutex<Vec<u64>>,
337 execution_micros: Mutex<Vec<u64>>,
338 total_duration_micros: Mutex<Vec<u64>>,
339 batch_sizes: Mutex<Vec<usize>>,
340}
341
342impl RuntimeMetrics {
343 pub fn snapshot(&self) -> RuntimeMetricsSnapshot {
344 let values = |values: &Mutex<Vec<u64>>| values.lock().expect("metrics poisoned").clone();
345 RuntimeMetricsSnapshot {
346 total_requests: self.total_requests.load(Ordering::Relaxed),
347 successful_requests: self.successful_requests.load(Ordering::Relaxed),
348 failed_requests: self.failed_requests.load(Ordering::Relaxed),
349 cancelled_requests: self.cancelled_requests.load(Ordering::Relaxed),
350 local_executions: self.local_executions.load(Ordering::Relaxed),
351 remote_executions: self.remote_executions.load(Ordering::Relaxed),
352 fallback_executions: self.fallback_executions.load(Ordering::Relaxed),
353 fallback_failures: self.fallback_failures.load(Ordering::Relaxed),
354 model_loads: self.model_loads.load(Ordering::Relaxed),
355 model_load_failures: self.model_load_failures.load(Ordering::Relaxed),
356 cache_hits: self.cache_hits.load(Ordering::Relaxed),
357 cache_misses: self.cache_misses.load(Ordering::Relaxed),
358 evictions: self.evictions.load(Ordering::Relaxed),
359 queued_requests: self.queued_requests.load(Ordering::Relaxed),
360 active_inferences: self.active_inferences.load(Ordering::Relaxed),
361 active_model_loads: self.active_model_loads.load(Ordering::Relaxed),
362 batch_executions: self.batch_executions.load(Ordering::Relaxed),
363 individual_executions: self.individual_executions.load(Ordering::Relaxed),
364 total_requests_batched: self.total_requests_batched.load(Ordering::Relaxed),
365 streaming_requests: self.streaming_requests.load(Ordering::Relaxed),
366 streamed_output_events: self.streamed_output_events.load(Ordering::Relaxed),
367 cancelled_streams: self.cancelled_streams.load(Ordering::Relaxed),
368 compatibility_streams: self.compatibility_streams.load(Ordering::Relaxed),
369 queue_wait_micros: values(&self.queue_wait_micros),
370 model_load_micros: values(&self.model_load_micros),
371 execution_micros: values(&self.execution_micros),
372 total_duration_micros: values(&self.total_duration_micros),
373 batch_sizes: self.batch_sizes.lock().expect("metrics poisoned").clone(),
374 }
375 }
376}
377
378#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
379pub struct ModelResource {
380 pub id: String,
381 pub version: Option<String>,
382 pub target: String,
383 pub state: ModelLifecycleState,
384 pub memory_bytes: u64,
385 pub last_used: u64,
386}
387
388#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
389pub struct RuntimeSnapshot {
390 pub active_inferences: usize,
391 pub active_model_loads: usize,
392 pub queued_inferences: usize,
393 pub loaded_models: usize,
394 pub cache_entries: usize,
395 pub cache_memory_bytes: u64,
396 pub metrics: RuntimeMetricsSnapshot,
397}
398
399#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
400pub enum ModelLifecycleState {
401 Discovered,
402 Validated,
403 Loaded,
404 Ready,
405 InUse,
406 Evicted,
407}
408
409#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
410pub struct RuntimeConfig {
411 pub selected_backend: Option<String>,
412 pub selected_provider: Option<String>,
413 pub prefer_acceleration: bool,
414 pub allow_remote_fallback: bool,
415 pub timeout_ms: Option<u64>,
416 pub max_concurrency: usize,
417 pub max_batch_size: Option<usize>,
418 pub max_concurrent_inferences: usize,
419 pub max_concurrent_model_loads: usize,
420 pub max_models: usize,
421 pub max_memory_bytes: Option<u64>,
422 pub max_stream_buffer: usize,
423}
424
425impl Default for RuntimeConfig {
426 fn default() -> Self {
427 Self {
428 selected_backend: None,
429 selected_provider: Some("local".to_owned()),
430 prefer_acceleration: false,
431 allow_remote_fallback: false,
432 timeout_ms: None,
433 max_concurrency: 4,
434 max_batch_size: Some(16),
435 max_concurrent_inferences: 4,
436 max_concurrent_model_loads: 1,
437 max_models: 8,
438 max_memory_bytes: None,
439 max_stream_buffer: 32,
440 }
441 }
442}
443
444#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
445pub struct RuntimeSelection {
446 pub model: String,
447 pub provider: String,
448 pub backend: Option<String>,
449 pub hardware: Option<String>,
450 pub fallback: bool,
451 pub fallback_from: Option<String>,
452 pub reason: String,
453 pub policy: ExecutionPolicy,
454}
455
456#[derive(Clone, Debug)]
457pub struct ModelRegistry {
458 models: Vec<ModelCapability>,
459}
460
461impl ModelRegistry {
462 pub fn list(&self) -> &[ModelCapability] {
463 &self.models
464 }
465}
466
467#[derive(Clone, Debug)]
468pub struct ProviderRegistry {
469 providers: Vec<ProviderCapability>,
470}
471
472impl ProviderRegistry {
473 pub fn list(&self) -> &[ProviderCapability] {
474 &self.providers
475 }
476}
477
478#[derive(Clone, Debug)]
479pub struct BackendRegistry {
480 backends: Vec<BackendCapability>,
481}
482
483impl BackendRegistry {
484 pub fn list(&self) -> &[BackendCapability] {
485 &self.backends
486 }
487}
488
489#[derive(Clone)]
490struct LoadedModel {
491 spec: ModelSpec,
492 handle: ModelHandle,
493 backend_name: String,
494 provider_name: String,
495 last_used: u64,
496 memory_bytes: u64,
497}
498
499#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
500pub struct RuntimeModelHandle {
502 model_id: String,
503 version: Option<String>,
504 provider: String,
505 backend: String,
506 state: ModelLifecycleState,
507}
508
509impl RuntimeModelHandle {
510 pub fn id(&self) -> &str {
511 &self.model_id
512 }
513 pub fn version(&self) -> Option<&str> {
514 self.version.as_deref()
515 }
516 pub fn provider(&self) -> &str {
517 &self.provider
518 }
519 pub fn backend(&self) -> &str {
520 &self.backend
521 }
522 pub fn state(&self) -> &ModelLifecycleState {
523 &self.state
524 }
525}
526
527#[derive(Clone, Debug, Eq)]
528struct ModelCacheKey {
529 id: String,
530 version: Option<String>,
531 provider: String,
532 backend: String,
533}
534
535impl PartialEq for ModelCacheKey {
536 fn eq(&self, other: &Self) -> bool {
537 self.id == other.id
538 && self.version == other.version
539 && self.provider == other.provider
540 && self.backend == other.backend
541 }
542}
543
544impl Hash for ModelCacheKey {
545 fn hash<H: Hasher>(&self, state: &mut H) {
546 self.id.hash(state);
547 self.version.hash(state);
548 self.provider.hash(state);
549 self.backend.hash(state);
550 }
551}
552
553pub struct RuntimeBuilder {
555 config: RuntimeConfig,
556 backends: BTreeMap<String, Arc<dyn Backend>>,
557 providers: BTreeMap<String, Arc<dyn Provider>>,
558 generation_providers: BTreeMap<String, Arc<dyn GenerationProvider>>,
559 decision_providers: BTreeMap<String, Arc<dyn DecisionModelProvider>>,
560 catalog: Option<Arc<dyn ModelCatalog>>,
561 capability_registry: CapabilityRegistry,
562 model_root: Option<std::path::PathBuf>,
563}
564
565impl Default for RuntimeBuilder {
566 fn default() -> Self {
567 let mut builder = Self {
568 config: RuntimeConfig::default(),
569 backends: BTreeMap::new(),
570 providers: BTreeMap::new(),
571 generation_providers: BTreeMap::new(),
572 decision_providers: BTreeMap::new(),
573 catalog: None,
574 capability_registry: CapabilityRegistry::with_builtins(),
575 model_root: default_model_root().ok(),
576 };
577 builder
578 .backends
579 .insert("cpu".to_owned(), Arc::new(CpuBackend));
580 builder
581 .backends
582 .insert("onnx".to_owned(), Arc::new(OnnxBackend));
583 builder
584 .decision_providers
585 .insert("onnx".to_owned(), Arc::new(OnnxBackend));
586 #[cfg(feature = "coreml")]
587 builder.decision_providers.insert(
588 "coreml".to_owned(),
589 Arc::new(ml_runtime_coreml_backend::CoreMlBackend),
590 );
591 builder
592 }
593}
594
595impl RuntimeBuilder {
596 pub fn backend(mut self, name: impl Into<String>) -> Self {
597 self.config.selected_backend = Some(name.into());
598 self
599 }
600
601 pub fn provider(mut self, name: impl Into<String>) -> Self {
602 self.config.selected_provider = Some(name.into());
603 self
604 }
605
606 pub fn remote(mut self, endpoint: impl Into<String>) -> Self {
608 let provider = HttpProvider::new(endpoint);
609 self.providers
610 .insert(provider.name().to_owned(), Arc::new(provider));
611 self
612 }
613
614 pub fn catalog<C>(mut self, catalog: C) -> Self
615 where
616 C: ModelCatalog + 'static,
617 {
618 self.catalog = Some(Arc::new(catalog));
619 self
620 }
621
622 pub fn model_root(mut self, root: impl Into<std::path::PathBuf>) -> Self {
623 self.model_root = Some(root.into());
624 self
625 }
626
627 pub fn prefer_acceleration(mut self, enabled: bool) -> Self {
628 self.config.prefer_acceleration = enabled;
629 self
630 }
631
632 pub fn allow_remote_fallback(mut self, enabled: bool) -> Self {
633 self.config.allow_remote_fallback = enabled;
634 self
635 }
636
637 pub fn timeout_ms(mut self, timeout_ms: u64) -> Self {
638 self.config.timeout_ms = Some(timeout_ms);
639 self
640 }
641
642 pub fn concurrency(mut self, max_concurrency: usize) -> Self {
643 self.config.max_concurrency = max_concurrency.max(1);
644 self.config.max_concurrent_inferences = max_concurrency.max(1);
645 self
646 }
647
648 pub fn max_concurrent_inferences(mut self, limit: usize) -> Self {
649 self.config.max_concurrent_inferences = limit.max(1);
650 self.config.max_concurrency = limit.max(1);
651 self
652 }
653
654 pub fn max_concurrent_model_loads(mut self, limit: usize) -> Self {
655 self.config.max_concurrent_model_loads = limit.max(1);
656 self
657 }
658
659 pub fn max_models(mut self, limit: usize) -> Self {
660 self.config.max_models = limit.max(1);
661 self
662 }
663
664 pub fn max_memory_bytes(mut self, limit: u64) -> Self {
665 self.config.max_memory_bytes = Some(limit);
666 self
667 }
668
669 pub fn max_batch_size(mut self, max_batch_size: usize) -> Self {
670 self.config.max_batch_size = Some(max_batch_size.max(1));
671 self
672 }
673
674 pub fn register_backend<B>(mut self, backend: B) -> Self
676 where
677 B: Backend + 'static,
678 {
679 let name = backend.name().to_owned();
680 self.backends.insert(name, Arc::new(backend));
681 self
682 }
683
684 pub fn register_backend_arc(mut self, backend: Arc<dyn Backend>) -> Self {
685 self.backends.insert(backend.name().to_owned(), backend);
686 self
687 }
688
689 pub fn register_provider<P>(mut self, provider: P) -> Self
691 where
692 P: Provider + 'static,
693 {
694 let name = provider.name().to_owned();
695 self.providers.insert(name, Arc::new(provider));
696 self
697 }
698
699 pub fn register_provider_arc(mut self, provider: Arc<dyn Provider>) -> Self {
700 self.providers.insert(provider.name().to_owned(), provider);
701 self
702 }
703
704 pub fn register_generation_provider<P>(mut self, provider: P) -> Self
708 where
709 P: GenerationProvider + 'static,
710 {
711 self.generation_providers
712 .insert(provider.name().to_owned(), Arc::new(provider));
713 self
714 }
715
716 pub fn register_decision_provider<P>(mut self, provider: P) -> Self
717 where
718 P: DecisionModelProvider + 'static,
719 {
720 self.decision_providers
721 .insert(provider.name().to_owned(), Arc::new(provider));
722 self
723 }
724
725 pub fn register_capability_provider<P>(mut self, provider: P) -> Self
726 where
727 P: CapabilityProvider + 'static,
728 {
729 self.capability_registry.register(provider);
730 self
731 }
732
733 pub fn build(self) -> Runtime {
734 Runtime {
735 config: self.config.clone(),
736 backends: self.backends,
737 providers: self.providers,
738 generation_providers: self.generation_providers,
739 decision_providers: self.decision_providers,
740 catalog: self.catalog,
741 capability_registry: self.capability_registry,
742 model_root: self.model_root,
743 loaded_models: RwLock::new(HashMap::new()),
744 inference_semaphore: Arc::new(Semaphore::new(
745 self.config.max_concurrent_inferences.max(1),
746 )),
747 load_semaphore: Arc::new(Semaphore::new(
748 self.config.max_concurrent_model_loads.max(1),
749 )),
750 cache_clock: AtomicU64::new(0),
751 metrics: Arc::new(RuntimeMetrics::default()),
752 execution_clock: AtomicU64::new(0),
753 execution_records: Mutex::new(VecDeque::new()),
754 }
755 }
756}
757
758pub struct Runtime {
764 config: RuntimeConfig,
765 backends: BTreeMap<String, Arc<dyn Backend>>,
766 providers: BTreeMap<String, Arc<dyn Provider>>,
767 generation_providers: BTreeMap<String, Arc<dyn GenerationProvider>>,
768 decision_providers: BTreeMap<String, Arc<dyn DecisionModelProvider>>,
769 catalog: Option<Arc<dyn ModelCatalog>>,
770 capability_registry: CapabilityRegistry,
771 model_root: Option<std::path::PathBuf>,
772 loaded_models: RwLock<HashMap<ModelCacheKey, LoadedModel>>,
773 inference_semaphore: Arc<Semaphore>,
774 load_semaphore: Arc<Semaphore>,
775 cache_clock: AtomicU64,
776 metrics: Arc<RuntimeMetrics>,
777 execution_clock: AtomicU64,
778 execution_records: Mutex<VecDeque<ExecutionRecord>>,
779}
780
781impl Default for Runtime {
782 fn default() -> Self {
783 Self::new()
784 }
785}
786
787impl Runtime {
788 pub fn new() -> Self {
789 Self::builder().build()
790 }
791
792 pub fn builder() -> RuntimeBuilder {
793 RuntimeBuilder::default()
794 }
795
796 pub fn config(&self) -> &RuntimeConfig {
797 &self.config
798 }
799
800 pub fn load_decision_model(
803 &self,
804 artifact: impl AsRef<Path>,
805 ) -> RuntimeResult<Box<dyn DecisionModel>> {
806 let artifact = artifact.as_ref();
807 if !artifact.exists() {
808 return Err(RuntimeError::ModelNotFound {
809 model: artifact.display().to_string(),
810 });
811 }
812 if !artifact.is_dir() {
813 return Err(RuntimeError::ModelUnavailable {
814 model: artifact.display().to_string(),
815 provider: "local".to_owned(),
816 reason: "typed-decision artifact must be a directory".to_owned(),
817 });
818 }
819 let candidates: Vec<_> = self
820 .decision_providers
821 .values()
822 .filter(|provider| provider.supports_artifact(artifact))
823 .collect();
824 match candidates.as_slice() {
825 [] => Err(RuntimeError::UnsupportedModel {
826 model: artifact.display().to_string(),
827 backend: "none".to_owned(),
828 reason: "no registered typed-decision provider accepts this artifact".to_owned(),
829 }),
830 [provider] => provider.load_decision_model(artifact),
831 _ => Err(RuntimeError::CapabilityMismatch {
832 reason: format!(
833 "multiple typed-decision providers accept {}",
834 artifact.display()
835 ),
836 }),
837 }
838 }
839
840 pub fn decision_capabilities(&self, model: &dyn DecisionModel) -> DecisionModelCapabilities {
843 model.capabilities()
844 }
845
846 pub fn explain_decision(
848 &self,
849 model: &dyn DecisionModel,
850 request: &DecisionGraphRequest,
851 ) -> RuntimeResult<DecisionExecutionPlan> {
852 DecisionPlanner.plan(request, &model.capabilities())
853 }
854
855 pub async fn execute_decision_graph(
858 &self,
859 model: Arc<dyn DecisionModel>,
860 request: DecisionGraphRequest,
861 cancellation: Option<CancellationToken>,
862 ) -> RuntimeResult<PlannedDecisionResult> {
863 decision::execute_graph(model, request, cancellation).await
864 }
865
866 pub async fn decide_async(
869 &self,
870 model: Arc<dyn DecisionModel>,
871 request: DecisionRequest,
872 policy: DecisionExecutionPolicy,
873 cancellation: Option<CancellationToken>,
874 ) -> RuntimeResult<PlannedDecisionResult> {
875 self.execute_decision_graph(
876 model,
877 DecisionGraphRequest::independent(request, policy),
878 cancellation,
879 )
880 .await
881 }
882
883 fn registered_model_root(&self) -> RuntimeResult<&Path> {
884 self.model_root
885 .as_deref()
886 .ok_or_else(|| RuntimeError::ModelUnavailable {
887 model: "registered models".to_owned(),
888 provider: "local".to_owned(),
889 reason: "model directory unavailable; set ML_RUNTIME_MODEL_DIR".to_owned(),
890 })
891 }
892
893 pub fn installed_models(&self) -> RuntimeResult<Vec<InstalledModelRecord>> {
894 let root = self.registered_model_root()?;
895 let installer = RegisteredModelInstaller::new(root)
896 .map_err(|error| RuntimeError::execution("model list", error.to_string()))?;
897 Ok(BuiltinModelRegistry
898 .list()
899 .into_iter()
900 .map(|model| {
901 let package = installer.installed_path(&model);
902 let (status, compiled) = if !package.exists() {
903 (InstalledModelStatus::NotInstalled, false)
904 } else if model.backend == "coreml" && !cfg!(target_os = "macos") {
905 (InstalledModelStatus::Unsupported, false)
906 } else {
907 match InstallationManifest::read(&package) {
908 Ok(manifest)
909 if manifest.model == model.name
910 && manifest.model_revision == model.revision
911 && manifest.runtime_requirement == model.runtime_requirement
912 && ml_runtime_model::runtime_requirement_satisfied(
913 &manifest.runtime_requirement,
914 VERSION,
915 )
916 && manifest.os == std::env::consts::OS
917 && manifest.architecture == std::env::consts::ARCH
918 && manifest.compiled_status == CompiledStatus::Ready
919 && manifest
920 .compiled_artifact
921 .as_ref()
922 .is_some_and(|artifact| artifact.path.exists()) =>
923 {
924 (InstalledModelStatus::Ready, true)
925 }
926 Ok(manifest) if manifest.compiled_status != CompiledStatus::Ready => {
927 (InstalledModelStatus::InstalledNotCompiled, false)
928 }
929 Ok(_) => (InstalledModelStatus::Incompatible, false),
930 Err(_) => (InstalledModelStatus::Corrupt, false),
931 }
932 };
933 InstalledModelRecord {
934 model: model.name,
935 revision: model.revision,
936 backend: model.backend,
937 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
938 status,
939 compiled,
940 package_path: package,
941 }
942 })
943 .collect())
944 }
945
946 pub async fn install_registered_model(
947 &self,
948 name: &str,
949 source_override: Option<&str>,
950 replace: bool,
951 ) -> RuntimeResult<ModelLifecycleResult> {
952 let started = Instant::now();
953 let model =
954 BuiltinModelRegistry
955 .resolve(name)
956 .map_err(|error| RuntimeError::ModelNotFound {
957 model: error.to_string(),
958 })?;
959 if model.backend == "coreml" && !cfg!(target_os = "macos") {
960 return Err(RuntimeError::backend_unavailable(
961 "coreml",
962 "Laya installation requires macOS; Core ML is unavailable on this platform",
963 ));
964 }
965 let root = self.registered_model_root()?;
966 let installer = RegisteredModelInstaller::new(root)
967 .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
968 let installed = installer
969 .install(
970 &model,
971 source_override,
972 if replace {
973 ml_runtime_model::InstallMode::Replace
974 } else {
975 ml_runtime_model::InstallMode::KeepExisting
976 },
977 None,
978 )
979 .await
980 .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
981
982 let (prepared_path, model_identity, prepared_identity) = match model.backend.as_str() {
983 "onnx" => {
984 let identity = OnnxBackend::validate_laya(&installed.package_path)?;
985 (
986 installed.package_path.join(&model.artifact_path),
987 identity.clone(),
988 identity,
989 )
990 }
991 "coreml" => {
992 #[cfg(feature = "coreml")]
993 {
994 let prepared = if installed.already_installed && !replace {
995 ml_runtime_coreml_backend::CoreMlBackend::validate_prepared_laya(
996 &installed.package_path,
997 )
998 .or_else(|_| {
999 ml_runtime_coreml_backend::CoreMlBackend::prepare_laya(
1000 &installed.package_path,
1001 )
1002 })?
1003 } else {
1004 ml_runtime_coreml_backend::CoreMlBackend::prepare_laya(
1005 &installed.package_path,
1006 )?
1007 };
1008 (
1009 prepared.path,
1010 prepared.model_identity,
1011 prepared.compiled_identity,
1012 )
1013 }
1014 #[cfg(not(feature = "coreml"))]
1015 {
1016 return Err(RuntimeError::backend_unavailable(
1017 "coreml",
1018 "this runtime was built without Core ML support",
1019 ));
1020 }
1021 }
1022 backend => {
1023 return Err(RuntimeError::UnsupportedModel {
1024 model: model.name.clone(),
1025 backend: backend.to_owned(),
1026 reason: "registered backend has no preparation lifecycle".to_owned(),
1027 })
1028 }
1029 };
1030
1031 {
1032 let manifest = InstallationManifest {
1033 schema_version: 1,
1034 model: model.name.clone(),
1035 model_revision: model.revision.clone(),
1036 model_checksum: model_identity,
1037 runtime_version: VERSION.to_owned(),
1038 runtime_requirement: model.runtime_requirement.clone(),
1039 backend: model.backend.clone(),
1040 os: std::env::consts::OS.to_owned(),
1041 architecture: std::env::consts::ARCH.to_owned(),
1042 installed_at_unix_seconds: std::time::SystemTime::now()
1043 .duration_since(std::time::UNIX_EPOCH)
1044 .unwrap_or_default()
1045 .as_secs(),
1046 compiled_status: CompiledStatus::Ready,
1047 compiled_artifact: Some(CompiledArtifactManifest {
1048 path: prepared_path,
1049 identity: prepared_identity,
1050 }),
1051 };
1052 manifest
1053 .write_atomic(&installed.package_path)
1054 .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
1055 Ok(ModelLifecycleResult {
1056 model: InstalledModelRecord {
1057 model: model.name,
1058 revision: model.revision,
1059 backend: model.backend,
1060 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1061 status: InstalledModelStatus::Ready,
1062 compiled: true,
1063 package_path: installed.package_path,
1064 },
1065 already_installed: installed.already_installed,
1066 duration_ms: started.elapsed().as_millis(),
1067 })
1068 }
1069 }
1070
1071 pub fn doctor_registered_model(&self, name: &str) -> RuntimeResult<ModelDoctorReport> {
1072 let model =
1073 BuiltinModelRegistry
1074 .resolve(name)
1075 .map_err(|error| RuntimeError::ModelNotFound {
1076 model: error.to_string(),
1077 })?;
1078 let root = self.registered_model_root()?;
1079 let installer = RegisteredModelInstaller::new(root)
1080 .map_err(|error| RuntimeError::execution("model doctor", error.to_string()))?;
1081 let package = installer.installed_path(&model);
1082 let mut diagnostics = Vec::new();
1083 if !package.is_dir() {
1084 diagnostics.push(ModelDiagnostic {
1085 check: "installation".to_owned(),
1086 ok: false,
1087 message: format!(
1088 "model is not installed; run `ml-runtime model install {}`",
1089 model.name
1090 ),
1091 });
1092 return Ok(ModelDoctorReport {
1093 model: InstalledModelRecord {
1094 model: model.name,
1095 revision: model.revision,
1096 backend: model.backend,
1097 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1098 status: InstalledModelStatus::NotInstalled,
1099 compiled: false,
1100 package_path: package,
1101 },
1102 diagnostics,
1103 manifest: None,
1104 });
1105 }
1106 let artifact_valid = match installer.validate(&model, &package) {
1107 Ok(()) => {
1108 diagnostics.push(ModelDiagnostic {
1109 check: "artifact".to_owned(),
1110 ok: true,
1111 message: format!(
1112 "registered files, {} artifact, tokenizer, and SHA-256 checksums are valid",
1113 model.backend
1114 ),
1115 });
1116 true
1117 }
1118 Err(error) => {
1119 diagnostics.push(ModelDiagnostic {
1120 check: "artifact".to_owned(),
1121 ok: false,
1122 message: format!(
1123 "{error}; reinstall with `ml-runtime model install {} --replace`",
1124 model.name
1125 ),
1126 });
1127 false
1128 }
1129 };
1130 let manifest = InstallationManifest::read(&package).ok();
1131 let compatible = manifest.as_ref().is_some_and(|manifest| {
1132 manifest.schema_version == 1
1133 && manifest.model == model.name
1134 && manifest.model_revision == model.revision
1135 && manifest.backend == model.backend
1136 && ml_runtime_model::runtime_requirement_satisfied(
1137 &manifest.runtime_requirement,
1138 VERSION,
1139 )
1140 && manifest.os == std::env::consts::OS
1141 && manifest.architecture == std::env::consts::ARCH
1142 });
1143 diagnostics.push(ModelDiagnostic {
1144 check: "metadata".to_owned(),
1145 ok: compatible,
1146 message: if compatible {
1147 "installation metadata and runtime/backend compatibility are valid".to_owned()
1148 } else {
1149 format!(
1150 "installation metadata is missing or incompatible; run `ml-runtime model install {} --replace`",
1151 model.name
1152 )
1153 },
1154 });
1155 let supported = model.supports_platform(std::env::consts::OS, std::env::consts::ARCH);
1156 diagnostics.push(ModelDiagnostic {
1157 check: "platform".to_owned(),
1158 ok: supported,
1159 message: if supported {
1160 format!("{} is supported on this platform", model.backend)
1161 } else {
1162 "Core ML is only supported on macOS".to_owned()
1163 },
1164 });
1165 let compiled_valid = supported
1166 && compatible
1167 && match model.backend.as_str() {
1168 "onnx" => OnnxBackend::validate_laya(&package).is_ok_and(|identity| {
1169 manifest.as_ref().is_some_and(|manifest| {
1170 manifest.model_checksum == identity
1171 && manifest.compiled_artifact.as_ref().is_some_and(|artifact| {
1172 artifact.path == package.join(&model.artifact_path)
1173 && artifact.identity == identity
1174 })
1175 })
1176 }),
1177 "coreml" => {
1178 #[cfg(feature = "coreml")]
1179 {
1180 ml_runtime_coreml_backend::CoreMlBackend::validate_prepared_laya(&package)
1181 .is_ok()
1182 }
1183 #[cfg(not(feature = "coreml"))]
1184 {
1185 false
1186 }
1187 }
1188 _ => false,
1189 };
1190 diagnostics.push(ModelDiagnostic {
1191 check: "compiled_artifact".to_owned(),
1192 ok: compiled_valid,
1193 message: if compiled_valid {
1194 format!("prepared {} artifact identity and schema are valid", model.backend)
1195 } else {
1196 format!(
1197 "compiled artifact is missing or invalid; run `ml-runtime model install {} --replace`",
1198 model.name
1199 )
1200 },
1201 });
1202 let status = if !supported {
1203 InstalledModelStatus::Unsupported
1204 } else if !artifact_valid {
1205 InstalledModelStatus::Corrupt
1206 } else if !compatible {
1207 InstalledModelStatus::Incompatible
1208 } else if !compiled_valid {
1209 InstalledModelStatus::InstalledNotCompiled
1210 } else {
1211 InstalledModelStatus::Ready
1212 };
1213 Ok(ModelDoctorReport {
1214 model: InstalledModelRecord {
1215 model: model.name,
1216 revision: model.revision,
1217 backend: model.backend,
1218 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1219 compiled: compiled_valid,
1220 package_path: package,
1221 status,
1222 },
1223 diagnostics,
1224 manifest,
1225 })
1226 }
1227
1228 pub fn remove_registered_model(&self, name: &str) -> RuntimeResult<InstalledModelRecord> {
1229 let model =
1230 BuiltinModelRegistry
1231 .resolve(name)
1232 .map_err(|error| RuntimeError::ModelNotFound {
1233 model: error.to_string(),
1234 })?;
1235 let root = self.registered_model_root()?;
1236 let installer = RegisteredModelInstaller::new(root)
1237 .map_err(|error| RuntimeError::execution("model remove", error.to_string()))?;
1238 let package = installer.installed_path(&model);
1239 if package.exists() {
1240 #[cfg(feature = "coreml")]
1241 if model.backend == "coreml" {
1242 ml_runtime_coreml_backend::CoreMlBackend::remove_prepared_laya(&package)?;
1243 }
1244 std::fs::remove_dir_all(&package)
1245 .map_err(|error| RuntimeError::execution("model remove", error.to_string()))?;
1246 }
1247 Ok(InstalledModelRecord {
1248 model: model.name,
1249 revision: model.revision,
1250 backend: model.backend,
1251 platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1252 status: InstalledModelStatus::NotInstalled,
1253 compiled: false,
1254 package_path: package,
1255 })
1256 }
1257
1258 pub fn load_model(&self, name: &str) -> RuntimeResult<Box<dyn DecisionModel>> {
1261 let registered =
1262 BuiltinModelRegistry
1263 .resolve(name)
1264 .map_err(|error| RuntimeError::ModelNotFound {
1265 model: error.to_string(),
1266 })?;
1267 let root = self
1268 .model_root
1269 .as_ref()
1270 .ok_or_else(|| RuntimeError::ModelUnavailable {
1271 model: name.to_owned(),
1272 provider: "local".to_owned(),
1273 reason: "model directory unavailable; set ML_RUNTIME_MODEL_DIR".to_owned(),
1274 })?;
1275 let installer = RegisteredModelInstaller::new(root).map_err(|error| {
1276 RuntimeError::execution("installed model validation", error.to_string())
1277 })?;
1278 let package = installer.installed_path(®istered);
1279 if !package.exists() {
1280 return Err(RuntimeError::ModelNotFound {
1281 model: format!("{name}; install it with `ml-runtime model install {name}`"),
1282 });
1283 }
1284 installer.validate(®istered, &package).map_err(|error| {
1285 RuntimeError::ModelIntegrity {
1286 model: name.to_owned(),
1287 reason: format!(
1288 "{error}; reinstall with `ml-runtime model install {name} --replace`"
1289 ),
1290 }
1291 })?;
1292 let manifest =
1293 InstallationManifest::read(&package).map_err(|error| RuntimeError::ModelIntegrity {
1294 model: name.to_owned(),
1295 reason: format!("{error}; run `ml-runtime model doctor {name}`"),
1296 })?;
1297 let compatible = manifest.schema_version == 1
1298 && manifest.model == registered.name
1299 && manifest.model_revision == registered.revision
1300 && manifest.backend == registered.backend
1301 && ml_runtime_model::runtime_requirement_satisfied(
1302 &manifest.runtime_requirement,
1303 VERSION,
1304 )
1305 && manifest.os == std::env::consts::OS
1306 && manifest.architecture == std::env::consts::ARCH
1307 && manifest.compiled_status == CompiledStatus::Ready
1308 && manifest.compiled_artifact.as_ref().is_some_and(|artifact| {
1309 artifact.path.exists()
1310 && (registered.backend != "onnx"
1311 || (artifact.path == package.join(®istered.artifact_path)
1312 && artifact.identity == manifest.model_checksum))
1313 });
1314 if !compatible {
1315 return Err(RuntimeError::ModelUnavailable {
1316 model: name.to_owned(),
1317 provider: "local".to_owned(),
1318 reason: format!(
1319 "installation is incomplete, incompatible, or not compiled; run `ml-runtime model doctor {name}`"
1320 ),
1321 });
1322 }
1323 self.load_decision_model(package)
1324 }
1325
1326 pub fn set_catalog<C>(&mut self, catalog: C)
1327 where
1328 C: ModelCatalog + 'static,
1329 {
1330 self.catalog = Some(Arc::new(catalog));
1331 }
1332
1333 pub async fn catalog_models(&self) -> RuntimeResult<Vec<ModelDescriptor>> {
1334 self.catalog
1335 .as_ref()
1336 .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1337 .list()
1338 .await
1339 .map_err(catalog_error)
1340 }
1341
1342 pub async fn resolve_model(
1343 &self,
1344 reference: &ModelReference,
1345 ) -> RuntimeResult<ModelDescriptor> {
1346 self.catalog
1347 .as_ref()
1348 .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1349 .resolve(reference)
1350 .await
1351 .map_err(catalog_error)
1352 }
1353
1354 pub async fn refresh_catalog(&self) -> RuntimeResult<()> {
1355 self.catalog
1356 .as_ref()
1357 .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1358 .refresh()
1359 .await
1360 .map_err(catalog_error)
1361 }
1362
1363 pub fn models(&self) -> ModelRegistry {
1364 let models = self
1365 .loaded_models
1366 .read()
1367 .expect("loaded model registry poisoned")
1368 .values()
1369 .map(|entry| ModelCapability {
1370 id: entry.spec.id.clone(),
1371 format: entry.spec.format.clone(),
1372 location: entry.spec.location.clone(),
1373 loaded: true,
1374 provider: Some(entry.provider_name.clone()),
1375 backend: Some(entry.backend_name.clone()),
1376 })
1377 .collect();
1378 ModelRegistry { models }
1379 }
1380
1381 pub fn providers(&self) -> ProviderRegistry {
1382 let providers = self
1383 .providers
1384 .iter()
1385 .map(|(name, provider)| {
1386 ProviderCapability::from_parts(name.clone(), provider.capabilities())
1387 })
1388 .collect();
1389 ProviderRegistry { providers }
1390 }
1391
1392 pub fn backends(&self) -> BackendRegistry {
1393 let backends = self
1394 .backends
1395 .iter()
1396 .map(|(name, backend)| {
1397 BackendCapability::from_parts(name.clone(), backend.capabilities())
1398 })
1399 .collect();
1400 BackendRegistry { backends }
1401 }
1402
1403 pub fn capabilities(&self) -> RuntimeCapabilities {
1404 RuntimeCapabilities {
1405 environment: self.environment(),
1406 platforms: vec![Platform::current()],
1407 providers: self.providers().providers,
1408 backends: self.backends().backends,
1409 models: self.models().models,
1410 capabilities: self.capability_registry.list(),
1411 }
1412 }
1413
1414 pub fn capability(&self, id: &str) -> Option<Capability> {
1415 self.capability_registry.get(id)
1416 }
1417
1418 pub fn capability_registry(&self) -> &CapabilityRegistry {
1419 &self.capability_registry
1420 }
1421
1422 pub fn load_capability_packages(
1426 &mut self,
1427 root: impl AsRef<std::path::Path>,
1428 ) -> RuntimeResult<Vec<String>> {
1429 self.capability_registry.load_local_packages(root)
1430 }
1431
1432 pub fn environment(&self) -> RuntimeEnvironment {
1433 RuntimeEnvironment::discover(
1434 self.capability_registry
1435 .providers()
1436 .iter()
1437 .map(|provider| provider.id().to_owned())
1438 .collect(),
1439 )
1440 }
1441
1442 pub fn resolve_capability(
1443 &self,
1444 request: &ExecutionRequest,
1445 ) -> RuntimeResult<CapabilityResolution> {
1446 self.resolve_capability_with_constraints(request, &ResolutionConstraints::default())
1447 }
1448
1449 pub fn resolve_capability_with_constraints(
1450 &self,
1451 request: &ExecutionRequest,
1452 constraints: &ResolutionConstraints,
1453 ) -> RuntimeResult<CapabilityResolution> {
1454 self.capability_registry
1455 .resolve(&request.capability, constraints)
1456 }
1457
1458 pub async fn execute_capability(
1459 &self,
1460 request: ExecutionRequest,
1461 authorizer: Option<&dyn CapabilityAuthorizer>,
1462 ) -> RuntimeResult<ExecutionResult> {
1463 let resolution = self
1464 .capability_registry
1465 .resolve(&request.capability, &ResolutionConstraints::default())?;
1466 let provider = self
1467 .capability_registry
1468 .provider(&resolution.provider)
1469 .ok_or_else(|| {
1470 RuntimeError::provider_unavailable(&resolution.provider, "not registered")
1471 })?;
1472 if resolution.provider == "builtin" {
1473 capability::execute(request, authorizer).await
1474 } else {
1475 if let Some(authorizer) = authorizer {
1476 if request.policy.require_authorization {
1477 let capability = self
1478 .capability_registry
1479 .get(&request.capability)
1480 .ok_or_else(|| RuntimeError::CapabilityMismatch {
1481 reason: format!("unknown capability {}", request.capability),
1482 })?;
1483 authorizer.authorize(&request, &capability).await?;
1484 }
1485 } else if self
1486 .capability_registry
1487 .get(&request.capability)
1488 .is_some_and(|capability| !capability.requirements.is_empty())
1489 {
1490 return Err(RuntimeError::Execution {
1491 operation: request.capability,
1492 reason: "authorization required".to_owned(),
1493 });
1494 }
1495 provider.execute(request).await
1496 }
1497 }
1498
1499 pub fn status(&self) -> RuntimeStatus {
1500 RuntimeStatus {
1501 loaded_models: self
1502 .loaded_models
1503 .read()
1504 .expect("loaded model registry poisoned")
1505 .len(),
1506 max_models: self.config.max_models,
1507 max_concurrent_inferences: self.config.max_concurrent_inferences,
1508 max_concurrent_model_loads: self.config.max_concurrent_model_loads,
1509 }
1510 }
1511
1512 pub fn metrics(&self) -> RuntimeMetricsSnapshot {
1513 self.metrics.snapshot()
1514 }
1515
1516 pub fn executions(&self) -> Vec<ExecutionRecord> {
1517 self.execution_records
1518 .lock()
1519 .expect("execution records poisoned")
1520 .iter()
1521 .cloned()
1522 .collect()
1523 }
1524
1525 pub fn resources(&self) -> Vec<ModelResource> {
1526 self.loaded_models
1527 .read()
1528 .expect("loaded model registry poisoned")
1529 .values()
1530 .map(|model| ModelResource {
1531 id: model.spec.id.clone(),
1532 version: model.spec.version.clone(),
1533 target: format!("{}/{}", model.provider_name, model.backend_name),
1534 state: ModelLifecycleState::Ready,
1535 memory_bytes: model.memory_bytes,
1536 last_used: model.last_used,
1537 })
1538 .collect()
1539 }
1540
1541 pub fn snapshot(&self) -> RuntimeSnapshot {
1542 let resources = self.resources();
1543 let metrics = self.metrics();
1544 RuntimeSnapshot {
1545 active_inferences: metrics.active_inferences,
1546 active_model_loads: metrics.active_model_loads,
1547 queued_inferences: (metrics.queued_requests as usize)
1548 .saturating_sub(metrics.active_inferences),
1549 loaded_models: resources.len(),
1550 cache_entries: resources.len(),
1551 cache_memory_bytes: resources.iter().map(|resource| resource.memory_bytes).sum(),
1552 metrics,
1553 }
1554 }
1555
1556 pub fn selection_for(&self, model: &ModelSpec) -> RuntimeResult<RuntimeSelection> {
1557 let backend_name = self.select_backend(model)?;
1558 let backend = self
1559 .backends
1560 .get(&backend_name)
1561 .ok_or_else(|| RuntimeError::backend_unavailable(&backend_name, "not registered"))?;
1562 Ok(RuntimeSelection {
1563 model: model.id.clone(),
1564 provider: "local".to_owned(),
1565 backend: Some(backend_name),
1566 hardware: backend.capabilities().hardware,
1567 fallback: false,
1568 fallback_from: None,
1569 reason: "preferred local execution".to_owned(),
1570 policy: ExecutionPolicy::LocalOnly,
1571 })
1572 }
1573
1574 pub async fn load(&self, model: ModelSpec) -> RuntimeResult<RuntimeModelHandle> {
1577 if self
1578 .config
1579 .selected_provider
1580 .as_deref()
1581 .is_some_and(|provider| provider != "local")
1582 {
1583 return Err(RuntimeError::provider_unavailable(
1584 self.config
1585 .selected_provider
1586 .clone()
1587 .unwrap_or_else(|| "remote".to_owned()),
1588 "load is only supported for local execution paths",
1589 ));
1590 }
1591
1592 let backend_name = self.select_backend(&model)?;
1593 let key = ModelCacheKey {
1594 id: model.id.clone(),
1595 version: model.version.clone(),
1596 provider: "local".to_owned(),
1597 backend: backend_name.clone(),
1598 };
1599 if let Some(existing) = self
1600 .loaded_models
1601 .read()
1602 .expect("loaded model registry poisoned")
1603 .get(&key)
1604 {
1605 self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1606 return Ok(runtime_handle(existing));
1607 }
1608 self.metrics.cache_misses.fetch_add(1, Ordering::Relaxed);
1609 self.metrics
1610 .active_model_loads
1611 .fetch_add(1, Ordering::Relaxed);
1612 let _permit = self
1613 .load_semaphore
1614 .acquire()
1615 .await
1616 .map_err(|_| RuntimeError::execution("load", "runtime load semaphore closed"))?;
1617 if let Some(existing) = self
1618 .loaded_models
1619 .read()
1620 .expect("loaded model registry poisoned")
1621 .get(&key)
1622 {
1623 self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1624 return Ok(runtime_handle(existing));
1625 }
1626 let backend = self
1627 .backends
1628 .get(&backend_name)
1629 .ok_or_else(|| RuntimeError::backend_unavailable(&backend_name, "not registered"))?
1630 .clone();
1631 let timeout_ms = self.config.timeout_ms;
1632 let loaded = run_with_timeout(timeout_ms, "load", backend.load(&model)).await;
1633 self.metrics
1634 .active_model_loads
1635 .fetch_sub(1, Ordering::Relaxed);
1636 let handle = match loaded {
1637 Ok(handle) => handle,
1638 Err(error) => {
1639 self.metrics
1640 .model_load_failures
1641 .fetch_add(1, Ordering::Relaxed);
1642 return Err(error);
1643 }
1644 };
1645
1646 let mut cache = self
1647 .loaded_models
1648 .write()
1649 .expect("loaded model registry poisoned");
1650 if cache.len() >= self.config.max_models {
1651 if let Some(oldest) = cache
1652 .iter()
1653 .min_by_key(|(_, model)| model.last_used)
1654 .map(|(key, _)| key.clone())
1655 {
1656 cache.remove(&oldest);
1657 self.metrics.evictions.fetch_add(1, Ordering::Relaxed);
1658 }
1659 }
1660 self.metrics.model_loads.fetch_add(1, Ordering::Relaxed);
1661 let loaded = LoadedModel {
1662 memory_bytes: model_memory_bytes(&model),
1663 spec: model,
1664 handle,
1665 backend_name,
1666 provider_name: "local".to_owned(),
1667 last_used: self.cache_clock.fetch_add(1, Ordering::Relaxed),
1668 };
1669 let runtime_handle = runtime_handle(&loaded);
1670 cache.insert(key, loaded);
1671 Ok(runtime_handle)
1672 }
1673
1674 pub async fn load_reference(
1675 &self,
1676 reference: ModelReference,
1677 ) -> RuntimeResult<RuntimeModelHandle> {
1678 match reference {
1679 ModelReference::Spec(spec) => self.load(spec).await,
1680 ModelReference::Id { id, version } => self
1681 .find_loaded(&id, version.as_ref())
1682 .map(|loaded| runtime_handle(&loaded)),
1683 }
1684 }
1685
1686 pub async fn infer(&self, request: InferenceRequest) -> RuntimeResult<InferenceResult> {
1688 self.infer_with_cancellation(request, CancellationToken::new())
1689 .await
1690 }
1691
1692 pub async fn infer_loaded(
1694 &self,
1695 model: &RuntimeModelHandle,
1696 input: Input,
1697 options: InferenceOptions,
1698 ) -> RuntimeResult<InferenceResult> {
1699 self.infer(InferenceRequest {
1700 model: ModelReference::id(model.id().to_owned(), model.version().map(str::to_owned)),
1701 input,
1702 options,
1703 })
1704 .await
1705 }
1706
1707 pub async fn infer_loaded_stream(
1708 &self,
1709 model: &RuntimeModelHandle,
1710 input: Input,
1711 mut options: InferenceOptions,
1712 ) -> RuntimeResult<InferenceStream> {
1713 options.stream = true;
1714 self.infer_stream(InferenceRequest {
1715 model: ModelReference::id(model.id().to_owned(), model.version().map(str::to_owned)),
1716 input,
1717 options,
1718 })
1719 .await
1720 }
1721
1722 pub async fn infer_batch(
1727 &self,
1728 mut requests: Vec<InferenceRequest>,
1729 ) -> RuntimeResult<Vec<InferenceResult>> {
1730 if requests.is_empty() {
1731 return Ok(Vec::new());
1732 }
1733 if let Some(limit) = self.config.max_batch_size {
1734 if requests.len() > limit {
1735 return Err(RuntimeError::ResourceLimit {
1736 reason: format!(
1737 "batch size {} exceeds configured limit {limit}",
1738 requests.len()
1739 ),
1740 });
1741 }
1742 }
1743 for request in &requests {
1744 self.validate_request(request)?;
1745 }
1746 for request in &mut requests {
1747 self.resolve_catalog_reference_for_local(request).await?;
1748 }
1749 let selection = self.select_for_request(&requests[0])?;
1750 let same_model = requests
1751 .iter()
1752 .all(|request| model_id(&request.model) == model_id(&requests[0].model));
1753 if selection.provider != "local" || !same_model {
1754 return futures_util::future::try_join_all(
1755 requests.into_iter().map(|request| self.infer(request)),
1756 )
1757 .await;
1758 }
1759 let loaded = self.resolve_loaded_model(&requests[0].model).await?;
1760 let backend = self
1761 .backends
1762 .get(&loaded.backend_name)
1763 .ok_or_else(|| {
1764 RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
1765 })?
1766 .clone();
1767 if !backend.capabilities().batching {
1768 return futures_util::future::try_join_all(
1769 requests.into_iter().map(|request| self.infer(request)),
1770 )
1771 .await;
1772 }
1773 let batch_size = requests.len();
1774 self.metrics
1775 .total_requests
1776 .fetch_add(batch_size as u64, Ordering::Relaxed);
1777 self.metrics
1778 .batch_executions
1779 .fetch_add(1, Ordering::Relaxed);
1780 self.metrics
1781 .total_requests_batched
1782 .fetch_add(batch_size as u64, Ordering::Relaxed);
1783 self.metrics
1784 .batch_sizes
1785 .lock()
1786 .expect("metrics poisoned")
1787 .push(batch_size);
1788 let _permit = self.inference_semaphore.acquire().await.map_err(|_| {
1789 RuntimeError::execution("infer_batch", "runtime inference semaphore closed")
1790 })?;
1791 let mut results = run_with_timeout(
1792 self.config.timeout_ms,
1793 "infer_batch",
1794 backend.infer_batch(&loaded.handle, &requests, Some(CancellationToken::new())),
1795 )
1796 .await?;
1797 if results.len() != batch_size {
1798 return Err(RuntimeError::execution(
1799 "infer_batch",
1800 format!(
1801 "backend returned {} results for {batch_size} requests",
1802 results.len()
1803 ),
1804 ));
1805 }
1806 for (result, request) in results.iter_mut().zip(requests.iter()) {
1807 let output = result.output.clone();
1808 decorate_metadata(
1809 &mut result.metadata,
1810 &output,
1811 &loaded.spec.id,
1812 "local",
1813 &loaded.backend_name,
1814 backend.capabilities().hardware.clone(),
1815 Duration::ZERO,
1816 estimate_input_tokens(&request.input),
1817 loaded.spec.version.clone(),
1818 selection.policy.clone(),
1819 false,
1820 None,
1821 Some(selection.reason.clone()),
1822 );
1823 result.metadata.batch_size = batch_size;
1824 result.metadata.cache_hit = true;
1825 }
1826 self.metrics
1827 .successful_requests
1828 .fetch_add(results.len() as u64, Ordering::Relaxed);
1829 Ok(results)
1830 }
1831
1832 pub async fn infer_with_cancellation(
1835 &self,
1836 request: InferenceRequest,
1837 cancellation: CancellationToken,
1838 ) -> RuntimeResult<InferenceResult> {
1839 let started = Instant::now();
1840 let request_id = format!(
1841 "request-{}",
1842 self.execution_clock.fetch_add(1, Ordering::Relaxed) + 1
1843 );
1844 self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
1845 self.metrics
1846 .active_inferences
1847 .fetch_add(1, Ordering::Relaxed);
1848 let model = model_details(&request.model);
1849 let result = self.infer_inner(request.clone(), cancellation).await;
1850 self.metrics
1851 .active_inferences
1852 .fetch_sub(1, Ordering::Relaxed);
1853 match result {
1854 Ok(mut result) => {
1855 result.metadata.request_id = Some(request_id.clone());
1856 self.record_execution(
1857 &request_id,
1858 started.elapsed(),
1859 &result.metadata,
1860 ExecutionCompletion::Succeeded,
1861 );
1862 self.metrics
1863 .successful_requests
1864 .fetch_add(1, Ordering::Relaxed);
1865 Ok(result)
1866 }
1867 Err(error) => {
1868 let completion = if matches!(error, RuntimeError::Cancelled) {
1869 self.metrics
1870 .cancelled_requests
1871 .fetch_add(1, Ordering::Relaxed);
1872 ExecutionCompletion::Cancelled
1873 } else {
1874 self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
1875 ExecutionCompletion::Failed
1876 };
1877 let metadata = ExecutionMetadata {
1878 request_id: Some(request_id.clone()),
1879 model: model.0,
1880 model_version: model.1,
1881 routing_policy: request.options.execution.clone(),
1882 streaming_requested: request.options.stream,
1883 ..ExecutionMetadata::default()
1884 };
1885 self.record_execution(&request_id, started.elapsed(), &metadata, completion);
1886 Err(error)
1887 }
1888 }
1889 }
1890
1891 async fn infer_inner(
1892 &self,
1893 mut request: InferenceRequest,
1894 cancellation: CancellationToken,
1895 ) -> RuntimeResult<InferenceResult> {
1896 self.validate_request(&request)?;
1897 if cancellation.is_cancelled() {
1898 return Err(RuntimeError::Cancelled);
1899 }
1900
1901 let _permit = tokio::select! {
1902 permit = self.inference_semaphore.acquire() => permit
1903 .map_err(|_| RuntimeError::execution("infer", "runtime inference semaphore closed"))?,
1904 _ = cancellation.cancelled() => return Err(RuntimeError::Cancelled),
1905 };
1906
1907 self.resolve_catalog_reference_for_local(&mut request)
1908 .await?;
1909
1910 let mut selection = self.select_for_request(&request)?;
1911 let timeout_ms = request.options.timeout_ms.or(self.config.timeout_ms);
1912 let input_tokens = estimate_input_tokens(&request.input);
1913
1914 match selection.provider.as_str() {
1915 "local" => {
1916 let cache_hit = self.is_loaded(&request.model);
1917 let model_load_started = Instant::now();
1918 if cache_hit {
1919 self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1920 } else {
1921 self.metrics.cache_misses.fetch_add(1, Ordering::Relaxed);
1922 }
1923
1924 let loaded = match self.resolve_loaded_model(&request.model).await {
1925 Ok(loaded) => loaded,
1926 Err(error) if self.can_fallback(&request, &selection, &error) => {
1927 selection = self.remote_selection(
1928 model_id(&request.model),
1929 true,
1930 Some("local model unavailable".to_owned()),
1931 &request.options,
1932 )?;
1933 return self
1934 .infer_remote(
1935 request,
1936 cancellation,
1937 timeout_ms,
1938 input_tokens,
1939 selection,
1940 )
1941 .await;
1942 }
1943 Err(error) => return Err(error),
1944 };
1945 let backend = self
1946 .backends
1947 .get(&loaded.backend_name)
1948 .ok_or_else(|| {
1949 RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
1950 })?
1951 .clone();
1952 let started = Instant::now();
1953 let result = run_with_timeout(
1954 timeout_ms,
1955 "infer",
1956 backend.infer(&loaded.handle, &request, Some(cancellation.clone())),
1957 )
1958 .await;
1959 let mut result = match result {
1960 Ok(result) => result,
1961 Err(error) if self.can_fallback(&request, &selection, &error) => {
1962 selection = self.remote_selection(
1963 model_id(&request.model),
1964 true,
1965 Some(error.to_string()),
1966 &request.options,
1967 )?;
1968 return self
1969 .infer_remote(
1970 request,
1971 cancellation,
1972 timeout_ms,
1973 input_tokens,
1974 selection,
1975 )
1976 .await;
1977 }
1978 Err(error) => return Err(error),
1979 };
1980 result.metadata.model_load_ms = Some(if cache_hit {
1981 0.0
1982 } else {
1983 model_load_started.elapsed().as_secs_f64() * 1000.0
1984 });
1985 let capabilities = backend.capabilities();
1986 let output = result.output.clone();
1987 decorate_metadata(
1988 &mut result.metadata,
1989 &output,
1990 &loaded.spec.id,
1991 "local",
1992 &loaded.backend_name,
1993 capabilities.hardware,
1994 started.elapsed(),
1995 input_tokens,
1996 loaded.spec.version.clone(),
1997 selection.policy.clone(),
1998 selection.fallback,
1999 selection.fallback_from.clone(),
2000 Some(selection.reason.clone()),
2001 );
2002 result.metadata.cache_hit = cache_hit;
2003 Ok(result)
2004 }
2005 _ => {
2006 self.infer_remote(request, cancellation, timeout_ms, input_tokens, selection)
2007 .await
2008 }
2009 }
2010 }
2011
2012 fn record_execution(
2013 &self,
2014 request_id: &str,
2015 total_duration: Duration,
2016 metadata: &ExecutionMetadata,
2017 completion: ExecutionCompletion,
2018 ) {
2019 let execution_id = format!(
2020 "execution-{}",
2021 self.execution_clock.fetch_add(1, Ordering::Relaxed) + 1
2022 );
2023 let execution_duration =
2024 Duration::from_secs_f64(metadata.execution_ms.unwrap_or(0.0) / 1000.0);
2025 let queued_duration =
2026 Duration::from_secs_f64(metadata.queue_wait_ms.unwrap_or(0.0) / 1000.0);
2027 let model_load_duration =
2028 Duration::from_secs_f64(metadata.model_load_ms.unwrap_or(0.0) / 1000.0);
2029 self.metrics
2030 .queue_wait_micros
2031 .lock()
2032 .expect("metrics poisoned")
2033 .push(queued_duration.as_micros() as u64);
2034 self.metrics
2035 .model_load_micros
2036 .lock()
2037 .expect("metrics poisoned")
2038 .push(model_load_duration.as_micros() as u64);
2039 self.metrics
2040 .execution_micros
2041 .lock()
2042 .expect("metrics poisoned")
2043 .push(execution_duration.as_micros() as u64);
2044 self.metrics
2045 .total_duration_micros
2046 .lock()
2047 .expect("metrics poisoned")
2048 .push(total_duration.as_micros() as u64);
2049 if metadata.provider == "local" {
2050 self.metrics
2051 .local_executions
2052 .fetch_add(1, Ordering::Relaxed);
2053 } else {
2054 self.metrics
2055 .remote_executions
2056 .fetch_add(1, Ordering::Relaxed);
2057 }
2058 if metadata.fallback {
2059 self.metrics
2060 .fallback_executions
2061 .fetch_add(1, Ordering::Relaxed);
2062 }
2063 if metadata.streaming_requested {
2064 self.metrics
2065 .streaming_requests
2066 .fetch_add(1, Ordering::Relaxed);
2067 }
2068 let record = ExecutionRecord {
2069 execution_id,
2070 request_id: request_id.to_owned(),
2071 model_id: metadata.model.clone(),
2072 model_version: metadata.model_version.clone().unwrap_or_default(),
2073 provider: metadata.provider.clone(),
2074 backend: metadata.backend.clone(),
2075 target: metadata.execution_target.clone(),
2076 routing_policy: metadata.routing_policy.clone(),
2077 fallback: metadata.fallback_from.clone(),
2078 cache_hit: metadata.cache_hit,
2079 batch_size: metadata.batch_size.max(1),
2080 queued_duration,
2081 model_load_duration,
2082 execution_duration,
2083 total_duration,
2084 streaming: metadata.streaming_requested,
2085 output_event_count: metadata.output_event_count,
2086 completion,
2087 };
2088 let mut records = self
2089 .execution_records
2090 .lock()
2091 .expect("execution records poisoned");
2092 records.push_back(record);
2093 while records.len() > 256 {
2094 records.pop_front();
2095 }
2096 }
2097
2098 async fn infer_remote(
2099 &self,
2100 request: InferenceRequest,
2101 cancellation: CancellationToken,
2102 timeout_ms: Option<u64>,
2103 input_tokens: Option<u64>,
2104 selection: RuntimeSelection,
2105 ) -> RuntimeResult<InferenceResult> {
2106 let provider_name = selection.provider.as_str();
2107 let provider = self
2108 .providers
2109 .get(provider_name)
2110 .ok_or_else(|| RuntimeError::provider_unavailable(provider_name, "not registered"))?
2111 .clone();
2112 let started = Instant::now();
2113 let mut result = run_with_timeout(
2114 timeout_ms,
2115 "infer",
2116 provider.infer(request.clone(), Some(cancellation)),
2117 )
2118 .await?;
2119 let backend_name = result.metadata.backend.clone();
2120 let hardware = result.metadata.hardware.clone();
2121 let output = result.output.clone();
2122 let (model, version) = model_details(&request.model);
2123 decorate_metadata(
2124 &mut result.metadata,
2125 &output,
2126 &model,
2127 provider_name,
2128 backend_name.as_str(),
2129 hardware,
2130 started.elapsed(),
2131 input_tokens,
2132 version,
2133 selection.policy,
2134 selection.fallback,
2135 selection.fallback_from,
2136 Some(selection.reason),
2137 );
2138 Ok(result)
2139 }
2140
2141 pub async fn infer_stream(&self, request: InferenceRequest) -> RuntimeResult<InferenceStream> {
2143 self.infer_stream_with_cancellation(request, CancellationToken::new())
2144 .await
2145 }
2146
2147 pub async fn infer_stream_with_cancellation(
2149 &self,
2150 mut request: InferenceRequest,
2151 cancellation: CancellationToken,
2152 ) -> RuntimeResult<InferenceStream> {
2153 self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
2154 self.metrics
2155 .streaming_requests
2156 .fetch_add(1, Ordering::Relaxed);
2157 self.validate_request(&request)?;
2158 if !request.options.stream {
2159 return Err(RuntimeError::CapabilityMismatch {
2160 reason: "infer_stream requires InferenceOptions::stream to be true".to_owned(),
2161 });
2162 }
2163 self.resolve_catalog_reference_for_local(&mut request)
2164 .await?;
2165 let selection = self.select_for_request(&request)?;
2166 let buffer = self.config.max_stream_buffer;
2167 if buffer == 0 {
2168 return Err(RuntimeError::ResourceLimit {
2169 reason: "max_stream_buffer must be greater than zero".to_owned(),
2170 });
2171 }
2172 match selection.provider.as_str() {
2173 "local" => {
2174 let loaded = self.resolve_loaded_model(&request.model).await?;
2175 let backend = self
2176 .backends
2177 .get(&loaded.backend_name)
2178 .ok_or_else(|| {
2179 RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
2180 })?
2181 .clone();
2182 let capabilities = backend.capabilities();
2183 let stream = if capabilities.streaming {
2184 backend
2185 .infer_stream(&loaded.handle, &request, Some(cancellation.clone()))
2186 .await?
2187 } else {
2188 self.metrics
2189 .compatibility_streams
2190 .fetch_add(1, Ordering::Relaxed);
2191 let result = backend
2192 .infer(&loaded.handle, &request, Some(cancellation.clone()))
2193 .await?;
2194 Box::pin(futures_util::stream::iter(vec![Ok(InferenceChunk {
2195 output: result.output,
2196 done: true,
2197 metadata: Some(result.metadata),
2198 })])) as BoxStream<_>
2199 };
2200 Ok(normalize_stream(
2201 stream,
2202 request,
2203 selection,
2204 capabilities.streaming,
2205 true,
2206 buffer,
2207 cancellation,
2208 self.metrics.clone(),
2209 ))
2210 }
2211 provider_name => {
2212 let provider = self
2213 .providers
2214 .get(provider_name)
2215 .ok_or_else(|| {
2216 RuntimeError::provider_unavailable(provider_name, "not registered")
2217 })?
2218 .clone();
2219 let capabilities = provider.capabilities();
2220 let stream = if capabilities.streaming {
2221 provider
2222 .infer_stream(request.clone(), Some(cancellation.clone()))
2223 .await?
2224 } else {
2225 self.metrics
2226 .compatibility_streams
2227 .fetch_add(1, Ordering::Relaxed);
2228 let result = provider
2229 .infer(request.clone(), Some(cancellation.clone()))
2230 .await?;
2231 Box::pin(futures_util::stream::iter(vec![Ok(InferenceChunk {
2232 output: result.output,
2233 done: true,
2234 metadata: Some(result.metadata),
2235 })])) as BoxStream<_>
2236 };
2237 Ok(normalize_stream(
2238 stream,
2239 request,
2240 selection,
2241 capabilities.streaming,
2242 false,
2243 buffer,
2244 cancellation,
2245 self.metrics.clone(),
2246 ))
2247 }
2248 }
2249 }
2250
2251 fn validate_request(&self, request: &InferenceRequest) -> RuntimeResult<()> {
2252 if request.options.require_local && request.options.require_remote {
2253 return Err(RuntimeError::InvalidRequest {
2254 reason: "request cannot require both local and remote execution".to_owned(),
2255 });
2256 }
2257 if let Some(limit) = self.config.max_batch_size {
2258 let batch_size = request
2259 .options
2260 .batch_size
2261 .unwrap_or_else(|| request.input.batch_size());
2262 if batch_size > limit {
2263 return Err(RuntimeError::ResourceLimit {
2264 reason: format!("batch size {batch_size} exceeds configured limit {limit}"),
2265 });
2266 }
2267 }
2268 Ok(())
2269 }
2270
2271 async fn resolve_catalog_reference_for_local(
2272 &self,
2273 request: &mut InferenceRequest,
2274 ) -> RuntimeResult<()> {
2275 let local_resolution = request.options.require_local
2276 || (!request.options.require_remote
2277 && matches!(
2278 request.options.execution,
2279 ExecutionPolicy::LocalOnly
2280 | ExecutionPolicy::PreferLocal
2281 | ExecutionPolicy::LocalThenRemote
2282 ));
2283 if local_resolution && !self.is_loaded(&request.model) {
2284 if let (Some(catalog), ModelReference::Id { .. }) = (&self.catalog, &request.model) {
2285 let descriptor = catalog
2286 .resolve(&request.model)
2287 .await
2288 .map_err(catalog_error)?;
2289 catalog
2290 .validate(&descriptor.id)
2291 .await
2292 .map_err(catalog_error)?;
2293 request.model = ModelReference::Spec(descriptor.spec());
2294 }
2295 }
2296 Ok(())
2297 }
2298
2299 async fn resolve_loaded_model(&self, reference: &ModelReference) -> RuntimeResult<LoadedModel> {
2300 match reference {
2301 ModelReference::Id { id, version } => self.find_loaded(id, version.as_ref()),
2302 ModelReference::Spec(spec) => {
2303 if let Ok(existing) = self.find_loaded(&spec.id, spec.version.as_ref()) {
2304 return Ok(existing);
2305 }
2306 self.load(spec.clone()).await?;
2307 self.find_loaded(&spec.id, spec.version.as_ref())
2308 }
2309 }
2310 }
2311
2312 fn find_loaded(&self, id: &str, version: Option<&String>) -> RuntimeResult<LoadedModel> {
2313 let mut cache = self
2314 .loaded_models
2315 .write()
2316 .expect("loaded model registry poisoned");
2317 let found = cache.iter_mut().find(|(key, loaded)| {
2318 key.id == id
2319 && version.is_none_or(|version| loaded.spec.version.as_ref() == Some(version))
2320 });
2321 if let Some((_, loaded)) = found {
2322 loaded.last_used = self.cache_clock.fetch_add(1, Ordering::Relaxed);
2323 return Ok(loaded.clone());
2324 }
2325
2326 Err(RuntimeError::ModelNotFound {
2327 model: id.to_owned(),
2328 })
2329 }
2330
2331 fn is_loaded(&self, reference: &ModelReference) -> bool {
2332 let (id, version) = model_details(reference);
2333 self.loaded_models
2334 .read()
2335 .expect("loaded model registry poisoned")
2336 .keys()
2337 .any(|key| {
2338 key.id == id
2339 && version
2340 .as_ref()
2341 .is_none_or(|v| key.version.as_ref() == Some(v))
2342 })
2343 }
2344
2345 fn select_for_request(&self, request: &InferenceRequest) -> RuntimeResult<RuntimeSelection> {
2346 let model_id = model_id(&request.model);
2347 let policy = if request.options.require_remote {
2348 ExecutionPolicy::RemoteOnly
2349 } else if request.options.require_local {
2350 ExecutionPolicy::LocalOnly
2351 } else if self.config.allow_remote_fallback {
2352 ExecutionPolicy::LocalThenRemote
2353 } else {
2354 request.options.execution.clone()
2355 };
2356 match policy {
2357 ExecutionPolicy::RemoteOnly => self.remote_selection(
2358 model_id,
2359 false,
2360 Some("remote-only policy".to_owned()),
2361 &request.options,
2362 ),
2363 ExecutionPolicy::LocalOnly => self.local_selection(&request.model, policy),
2364 ExecutionPolicy::PreferLocal => match self
2365 .local_selection(&request.model, policy.clone())
2366 {
2367 Ok(selection) => Ok(selection),
2368 Err(error) => {
2369 self.remote_selection(model_id, true, Some(error.to_string()), &request.options)
2370 }
2371 },
2372 ExecutionPolicy::LocalThenRemote => match self
2373 .local_selection(&request.model, policy.clone())
2374 {
2375 Ok(selection) => Ok(selection),
2376 Err(error) => {
2377 self.remote_selection(model_id, true, Some(error.to_string()), &request.options)
2378 }
2379 },
2380 ExecutionPolicy::PreferRemote | ExecutionPolicy::RemoteThenLocal => match self
2381 .remote_selection(
2382 model_id.clone(),
2383 false,
2384 Some("preferred remote execution".to_owned()),
2385 &request.options,
2386 ) {
2387 Ok(selection) => Ok(selection),
2388 Err(_) => self.local_selection(&request.model, policy),
2389 },
2390 }
2391 }
2392
2393 fn local_selection(
2394 &self,
2395 reference: &ModelReference,
2396 policy: ExecutionPolicy,
2397 ) -> RuntimeResult<RuntimeSelection> {
2398 match reference {
2399 ModelReference::Id { id, version } => {
2400 let loaded = self.find_loaded(id, version.as_ref())?;
2401 Ok(RuntimeSelection {
2402 model: loaded.spec.id,
2403 provider: "local".to_owned(),
2404 backend: Some(loaded.backend_name.clone()),
2405 hardware: self
2406 .backends
2407 .get(&loaded.backend_name)
2408 .and_then(|backend| backend.capabilities().hardware),
2409 fallback: false,
2410 fallback_from: None,
2411 reason: "preferred local execution".to_owned(),
2412 policy,
2413 })
2414 }
2415 ModelReference::Spec(spec) => {
2416 let backend = self.select_backend(spec)?;
2417 let hardware = self
2418 .backends
2419 .get(&backend)
2420 .and_then(|b| b.capabilities().hardware);
2421 Ok(RuntimeSelection {
2422 model: spec.id.clone(),
2423 provider: "local".to_owned(),
2424 backend: Some(backend),
2425 hardware,
2426 fallback: false,
2427 fallback_from: None,
2428 reason: "preferred local execution".to_owned(),
2429 policy,
2430 })
2431 }
2432 }
2433 }
2434
2435 fn remote_selection(
2436 &self,
2437 model: String,
2438 fallback: bool,
2439 reason: Option<String>,
2440 options: &InferenceOptions,
2441 ) -> RuntimeResult<RuntimeSelection> {
2442 let selected = if let Some(provider_name) = self.config.selected_provider.as_deref() {
2443 if provider_name != "local" {
2444 Some(provider_name.to_owned())
2445 } else {
2446 None
2447 }
2448 } else {
2449 None
2450 }
2451 .or_else(|| {
2452 self.providers.iter().find_map(|(name, provider)| {
2453 let capabilities = provider.capabilities();
2454 if capabilities.available && capabilities.remote {
2455 Some(name.clone())
2456 } else {
2457 None
2458 }
2459 })
2460 })
2461 .ok_or_else(|| {
2462 RuntimeError::provider_unavailable("remote", "no remote provider registered")
2463 })?;
2464
2465 Ok(RuntimeSelection {
2466 model,
2467 provider: selected,
2468 backend: None,
2469 hardware: None,
2470 fallback,
2471 fallback_from: fallback.then(|| "local".to_owned()),
2472 reason: reason.unwrap_or_else(|| "remote execution selected by policy".to_owned()),
2473 policy: if fallback && options.execution == ExecutionPolicy::LocalOnly {
2474 ExecutionPolicy::LocalThenRemote
2475 } else {
2476 options.execution.clone()
2477 },
2478 })
2479 }
2480
2481 fn can_fallback(
2482 &self,
2483 request: &InferenceRequest,
2484 selection: &RuntimeSelection,
2485 error: &RuntimeError,
2486 ) -> bool {
2487 matches!(
2488 selection.policy,
2489 ExecutionPolicy::LocalThenRemote | ExecutionPolicy::PreferLocal
2490 ) && !request.options.require_local
2491 && matches!(
2492 error,
2493 RuntimeError::ProviderUnavailable { .. }
2494 | RuntimeError::ModelUnavailable { .. }
2495 | RuntimeError::BackendUnavailable { .. }
2496 | RuntimeError::ModelNotFound { .. }
2497 | RuntimeError::CapabilityMismatch { .. }
2498 | RuntimeError::Timeout { .. }
2499 | RuntimeError::Transport { .. }
2500 | RuntimeError::Protocol { .. }
2501 )
2502 }
2503
2504 fn select_backend(&self, model: &ModelSpec) -> RuntimeResult<String> {
2505 if let Some(selected) = self.config.selected_backend.as_deref() {
2506 let backend = self
2507 .backends
2508 .get(selected)
2509 .ok_or_else(|| RuntimeError::backend_unavailable(selected, "not registered"))?;
2510 let capabilities = backend.capabilities();
2511 if !capabilities.available {
2512 return Err(RuntimeError::backend_unavailable(
2513 selected,
2514 "backend is not available on this platform",
2515 ));
2516 }
2517 if !backend.supports(model) {
2518 return Err(RuntimeError::UnsupportedModel {
2519 model: model.id.clone(),
2520 backend: selected.to_owned(),
2521 reason: "model format is not supported".to_owned(),
2522 });
2523 }
2524 return Ok(selected.to_owned());
2525 }
2526
2527 let mut candidates: Vec<_> = self
2528 .backends
2529 .iter()
2530 .filter_map(|(name, backend)| {
2531 let capabilities = backend.capabilities();
2532 if capabilities.available && backend.supports(model) {
2533 Some((
2534 name.clone(),
2535 !capabilities.accelerators.is_empty(),
2536 capabilities.hardware.clone(),
2537 ))
2538 } else {
2539 None
2540 }
2541 })
2542 .collect();
2543
2544 candidates.sort_by(|left, right| left.0.cmp(&right.0));
2545 if self.config.prefer_acceleration {
2546 candidates
2547 .sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0)));
2548 }
2549
2550 candidates
2551 .into_iter()
2552 .next()
2553 .map(|candidate| candidate.0)
2554 .ok_or_else(|| {
2555 RuntimeError::backend_unavailable("local", "no compatible backend registered")
2556 })
2557 }
2558}
2559
2560async fn run_with_timeout<F, T>(
2561 timeout_ms: Option<u64>,
2562 operation: &str,
2563 future: F,
2564) -> RuntimeResult<T>
2565where
2566 F: std::future::Future<Output = RuntimeResult<T>>,
2567{
2568 if let Some(timeout_ms) = timeout_ms {
2569 timeout(Duration::from_millis(timeout_ms), future)
2570 .await
2571 .map_err(|_| RuntimeError::Timeout {
2572 operation: operation.to_owned(),
2573 })?
2574 } else {
2575 future.await
2576 }
2577}
2578
2579fn model_id(reference: &ModelReference) -> String {
2580 match reference {
2581 ModelReference::Id { id, .. } => id.clone(),
2582 ModelReference::Spec(spec) => spec.id.clone(),
2583 }
2584}
2585
2586fn runtime_handle(loaded: &LoadedModel) -> RuntimeModelHandle {
2587 RuntimeModelHandle {
2588 model_id: loaded.spec.id.clone(),
2589 version: loaded.spec.version.clone(),
2590 provider: loaded.provider_name.clone(),
2591 backend: loaded.backend_name.clone(),
2592 state: ModelLifecycleState::Ready,
2593 }
2594}
2595
2596fn model_details(reference: &ModelReference) -> (String, Option<String>) {
2597 match reference {
2598 ModelReference::Id { id, version } => (id.clone(), version.clone()),
2599 ModelReference::Spec(spec) => (spec.id.clone(), spec.version.clone()),
2600 }
2601}
2602
2603fn catalog_error(error: CatalogError) -> RuntimeError {
2604 match error {
2605 CatalogError::ModelNotFound(model) => RuntimeError::ModelNotFound { model },
2606 CatalogError::InvalidIdentity(model) => RuntimeError::InvalidRequest {
2607 reason: format!("invalid model identity: {model}"),
2608 },
2609 CatalogError::ArtifactIntegrity { model, reason } => {
2610 RuntimeError::ModelIntegrity { model, reason }
2611 }
2612 other => RuntimeError::Execution {
2613 operation: "catalog".to_owned(),
2614 reason: other.to_string(),
2615 },
2616 }
2617}
2618
2619fn estimate_input_tokens(input: &Input) -> Option<u64> {
2620 match input {
2621 Input::Text(text) => Some(text.split_whitespace().count() as u64),
2622 Input::Tokens(tokens) => Some(tokens.len() as u64),
2623 _ => None,
2624 }
2625}
2626
2627fn estimate_output_tokens(output: &Output) -> Option<u64> {
2628 match output {
2629 Output::Text(text) => Some(text.split_whitespace().count() as u64),
2630 Output::Tokens(tokens) => Some(tokens.len() as u64),
2631 _ => None,
2632 }
2633}
2634
2635fn model_memory_bytes(model: &ModelSpec) -> u64 {
2636 match &model.location {
2637 ModelLocation::Path(path) => std::fs::metadata(path)
2638 .map(|metadata| metadata.len())
2639 .unwrap_or(0),
2640 _ => 0,
2641 }
2642}
2643
2644#[allow(clippy::too_many_arguments)]
2645fn decorate_metadata(
2646 metadata: &mut ExecutionMetadata,
2647 output: &Output,
2648 model: &str,
2649 provider: &str,
2650 backend: &str,
2651 hardware: Option<String>,
2652 elapsed: Duration,
2653 input_tokens: Option<u64>,
2654 model_version: Option<String>,
2655 routing_policy: ExecutionPolicy,
2656 fallback: bool,
2657 fallback_from: Option<String>,
2658 fallback_reason: Option<String>,
2659) {
2660 metadata.model = model.to_owned();
2661 metadata.model_version = model_version;
2662 metadata.provider = provider.to_owned();
2663 metadata.backend = backend.to_owned();
2664 metadata.execution_target = provider.to_owned();
2665 metadata.routing_policy = routing_policy;
2666 metadata.fallback = fallback;
2667 metadata.fallback_from = fallback_from;
2668 metadata.fallback_reason = fallback_reason;
2669 metadata.hardware = hardware;
2670 metadata.latency_ms = Some(elapsed.as_secs_f64() * 1000.0);
2671 metadata.execution_ms = metadata.latency_ms;
2672 metadata.input_tokens = input_tokens;
2673 metadata.output_tokens = estimate_output_tokens(output);
2674}
2675
2676#[allow(clippy::too_many_arguments)]
2677fn normalize_stream(
2678 mut source: BoxStream<RuntimeResult<InferenceChunk>>,
2679 request: InferenceRequest,
2680 selection: RuntimeSelection,
2681 streaming_supported: bool,
2682 cache_hit: bool,
2683 buffer: usize,
2684 cancellation: CancellationToken,
2685 metrics: Arc<RuntimeMetrics>,
2686) -> InferenceStream {
2687 let (sender, receiver) = mpsc::channel(buffer);
2688 tokio::spawn(async move {
2689 let (model, version) = model_details(&request.model);
2690 let mut metadata = ExecutionMetadata {
2691 model,
2692 model_version: version,
2693 provider: selection.provider.clone(),
2694 execution_target: selection.provider.clone(),
2695 routing_policy: selection.policy.clone(),
2696 fallback: selection.fallback,
2697 fallback_from: selection.fallback_from.clone(),
2698 fallback_reason: Some(selection.reason.clone()),
2699 streaming_requested: true,
2700 streaming_supported,
2701 cache_hit,
2702 completion_state: Some("started".to_owned()),
2703 ..ExecutionMetadata::default()
2704 };
2705 if sender
2706 .send(Ok(InferenceStreamEvent::Started(metadata.clone())))
2707 .await
2708 .is_err()
2709 {
2710 return;
2711 }
2712
2713 let mut output_count = 0;
2714 while let Some(item) = tokio::select! {
2715 item = source.next() => item,
2716 _ = cancellation.cancelled() => {
2717 metrics.cancelled_streams.fetch_add(1, Ordering::Relaxed);
2718 let _ = sender.send(Err(RuntimeError::Cancelled)).await;
2719 return;
2720 }
2721 } {
2722 match item {
2723 Ok(chunk) => {
2724 if let Some(chunk_metadata) = chunk.metadata {
2725 metadata = chunk_metadata;
2726 }
2727 metadata.streaming_requested = true;
2728 metadata.streaming_supported = streaming_supported;
2729 metadata.cache_hit = cache_hit;
2730 metadata.output_event_count = output_count + 1;
2731 output_count += 1;
2732 if sender
2733 .send(Ok(InferenceStreamEvent::Output(chunk.output)))
2734 .await
2735 .is_err()
2736 {
2737 return;
2738 }
2739 metrics
2740 .streamed_output_events
2741 .fetch_add(1, Ordering::Relaxed);
2742 if chunk.done {
2743 metadata.output_event_count = output_count;
2744 metadata.completion_state = Some("completed".to_owned());
2745 let _ = sender
2746 .send(Ok(InferenceStreamEvent::Completed(metadata)))
2747 .await;
2748 return;
2749 }
2750 }
2751 Err(error) => {
2752 let _ = sender.send(Err(error)).await;
2753 return;
2754 }
2755 }
2756 }
2757 metadata.output_event_count = output_count;
2758 metadata.completion_state = Some("completed".to_owned());
2759 let _ = sender
2760 .send(Ok(InferenceStreamEvent::Completed(metadata)))
2761 .await;
2762 });
2763 Box::pin(ReceiverStream::new(receiver))
2764}
2765
2766#[cfg(test)]
2767mod tests {
2768 use super::*;
2769 use async_trait::async_trait;
2770 use futures_util::StreamExt;
2771 use ml_runtime_backend::BackendCapabilities;
2772 use ml_runtime_common::BoxStream;
2773 use ml_runtime_cpu_backend::CpuBackend;
2774 use ml_runtime_inference::InferenceOptions;
2775 use ml_runtime_provider::{Provider, ProviderCapabilities};
2776 use serde_json::json;
2777 use tokio_stream::iter;
2778
2779 #[derive(Clone)]
2780 struct BlockingBackend {
2781 started: Arc<Semaphore>,
2782 }
2783
2784 #[async_trait]
2785 impl Backend for BlockingBackend {
2786 fn name(&self) -> &str {
2787 "blocking"
2788 }
2789
2790 fn capabilities(&self) -> BackendCapabilities {
2791 BackendCapabilities {
2792 available: true,
2793 local: true,
2794 streaming: true,
2795 cancellation: true,
2796 supported_formats: vec![ModelFormat::Unknown],
2797 ..BackendCapabilities::default()
2798 }
2799 }
2800
2801 fn supports(&self, model: &ModelSpec) -> bool {
2802 model.format == ModelFormat::Unknown
2803 }
2804
2805 async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle> {
2806 Ok(ModelHandle::new(
2807 model.id.clone(),
2808 self.name(),
2809 model.format.clone(),
2810 Arc::new(()),
2811 ))
2812 }
2813
2814 async fn infer(
2815 &self,
2816 _model: &ModelHandle,
2817 _request: &InferenceRequest,
2818 cancellation: Option<CancellationToken>,
2819 ) -> RuntimeResult<InferenceResult> {
2820 self.started.add_permits(1);
2821 cancellation
2822 .expect("runtime supplies cancellation")
2823 .cancelled()
2824 .await;
2825 Err(RuntimeError::Cancelled)
2826 }
2827
2828 async fn infer_stream(
2829 &self,
2830 _model: &ModelHandle,
2831 _request: &InferenceRequest,
2832 _cancellation: Option<CancellationToken>,
2833 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
2834 Ok(Box::pin(futures_util::stream::pending()))
2835 }
2836 }
2837
2838 #[test]
2839 fn runtime_version_is_workspace_version() {
2840 assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
2841 }
2842
2843 #[derive(Clone)]
2844 struct MockRemoteProvider;
2845
2846 #[async_trait]
2847 impl Provider for MockRemoteProvider {
2848 fn name(&self) -> &str {
2849 "mock-remote"
2850 }
2851
2852 fn capabilities(&self) -> ProviderCapabilities {
2853 ProviderCapabilities {
2854 available: true,
2855 local: false,
2856 remote: true,
2857 streaming: true,
2858 cancellation: true,
2859 structured_output: true,
2860 batching: true,
2861 endpoint: Some("mock://remote".to_owned()),
2862 notes: vec!["test remote provider".to_owned()],
2863 }
2864 }
2865
2866 async fn infer(
2867 &self,
2868 request: InferenceRequest,
2869 cancellation: Option<CancellationToken>,
2870 ) -> RuntimeResult<InferenceResult> {
2871 if model_id(&request.model) == "blocking-remote" {
2872 cancellation
2873 .expect("runtime supplies cancellation")
2874 .cancelled()
2875 .await;
2876 return Err(RuntimeError::Cancelled);
2877 }
2878 if cancellation
2879 .as_ref()
2880 .is_some_and(CancellationToken::is_cancelled)
2881 {
2882 return Err(RuntimeError::Cancelled);
2883 }
2884 Ok(InferenceResult {
2885 output: Output::Structured(json!({"remote": model_id(&request.model)})),
2886 metadata: ExecutionMetadata {
2887 backend: "remote-api".to_owned(),
2888 hardware: Some("network".to_owned()),
2889 ..ExecutionMetadata::default()
2890 },
2891 })
2892 }
2893
2894 async fn infer_stream(
2895 &self,
2896 request: InferenceRequest,
2897 _cancellation: Option<CancellationToken>,
2898 ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
2899 let stream = iter(vec![Ok(InferenceChunk {
2900 output: Output::Text(model_id(&request.model)),
2901 done: true,
2902 metadata: None,
2903 })]);
2904 Ok(Box::pin(stream))
2905 }
2906 }
2907
2908 #[tokio::test]
2909 async fn loads_and_runs_local_cpu_models() {
2910 let runtime = Runtime::builder()
2911 .register_backend(CpuBackend::default())
2912 .build();
2913 let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
2914 runtime.load(model.clone()).await.unwrap();
2915
2916 let result = runtime
2917 .infer(InferenceRequest {
2918 model: model.into(),
2919 input: Input::Text("hello runtime".to_owned()),
2920 options: InferenceOptions::default(),
2921 })
2922 .await
2923 .unwrap();
2924
2925 assert_eq!(result.output, Output::Text("hello runtime".to_owned()));
2926 assert_eq!(result.metadata.provider, "local");
2927 assert_eq!(result.metadata.backend, "cpu");
2928 }
2929
2930 #[tokio::test]
2931 async fn supports_remote_provider_selection() {
2932 let runtime = Runtime::builder()
2933 .register_backend(CpuBackend::default())
2934 .register_provider(MockRemoteProvider)
2935 .provider("mock-remote")
2936 .build();
2937
2938 let result = runtime
2939 .infer(InferenceRequest {
2940 model: ModelReference::id("remote-model", None),
2941 input: Input::Text("hello".to_owned()),
2942 options: InferenceOptions {
2943 require_remote: true,
2944 ..InferenceOptions::default()
2945 },
2946 })
2947 .await
2948 .unwrap();
2949
2950 assert_eq!(result.metadata.provider, "mock-remote");
2951 assert_eq!(result.metadata.backend, "remote-api");
2952 }
2953
2954 #[tokio::test]
2955 async fn only_falls_back_to_remote_when_enabled() {
2956 let request = InferenceRequest {
2957 model: ModelReference::id("missing", None),
2958 input: Input::Text("hello".to_owned()),
2959 options: InferenceOptions::default(),
2960 };
2961
2962 let local_only = Runtime::builder()
2963 .register_backend(CpuBackend::default())
2964 .register_provider(MockRemoteProvider)
2965 .build();
2966 assert!(matches!(
2967 local_only.infer(request.clone()).await,
2968 Err(RuntimeError::ModelNotFound { .. })
2969 ));
2970
2971 let hybrid = Runtime::builder()
2972 .register_backend(CpuBackend::default())
2973 .register_provider(MockRemoteProvider)
2974 .allow_remote_fallback(true)
2975 .build();
2976 let result = hybrid.infer(request).await.unwrap();
2977 assert_eq!(result.metadata.provider, "mock-remote");
2978 assert!(result.metadata.fallback);
2979 assert_eq!(result.metadata.fallback_from.as_deref(), Some("local"));
2980 }
2981
2982 #[tokio::test]
2983 async fn explicit_policies_choose_deterministic_targets() {
2984 let local = Runtime::builder()
2985 .register_backend(CpuBackend::default())
2986 .register_provider(MockRemoteProvider)
2987 .build();
2988 let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
2989 local.load(model.clone()).await.unwrap();
2990
2991 let local_result = local
2992 .infer(InferenceRequest {
2993 model: model.clone().into(),
2994 input: Input::Text("hello".to_owned()),
2995 options: InferenceOptions {
2996 execution: ExecutionPolicy::PreferLocal,
2997 ..InferenceOptions::default()
2998 },
2999 })
3000 .await
3001 .unwrap();
3002 assert_eq!(local_result.metadata.provider, "local");
3003
3004 let remote_result = local
3005 .infer(InferenceRequest {
3006 model: ModelReference::id("remote-model", None),
3007 input: Input::Text("hello".to_owned()),
3008 options: InferenceOptions {
3009 execution: ExecutionPolicy::PreferRemote,
3010 ..InferenceOptions::default()
3011 },
3012 })
3013 .await
3014 .unwrap();
3015 assert_eq!(remote_result.metadata.provider, "mock-remote");
3016 assert_eq!(
3017 remote_result.metadata.routing_policy,
3018 ExecutionPolicy::PreferRemote
3019 );
3020 }
3021
3022 #[tokio::test]
3023 async fn rejects_requests_above_batch_limit() {
3024 let runtime = Runtime::builder()
3025 .register_backend(CpuBackend::default())
3026 .max_batch_size(2)
3027 .build();
3028
3029 let err = runtime
3030 .infer(InferenceRequest {
3031 model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3032 input: Input::Tokens(vec![1, 2, 3]),
3033 options: InferenceOptions {
3034 batch_size: Some(3),
3035 ..InferenceOptions::default()
3036 },
3037 })
3038 .await
3039 .unwrap_err();
3040
3041 assert!(matches!(err, RuntimeError::ResourceLimit { .. }));
3042 }
3043
3044 #[tokio::test]
3045 async fn propagates_cancellation() {
3046 let runtime = Runtime::builder()
3047 .register_backend(CpuBackend::default())
3048 .build();
3049 let token = CancellationToken::new();
3050 token.cancel();
3051
3052 let err = runtime
3053 .infer_with_cancellation(
3054 InferenceRequest {
3055 model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory)
3056 .into(),
3057 input: Input::Text("hello".to_owned()),
3058 options: InferenceOptions::default(),
3059 },
3060 token,
3061 )
3062 .await
3063 .unwrap_err();
3064
3065 assert!(matches!(err, RuntimeError::Cancelled));
3066 }
3067
3068 #[tokio::test]
3069 async fn cancels_active_and_queued_inference() {
3070 let started = Arc::new(Semaphore::new(0));
3071 let runtime = Arc::new(
3072 Runtime::builder()
3073 .register_backend(BlockingBackend {
3074 started: Arc::clone(&started),
3075 })
3076 .backend("blocking")
3077 .concurrency(1)
3078 .build(),
3079 );
3080 let make_request = || {
3081 InferenceRequest::new(
3082 ModelSpec::new("blocking", ModelFormat::Unknown, ModelLocation::Memory),
3083 "input",
3084 )
3085 };
3086
3087 let active_token = CancellationToken::new();
3088 let active = {
3089 let runtime = Arc::clone(&runtime);
3090 let token = active_token.clone();
3091 tokio::spawn(
3092 async move { runtime.infer_with_cancellation(make_request(), token).await },
3093 )
3094 };
3095 let permit = started.acquire().await.unwrap();
3096 permit.forget();
3097
3098 let queued_token = CancellationToken::new();
3099 let queued = {
3100 let runtime = Arc::clone(&runtime);
3101 let token = queued_token.clone();
3102 tokio::spawn(
3103 async move { runtime.infer_with_cancellation(make_request(), token).await },
3104 )
3105 };
3106 tokio::task::yield_now().await;
3107 queued_token.cancel();
3108 assert!(matches!(
3109 queued.await.unwrap(),
3110 Err(RuntimeError::Cancelled)
3111 ));
3112
3113 active_token.cancel();
3114 assert!(matches!(
3115 active.await.unwrap(),
3116 Err(RuntimeError::Cancelled)
3117 ));
3118 }
3119
3120 #[tokio::test]
3121 async fn cancels_an_active_stream() {
3122 let runtime = Runtime::builder()
3123 .register_backend(BlockingBackend {
3124 started: Arc::new(Semaphore::new(0)),
3125 })
3126 .backend("blocking")
3127 .build();
3128 let token = CancellationToken::new();
3129 let mut stream = runtime
3130 .infer_stream_with_cancellation(
3131 InferenceRequest::new(
3132 ModelSpec::new("blocking", ModelFormat::Unknown, ModelLocation::Memory),
3133 "input",
3134 )
3135 .with_options(InferenceOptions {
3136 stream: true,
3137 ..InferenceOptions::default()
3138 }),
3139 token.clone(),
3140 )
3141 .await
3142 .unwrap();
3143 assert!(matches!(
3144 stream.next().await.unwrap().unwrap(),
3145 InferenceStreamEvent::Started(_)
3146 ));
3147 token.cancel();
3148 assert!(matches!(
3149 stream.next().await.unwrap(),
3150 Err(RuntimeError::Cancelled)
3151 ));
3152 }
3153
3154 #[tokio::test]
3155 async fn propagates_remote_cancellation() {
3156 let runtime = Arc::new(
3157 Runtime::builder()
3158 .register_provider(MockRemoteProvider)
3159 .provider("mock-remote")
3160 .build(),
3161 );
3162 let token = CancellationToken::new();
3163 let task = {
3164 let runtime = Arc::clone(&runtime);
3165 let token = token.clone();
3166 tokio::spawn(async move {
3167 runtime
3168 .infer_with_cancellation(
3169 InferenceRequest::new(ModelReference::latest("blocking-remote"), "input")
3170 .with_options(InferenceOptions {
3171 execution: ExecutionPolicy::RemoteOnly,
3172 ..InferenceOptions::default()
3173 }),
3174 token,
3175 )
3176 .await
3177 })
3178 };
3179 tokio::task::yield_now().await;
3180 token.cancel();
3181 assert!(matches!(task.await.unwrap(), Err(RuntimeError::Cancelled)));
3182 }
3183
3184 #[tokio::test]
3185 async fn streams_from_local_backend() {
3186 let runtime = Runtime::builder()
3187 .register_backend(CpuBackend::default())
3188 .build();
3189
3190 let mut stream = runtime
3191 .infer_stream(InferenceRequest {
3192 model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3193 input: Input::Text("hello runtime".to_owned()),
3194 options: InferenceOptions {
3195 stream: true,
3196 ..InferenceOptions::default()
3197 },
3198 })
3199 .await
3200 .unwrap();
3201
3202 assert!(matches!(
3203 stream.next().await.unwrap().unwrap(),
3204 InferenceStreamEvent::Started(_)
3205 ));
3206 assert!(matches!(
3207 stream.next().await.unwrap().unwrap(),
3208 InferenceStreamEvent::Output(Output::Text(text)) if text == "hello"
3209 ));
3210 assert!(matches!(
3211 stream.next().await.unwrap().unwrap(),
3212 InferenceStreamEvent::Output(Output::Text(text)) if text == "runtime"
3213 ));
3214 assert!(matches!(
3215 stream.next().await.unwrap().unwrap(),
3216 InferenceStreamEvent::Completed(metadata)
3217 if metadata.output_event_count == 2 && metadata.streaming_supported
3218 ));
3219 }
3220
3221 #[tokio::test]
3222 async fn rejects_stream_calls_without_stream_flag() {
3223 let runtime = Runtime::builder()
3224 .register_backend(CpuBackend::default())
3225 .build();
3226 let result = runtime
3227 .infer_stream(InferenceRequest {
3228 model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3229 input: Input::Text("hello".to_owned()),
3230 options: InferenceOptions::default(),
3231 })
3232 .await;
3233
3234 assert!(matches!(
3235 result,
3236 Err(RuntimeError::CapabilityMismatch { .. })
3237 ));
3238 }
3239
3240 #[tokio::test]
3241 async fn streams_from_remote_provider() {
3242 let runtime = Runtime::builder()
3243 .register_backend(CpuBackend::default())
3244 .register_provider(MockRemoteProvider)
3245 .provider("mock-remote")
3246 .build();
3247
3248 let mut stream = runtime
3249 .infer_stream(InferenceRequest {
3250 model: ModelReference::id("remote-model", None),
3251 input: Input::Text("hello".to_owned()),
3252 options: InferenceOptions {
3253 stream: true,
3254 require_remote: true,
3255 ..InferenceOptions::default()
3256 },
3257 })
3258 .await
3259 .unwrap();
3260
3261 assert!(matches!(
3262 stream.next().await.unwrap().unwrap(),
3263 InferenceStreamEvent::Started(_)
3264 ));
3265 assert!(matches!(
3266 stream.next().await.unwrap().unwrap(),
3267 InferenceStreamEvent::Output(Output::Text(text)) if text == "remote-model"
3268 ));
3269 assert!(matches!(
3270 stream.next().await.unwrap().unwrap(),
3271 InferenceStreamEvent::Completed(metadata)
3272 if metadata.output_event_count == 1
3273 ));
3274 }
3275
3276 #[tokio::test]
3277 async fn explicit_handles_reuse_loaded_models_and_report_cache_hits() {
3278 let runtime = Runtime::builder()
3279 .register_backend(CpuBackend::default())
3280 .build();
3281 let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
3282 let handle = runtime.load(model).await.unwrap();
3283 assert_eq!(handle.id(), "echo");
3284 let result = runtime
3285 .infer_loaded(
3286 &handle,
3287 Input::Text("cached".to_owned()),
3288 Default::default(),
3289 )
3290 .await
3291 .unwrap();
3292 assert!(result.metadata.cache_hit);
3293 assert_eq!(runtime.models().list().len(), 1);
3294 }
3295
3296 #[tokio::test]
3297 async fn loaded_stream_reuses_the_cached_model() {
3298 let runtime = Runtime::builder()
3299 .register_backend(CpuBackend::default())
3300 .build();
3301 let handle = runtime
3302 .load(ModelSpec::new(
3303 "echo",
3304 ModelFormat::Unknown,
3305 ModelLocation::Memory,
3306 ))
3307 .await
3308 .unwrap();
3309 let mut stream = runtime
3310 .infer_loaded_stream(
3311 &handle,
3312 Input::Text("cached stream".to_owned()),
3313 InferenceOptions::default(),
3314 )
3315 .await
3316 .unwrap();
3317 let events = futures_util::StreamExt::collect::<Vec<_>>(&mut stream).await;
3318 assert!(matches!(
3319 events.first().and_then(|event| event.as_ref().ok()),
3320 Some(InferenceStreamEvent::Started(_))
3321 ));
3322 assert!(matches!(
3323 events.last().and_then(|event| event.as_ref().ok()),
3324 Some(InferenceStreamEvent::Completed(metadata)) if metadata.cache_hit
3325 ));
3326 assert_eq!(runtime.models().list().len(), 1);
3327 }
3328
3329 #[tokio::test]
3330 async fn cache_evicts_the_least_recently_used_target() {
3331 let runtime = Runtime::builder()
3332 .register_backend(CpuBackend::default())
3333 .max_models(1)
3334 .build();
3335 runtime
3336 .load(ModelSpec::new(
3337 "first",
3338 ModelFormat::Unknown,
3339 ModelLocation::Memory,
3340 ))
3341 .await
3342 .unwrap();
3343 runtime
3344 .load(ModelSpec::new(
3345 "second",
3346 ModelFormat::Unknown,
3347 ModelLocation::Memory,
3348 ))
3349 .await
3350 .unwrap();
3351 assert_eq!(runtime.models().list().len(), 1);
3352 assert_eq!(runtime.models().list()[0].id, "second");
3353 }
3354
3355 #[tokio::test]
3356 async fn batch_inference_preserves_result_order_and_identity() {
3357 let runtime = Runtime::builder()
3358 .register_backend(CpuBackend::default())
3359 .build();
3360 let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
3361 let results = runtime
3362 .infer_batch(vec![
3363 InferenceRequest {
3364 model: model.clone().into(),
3365 input: Input::Text("a".to_owned()),
3366 options: Default::default(),
3367 },
3368 InferenceRequest {
3369 model: model.into(),
3370 input: Input::Text("b".to_owned()),
3371 options: Default::default(),
3372 },
3373 ])
3374 .await
3375 .unwrap();
3376 assert_eq!(results.len(), 2);
3377 assert_eq!(results[0].output, Output::Text("a".to_owned()));
3378 assert_eq!(results[1].output, Output::Text("b".to_owned()));
3379 assert_eq!(results[0].metadata.batch_size, 2);
3380 assert_eq!(results[1].metadata.batch_size, 2);
3381 }
3382
3383 #[tokio::test]
3384 async fn records_execution_metrics_and_snapshot() {
3385 let runtime = Runtime::builder()
3386 .register_backend(CpuBackend::default())
3387 .build();
3388 let model = ModelSpec::new("observed", ModelFormat::Unknown, ModelLocation::Memory);
3389 let result = runtime
3390 .infer(InferenceRequest {
3391 model: model.into(),
3392 input: Input::Text("hello".to_owned()),
3393 options: Default::default(),
3394 })
3395 .await
3396 .unwrap();
3397
3398 let metrics = runtime.metrics();
3399 assert_eq!(metrics.total_requests, 1);
3400 assert_eq!(metrics.successful_requests, 1);
3401 assert_eq!(runtime.executions().len(), 1);
3402 assert_eq!(
3403 runtime.executions()[0].completion,
3404 ExecutionCompletion::Succeeded
3405 );
3406 assert_eq!(
3407 runtime.executions()[0].request_id,
3408 result.metadata.request_id.clone().unwrap()
3409 );
3410 assert_eq!(runtime.snapshot().loaded_models, 1);
3411 }
3412}