Skip to main content

rust_ml_runtime/
lib.rs

1#![cfg_attr(test, allow(clippy::default_constructed_unit_structs))]
2
3use futures_util::StreamExt;
4use ml_runtime_backend::{Backend, BackendCapability};
5use ml_runtime_common::BoxStream;
6use ml_runtime_cpu_backend::CpuBackend;
7use ml_runtime_http_provider::HttpProvider;
8use ml_runtime_inference::InferenceChunk;
9use ml_runtime_model::{CatalogError, ModelCatalog, ModelHandle};
10use ml_runtime_onnx_backend::OnnxBackend;
11use ml_runtime_provider::{Provider, ProviderCapability};
12use serde::{Deserialize, Serialize};
13use std::{
14    collections::{BTreeMap, HashMap, VecDeque},
15    hash::{Hash, Hasher},
16    path::{Path, PathBuf},
17    sync::{
18        atomic::{AtomicU64, AtomicUsize, Ordering},
19        Arc, Mutex, RwLock,
20    },
21    time::{Duration, Instant},
22};
23use tokio::{
24    sync::{mpsc, Semaphore},
25    time::timeout,
26};
27use tokio_stream::wrappers::ReceiverStream;
28
29mod capability;
30mod decision;
31
32pub use capability::{
33    AvailabilityState, BuiltinCapabilityProvider, Capability, CapabilityAuthorizer,
34    CapabilityAvailability, CapabilityExecutionMetadata, CapabilityLimits,
35    CapabilityPackageManifest, CapabilityPolicy, CapabilityProvider, CapabilityRegistry,
36    CapabilityResolution, ExecutionRequest, ExecutionResult, LocalProviderPackage,
37    ManifestCapabilityProvider, PackageCapability, PackageManifest, ProviderDescriptor,
38    ProviderManifest, ResolutionCandidate, ResolutionConstraints,
39};
40pub use decision::DecisionPlanner;
41
42pub use ml_runtime_backend;
43pub use ml_runtime_backend::{DecisionModel, DecisionModelProvider};
44pub use ml_runtime_common;
45pub use ml_runtime_common::{CancellationToken, RuntimeError, RuntimeResult};
46pub use ml_runtime_inference;
47pub use ml_runtime_inference::{
48    DecisionDependency, DecisionDependencyCondition, DecisionExecution,
49    DecisionExecutionDiagnostics, DecisionExecutionPlan, DecisionExecutionPolicy,
50    DecisionExecutionStrategy, DecisionGraphRequest, DecisionModelCapabilities, DecisionNode,
51    DecisionOption, DecisionPlanStage, DecisionProvenance, DecisionQuestion, DecisionRequest,
52    DecisionResult, DecisionType, DecisionValue, DeviceKind, ExecutionMetadata, ExecutionPolicy,
53    InferenceOptions, InferenceRequest, InferenceResult, InferenceStreamEvent, Input,
54    ModelArchitecture, ModelDescription, ModelIdentity, Output, PlannedDecisionResult, Tensor,
55    TypedDecision,
56};
57pub use ml_runtime_model;
58pub use ml_runtime_model::{
59    default_model_root, BuiltinModelRegistry, CompiledArtifactManifest, CompiledStatus,
60    FilesystemModelCatalog, InstallationManifest, ModelDescriptor, ModelFormat, ModelId,
61    ModelLocation, ModelReference, ModelSpec, RegisteredModelInstaller,
62};
63pub use ml_runtime_provider;
64
65pub const VERSION: &str = env!("CARGO_PKG_VERSION");
66
67#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
68#[serde(rename_all = "snake_case")]
69pub enum InstalledModelStatus {
70    NotInstalled,
71    InstalledNotCompiled,
72    Ready,
73    Corrupt,
74    Incompatible,
75    Unsupported,
76}
77
78impl std::fmt::Display for InstalledModelStatus {
79    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
80        formatter.write_str(match self {
81            Self::NotInstalled => "not_installed",
82            Self::InstalledNotCompiled => "installed_not_compiled",
83            Self::Ready => "ready",
84            Self::Corrupt => "corrupt",
85            Self::Incompatible => "incompatible",
86            Self::Unsupported => "unsupported",
87        })
88    }
89}
90
91#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
92pub struct InstalledModelRecord {
93    pub model: String,
94    pub revision: String,
95    pub backend: String,
96    pub platform: String,
97    pub status: InstalledModelStatus,
98    pub compiled: bool,
99    pub package_path: PathBuf,
100}
101
102#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
103pub struct ModelDiagnostic {
104    pub check: String,
105    pub ok: bool,
106    pub message: String,
107}
108
109#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
110pub struct ModelDoctorReport {
111    pub model: InstalledModelRecord,
112    pub diagnostics: Vec<ModelDiagnostic>,
113    pub manifest: Option<InstallationManifest>,
114}
115
116#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
117pub struct ModelLifecycleResult {
118    pub model: InstalledModelRecord,
119    pub already_installed: bool,
120    pub duration_ms: u128,
121}
122
123/// Bounded asynchronous stream of normalized runtime-owned inference events.
124pub type InferenceStream = BoxStream<RuntimeResult<InferenceStreamEvent>>;
125
126#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
127pub enum Platform {
128    Linux,
129    Macos,
130    Windows,
131    Unknown,
132}
133
134impl Platform {
135    fn current() -> Self {
136        if cfg!(target_os = "linux") {
137            Self::Linux
138        } else if cfg!(target_os = "macos") {
139            Self::Macos
140        } else if cfg!(target_os = "windows") {
141            Self::Windows
142        } else {
143            Self::Unknown
144        }
145    }
146}
147
148#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
149pub struct ModelCapability {
150    pub id: String,
151    pub format: ModelFormat,
152    pub location: ModelLocation,
153    pub loaded: bool,
154    pub provider: Option<String>,
155    pub backend: Option<String>,
156}
157
158#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
159pub struct RuntimeCapabilities {
160    pub environment: RuntimeEnvironment,
161    pub platforms: Vec<Platform>,
162    pub providers: Vec<ProviderCapability>,
163    pub backends: Vec<BackendCapability>,
164    pub models: Vec<ModelCapability>,
165    pub capabilities: Vec<Capability>,
166}
167
168#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
169pub struct RuntimeEnvironment {
170    pub platform: String,
171    pub architecture: String,
172    pub operating_system: String,
173    pub cpu: String,
174    pub memory_bytes: Option<u64>,
175    pub accelerators: Vec<String>,
176    pub filesystem: String,
177    pub network: String,
178    pub developer_tools: Vec<String>,
179    pub installed_providers: Vec<String>,
180}
181
182impl RuntimeEnvironment {
183    pub fn discover(provider_ids: Vec<String>) -> Self {
184        Self {
185            platform: std::env::consts::OS.to_owned(),
186            architecture: std::env::consts::ARCH.to_owned(),
187            operating_system: std::env::consts::OS.to_owned(),
188            cpu: std::thread::available_parallelism()
189                .map(|count| format!("{} logical cores", count.get()))
190                .unwrap_or_else(|_| "unknown".to_owned()),
191            memory_bytes: available_memory_bytes(),
192            accelerators: Vec::new(),
193            filesystem: std::env::current_dir()
194                .map(|path| path.display().to_string())
195                .unwrap_or_else(|_| "unavailable".to_owned()),
196            network: "available by explicit policy".to_owned(),
197            developer_tools: ["git"]
198                .iter()
199                .filter(|tool| command_available(tool))
200                .map(|tool| (*tool).to_owned())
201                .collect(),
202            installed_providers: provider_ids,
203        }
204    }
205}
206
207fn command_available(command: &str) -> bool {
208    std::process::Command::new(command)
209        .arg("--version")
210        .output()
211        .map(|output| output.status.success())
212        .unwrap_or(false)
213}
214
215fn available_memory_bytes() -> Option<u64> {
216    let contents = std::fs::read_to_string("/proc/meminfo").ok()?;
217    contents.lines().find_map(|line| {
218        let value = line
219            .strip_prefix("MemTotal:")?
220            .split_whitespace()
221            .next()?
222            .parse::<u64>()
223            .ok()?;
224        Some(value * 1024)
225    })
226}
227
228#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
229pub struct RuntimeStatus {
230    pub loaded_models: usize,
231    pub max_models: usize,
232    pub max_concurrent_inferences: usize,
233    pub max_concurrent_model_loads: usize,
234}
235
236pub type ExecutionId = String;
237pub type RequestId = String;
238
239#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
240pub enum ExecutionCompletion {
241    Succeeded,
242    Failed,
243    Cancelled,
244}
245
246#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
247pub struct ExecutionRecord {
248    pub execution_id: ExecutionId,
249    pub request_id: RequestId,
250    pub model_id: String,
251    pub model_version: String,
252    pub provider: String,
253    pub backend: String,
254    pub target: String,
255    pub routing_policy: ExecutionPolicy,
256    pub fallback: Option<String>,
257    pub cache_hit: bool,
258    pub batch_size: usize,
259    pub queued_duration: Duration,
260    pub model_load_duration: Duration,
261    pub execution_duration: Duration,
262    pub total_duration: Duration,
263    pub streaming: bool,
264    pub output_event_count: usize,
265    pub completion: ExecutionCompletion,
266}
267
268#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
269pub struct RuntimeMetricsSnapshot {
270    pub total_requests: u64,
271    pub successful_requests: u64,
272    pub failed_requests: u64,
273    pub cancelled_requests: u64,
274    pub local_executions: u64,
275    pub remote_executions: u64,
276    pub fallback_executions: u64,
277    pub fallback_failures: u64,
278    pub model_loads: u64,
279    pub model_load_failures: u64,
280    pub cache_hits: u64,
281    pub cache_misses: u64,
282    pub evictions: u64,
283    pub queued_requests: u64,
284    pub active_inferences: usize,
285    pub active_model_loads: usize,
286    pub batch_executions: u64,
287    pub individual_executions: u64,
288    pub total_requests_batched: u64,
289    pub streaming_requests: u64,
290    pub streamed_output_events: u64,
291    pub cancelled_streams: u64,
292    pub compatibility_streams: u64,
293    pub queue_wait_micros: Vec<u64>,
294    pub model_load_micros: Vec<u64>,
295    pub execution_micros: Vec<u64>,
296    pub total_duration_micros: Vec<u64>,
297    pub batch_sizes: Vec<usize>,
298}
299
300#[derive(Default)]
301pub struct RuntimeMetrics {
302    total_requests: AtomicU64,
303    successful_requests: AtomicU64,
304    failed_requests: AtomicU64,
305    cancelled_requests: AtomicU64,
306    local_executions: AtomicU64,
307    remote_executions: AtomicU64,
308    fallback_executions: AtomicU64,
309    fallback_failures: AtomicU64,
310    model_loads: AtomicU64,
311    model_load_failures: AtomicU64,
312    cache_hits: AtomicU64,
313    cache_misses: AtomicU64,
314    evictions: AtomicU64,
315    queued_requests: AtomicU64,
316    active_inferences: AtomicUsize,
317    active_model_loads: AtomicUsize,
318    batch_executions: AtomicU64,
319    individual_executions: AtomicU64,
320    total_requests_batched: AtomicU64,
321    streaming_requests: AtomicU64,
322    streamed_output_events: AtomicU64,
323    cancelled_streams: AtomicU64,
324    compatibility_streams: AtomicU64,
325    queue_wait_micros: Mutex<Vec<u64>>,
326    model_load_micros: Mutex<Vec<u64>>,
327    execution_micros: Mutex<Vec<u64>>,
328    total_duration_micros: Mutex<Vec<u64>>,
329    batch_sizes: Mutex<Vec<usize>>,
330}
331
332impl RuntimeMetrics {
333    pub fn snapshot(&self) -> RuntimeMetricsSnapshot {
334        let values = |values: &Mutex<Vec<u64>>| values.lock().expect("metrics poisoned").clone();
335        RuntimeMetricsSnapshot {
336            total_requests: self.total_requests.load(Ordering::Relaxed),
337            successful_requests: self.successful_requests.load(Ordering::Relaxed),
338            failed_requests: self.failed_requests.load(Ordering::Relaxed),
339            cancelled_requests: self.cancelled_requests.load(Ordering::Relaxed),
340            local_executions: self.local_executions.load(Ordering::Relaxed),
341            remote_executions: self.remote_executions.load(Ordering::Relaxed),
342            fallback_executions: self.fallback_executions.load(Ordering::Relaxed),
343            fallback_failures: self.fallback_failures.load(Ordering::Relaxed),
344            model_loads: self.model_loads.load(Ordering::Relaxed),
345            model_load_failures: self.model_load_failures.load(Ordering::Relaxed),
346            cache_hits: self.cache_hits.load(Ordering::Relaxed),
347            cache_misses: self.cache_misses.load(Ordering::Relaxed),
348            evictions: self.evictions.load(Ordering::Relaxed),
349            queued_requests: self.queued_requests.load(Ordering::Relaxed),
350            active_inferences: self.active_inferences.load(Ordering::Relaxed),
351            active_model_loads: self.active_model_loads.load(Ordering::Relaxed),
352            batch_executions: self.batch_executions.load(Ordering::Relaxed),
353            individual_executions: self.individual_executions.load(Ordering::Relaxed),
354            total_requests_batched: self.total_requests_batched.load(Ordering::Relaxed),
355            streaming_requests: self.streaming_requests.load(Ordering::Relaxed),
356            streamed_output_events: self.streamed_output_events.load(Ordering::Relaxed),
357            cancelled_streams: self.cancelled_streams.load(Ordering::Relaxed),
358            compatibility_streams: self.compatibility_streams.load(Ordering::Relaxed),
359            queue_wait_micros: values(&self.queue_wait_micros),
360            model_load_micros: values(&self.model_load_micros),
361            execution_micros: values(&self.execution_micros),
362            total_duration_micros: values(&self.total_duration_micros),
363            batch_sizes: self.batch_sizes.lock().expect("metrics poisoned").clone(),
364        }
365    }
366}
367
368#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
369pub struct ModelResource {
370    pub id: String,
371    pub version: Option<String>,
372    pub target: String,
373    pub state: ModelLifecycleState,
374    pub memory_bytes: u64,
375    pub last_used: u64,
376}
377
378#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
379pub struct RuntimeSnapshot {
380    pub active_inferences: usize,
381    pub active_model_loads: usize,
382    pub queued_inferences: usize,
383    pub loaded_models: usize,
384    pub cache_entries: usize,
385    pub cache_memory_bytes: u64,
386    pub metrics: RuntimeMetricsSnapshot,
387}
388
389#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
390pub enum ModelLifecycleState {
391    Discovered,
392    Validated,
393    Loaded,
394    Ready,
395    InUse,
396    Evicted,
397}
398
399#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
400pub struct RuntimeConfig {
401    pub selected_backend: Option<String>,
402    pub selected_provider: Option<String>,
403    pub prefer_acceleration: bool,
404    pub allow_remote_fallback: bool,
405    pub timeout_ms: Option<u64>,
406    pub max_concurrency: usize,
407    pub max_batch_size: Option<usize>,
408    pub max_concurrent_inferences: usize,
409    pub max_concurrent_model_loads: usize,
410    pub max_models: usize,
411    pub max_memory_bytes: Option<u64>,
412    pub max_stream_buffer: usize,
413}
414
415impl Default for RuntimeConfig {
416    fn default() -> Self {
417        Self {
418            selected_backend: None,
419            selected_provider: Some("local".to_owned()),
420            prefer_acceleration: false,
421            allow_remote_fallback: false,
422            timeout_ms: None,
423            max_concurrency: 4,
424            max_batch_size: Some(16),
425            max_concurrent_inferences: 4,
426            max_concurrent_model_loads: 1,
427            max_models: 8,
428            max_memory_bytes: None,
429            max_stream_buffer: 32,
430        }
431    }
432}
433
434#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
435pub struct RuntimeSelection {
436    pub model: String,
437    pub provider: String,
438    pub backend: Option<String>,
439    pub hardware: Option<String>,
440    pub fallback: bool,
441    pub fallback_from: Option<String>,
442    pub reason: String,
443    pub policy: ExecutionPolicy,
444}
445
446#[derive(Clone, Debug)]
447pub struct ModelRegistry {
448    models: Vec<ModelCapability>,
449}
450
451impl ModelRegistry {
452    pub fn list(&self) -> &[ModelCapability] {
453        &self.models
454    }
455}
456
457#[derive(Clone, Debug)]
458pub struct ProviderRegistry {
459    providers: Vec<ProviderCapability>,
460}
461
462impl ProviderRegistry {
463    pub fn list(&self) -> &[ProviderCapability] {
464        &self.providers
465    }
466}
467
468#[derive(Clone, Debug)]
469pub struct BackendRegistry {
470    backends: Vec<BackendCapability>,
471}
472
473impl BackendRegistry {
474    pub fn list(&self) -> &[BackendCapability] {
475        &self.backends
476    }
477}
478
479#[derive(Clone)]
480struct LoadedModel {
481    spec: ModelSpec,
482    handle: ModelHandle,
483    backend_name: String,
484    provider_name: String,
485    last_used: u64,
486    memory_bytes: u64,
487}
488
489#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
490/// Opaque runtime-owned reference to a loaded model resource.
491pub struct RuntimeModelHandle {
492    model_id: String,
493    version: Option<String>,
494    provider: String,
495    backend: String,
496    state: ModelLifecycleState,
497}
498
499impl RuntimeModelHandle {
500    pub fn id(&self) -> &str {
501        &self.model_id
502    }
503    pub fn version(&self) -> Option<&str> {
504        self.version.as_deref()
505    }
506    pub fn provider(&self) -> &str {
507        &self.provider
508    }
509    pub fn backend(&self) -> &str {
510        &self.backend
511    }
512    pub fn state(&self) -> &ModelLifecycleState {
513        &self.state
514    }
515}
516
517#[derive(Clone, Debug, Eq)]
518struct ModelCacheKey {
519    id: String,
520    version: Option<String>,
521    provider: String,
522    backend: String,
523}
524
525impl PartialEq for ModelCacheKey {
526    fn eq(&self, other: &Self) -> bool {
527        self.id == other.id
528            && self.version == other.version
529            && self.provider == other.provider
530            && self.backend == other.backend
531    }
532}
533
534impl Hash for ModelCacheKey {
535    fn hash<H: Hasher>(&self, state: &mut H) {
536        self.id.hash(state);
537        self.version.hash(state);
538        self.provider.hash(state);
539        self.backend.hash(state);
540    }
541}
542
543/// Explicit construction and extension surface for an application-owned [`Runtime`].
544pub struct RuntimeBuilder {
545    config: RuntimeConfig,
546    backends: BTreeMap<String, Arc<dyn Backend>>,
547    providers: BTreeMap<String, Arc<dyn Provider>>,
548    decision_providers: BTreeMap<String, Arc<dyn DecisionModelProvider>>,
549    catalog: Option<Arc<dyn ModelCatalog>>,
550    capability_registry: CapabilityRegistry,
551    model_root: Option<std::path::PathBuf>,
552}
553
554impl Default for RuntimeBuilder {
555    fn default() -> Self {
556        let mut builder = Self {
557            config: RuntimeConfig::default(),
558            backends: BTreeMap::new(),
559            providers: BTreeMap::new(),
560            decision_providers: BTreeMap::new(),
561            catalog: None,
562            capability_registry: CapabilityRegistry::with_builtins(),
563            model_root: default_model_root().ok(),
564        };
565        builder
566            .backends
567            .insert("cpu".to_owned(), Arc::new(CpuBackend));
568        builder
569            .backends
570            .insert("onnx".to_owned(), Arc::new(OnnxBackend));
571        builder
572            .decision_providers
573            .insert("onnx".to_owned(), Arc::new(OnnxBackend));
574        #[cfg(feature = "coreml")]
575        builder.decision_providers.insert(
576            "coreml".to_owned(),
577            Arc::new(ml_runtime_coreml_backend::CoreMlBackend),
578        );
579        builder
580    }
581}
582
583impl RuntimeBuilder {
584    pub fn backend(mut self, name: impl Into<String>) -> Self {
585        self.config.selected_backend = Some(name.into());
586        self
587    }
588
589    pub fn provider(mut self, name: impl Into<String>) -> Self {
590        self.config.selected_provider = Some(name.into());
591        self
592    }
593
594    /// Configures the built-in HTTP provider as the selected remote target.
595    pub fn remote(mut self, endpoint: impl Into<String>) -> Self {
596        let provider = HttpProvider::new(endpoint);
597        self.providers
598            .insert(provider.name().to_owned(), Arc::new(provider));
599        self
600    }
601
602    pub fn catalog<C>(mut self, catalog: C) -> Self
603    where
604        C: ModelCatalog + 'static,
605    {
606        self.catalog = Some(Arc::new(catalog));
607        self
608    }
609
610    pub fn model_root(mut self, root: impl Into<std::path::PathBuf>) -> Self {
611        self.model_root = Some(root.into());
612        self
613    }
614
615    pub fn prefer_acceleration(mut self, enabled: bool) -> Self {
616        self.config.prefer_acceleration = enabled;
617        self
618    }
619
620    pub fn allow_remote_fallback(mut self, enabled: bool) -> Self {
621        self.config.allow_remote_fallback = enabled;
622        self
623    }
624
625    pub fn timeout_ms(mut self, timeout_ms: u64) -> Self {
626        self.config.timeout_ms = Some(timeout_ms);
627        self
628    }
629
630    pub fn concurrency(mut self, max_concurrency: usize) -> Self {
631        self.config.max_concurrency = max_concurrency.max(1);
632        self.config.max_concurrent_inferences = max_concurrency.max(1);
633        self
634    }
635
636    pub fn max_concurrent_inferences(mut self, limit: usize) -> Self {
637        self.config.max_concurrent_inferences = limit.max(1);
638        self.config.max_concurrency = limit.max(1);
639        self
640    }
641
642    pub fn max_concurrent_model_loads(mut self, limit: usize) -> Self {
643        self.config.max_concurrent_model_loads = limit.max(1);
644        self
645    }
646
647    pub fn max_models(mut self, limit: usize) -> Self {
648        self.config.max_models = limit.max(1);
649        self
650    }
651
652    pub fn max_memory_bytes(mut self, limit: u64) -> Self {
653        self.config.max_memory_bytes = Some(limit);
654        self
655    }
656
657    pub fn max_batch_size(mut self, max_batch_size: usize) -> Self {
658        self.config.max_batch_size = Some(max_batch_size.max(1));
659        self
660    }
661
662    /// Registers an experimental/custom backend extension by its declared name.
663    pub fn register_backend<B>(mut self, backend: B) -> Self
664    where
665        B: Backend + 'static,
666    {
667        let name = backend.name().to_owned();
668        self.backends.insert(name, Arc::new(backend));
669        self
670    }
671
672    pub fn register_backend_arc(mut self, backend: Arc<dyn Backend>) -> Self {
673        self.backends.insert(backend.name().to_owned(), backend);
674        self
675    }
676
677    /// Registers an experimental/custom provider extension by its declared name.
678    pub fn register_provider<P>(mut self, provider: P) -> Self
679    where
680        P: Provider + 'static,
681    {
682        let name = provider.name().to_owned();
683        self.providers.insert(name, Arc::new(provider));
684        self
685    }
686
687    pub fn register_provider_arc(mut self, provider: Arc<dyn Provider>) -> Self {
688        self.providers.insert(provider.name().to_owned(), provider);
689        self
690    }
691
692    pub fn register_decision_provider<P>(mut self, provider: P) -> Self
693    where
694        P: DecisionModelProvider + 'static,
695    {
696        self.decision_providers
697            .insert(provider.name().to_owned(), Arc::new(provider));
698        self
699    }
700
701    pub fn register_capability_provider<P>(mut self, provider: P) -> Self
702    where
703        P: CapabilityProvider + 'static,
704    {
705        self.capability_registry.register(provider);
706        self
707    }
708
709    pub fn build(self) -> Runtime {
710        Runtime {
711            config: self.config.clone(),
712            backends: self.backends,
713            providers: self.providers,
714            decision_providers: self.decision_providers,
715            catalog: self.catalog,
716            capability_registry: self.capability_registry,
717            model_root: self.model_root,
718            loaded_models: RwLock::new(HashMap::new()),
719            inference_semaphore: Arc::new(Semaphore::new(
720                self.config.max_concurrent_inferences.max(1),
721            )),
722            load_semaphore: Arc::new(Semaphore::new(
723                self.config.max_concurrent_model_loads.max(1),
724            )),
725            cache_clock: AtomicU64::new(0),
726            metrics: Arc::new(RuntimeMetrics::default()),
727            execution_clock: AtomicU64::new(0),
728            execution_records: Mutex::new(VecDeque::new()),
729        }
730    }
731}
732
733/// Coordinates catalog resolution, routing, model lifecycle, and inference.
734///
735/// A runtime is explicit application state; it does not create global state or
736/// background services. Use [`Runtime::builder`] to obtain built-in CPU and
737/// ONNX execution.
738pub struct Runtime {
739    config: RuntimeConfig,
740    backends: BTreeMap<String, Arc<dyn Backend>>,
741    providers: BTreeMap<String, Arc<dyn Provider>>,
742    decision_providers: BTreeMap<String, Arc<dyn DecisionModelProvider>>,
743    catalog: Option<Arc<dyn ModelCatalog>>,
744    capability_registry: CapabilityRegistry,
745    model_root: Option<std::path::PathBuf>,
746    loaded_models: RwLock<HashMap<ModelCacheKey, LoadedModel>>,
747    inference_semaphore: Arc<Semaphore>,
748    load_semaphore: Arc<Semaphore>,
749    cache_clock: AtomicU64,
750    metrics: Arc<RuntimeMetrics>,
751    execution_clock: AtomicU64,
752    execution_records: Mutex<VecDeque<ExecutionRecord>>,
753}
754
755impl Default for Runtime {
756    fn default() -> Self {
757        Self::new()
758    }
759}
760
761impl Runtime {
762    pub fn new() -> Self {
763        Self::builder().build()
764    }
765
766    pub fn builder() -> RuntimeBuilder {
767        RuntimeBuilder::default()
768    }
769
770    pub fn config(&self) -> &RuntimeConfig {
771        &self.config
772    }
773
774    /// Loads an immutable local typed-decision artifact through the single matching backend.
775    /// There is no fallback: missing, ambiguous, or invalid artifacts fail closed.
776    pub fn load_decision_model(
777        &self,
778        artifact: impl AsRef<Path>,
779    ) -> RuntimeResult<Box<dyn DecisionModel>> {
780        let artifact = artifact.as_ref();
781        if !artifact.exists() {
782            return Err(RuntimeError::ModelNotFound {
783                model: artifact.display().to_string(),
784            });
785        }
786        if !artifact.is_dir() {
787            return Err(RuntimeError::ModelUnavailable {
788                model: artifact.display().to_string(),
789                provider: "local".to_owned(),
790                reason: "typed-decision artifact must be a directory".to_owned(),
791            });
792        }
793        let candidates: Vec<_> = self
794            .decision_providers
795            .values()
796            .filter(|provider| provider.supports_artifact(artifact))
797            .collect();
798        match candidates.as_slice() {
799            [] => Err(RuntimeError::UnsupportedModel {
800                model: artifact.display().to_string(),
801                backend: "none".to_owned(),
802                reason: "no registered typed-decision provider accepts this artifact".to_owned(),
803            }),
804            [provider] => provider.load_decision_model(artifact),
805            _ => Err(RuntimeError::CapabilityMismatch {
806                reason: format!(
807                    "multiple typed-decision providers accept {}",
808                    artifact.display()
809                ),
810            }),
811        }
812    }
813
814    /// Returns model/backend execution facts without requiring applications to
815    /// inspect artifact layouts or backend names.
816    pub fn decision_capabilities(&self, model: &dyn DecisionModel) -> DecisionModelCapabilities {
817        model.capabilities()
818    }
819
820    /// Builds the deterministic execution plan for a loaded model and graph.
821    pub fn explain_decision(
822        &self,
823        model: &dyn DecisionModel,
824        request: &DecisionGraphRequest,
825    ) -> RuntimeResult<DecisionExecutionPlan> {
826        DecisionPlanner.plan(request, &model.capabilities())
827    }
828
829    /// Executes a graph asynchronously. Synchronous native model calls are
830    /// moved off the caller's async executor thread.
831    pub async fn execute_decision_graph(
832        &self,
833        model: Arc<dyn DecisionModel>,
834        request: DecisionGraphRequest,
835        cancellation: Option<CancellationToken>,
836    ) -> RuntimeResult<PlannedDecisionResult> {
837        decision::execute_graph(model, request, cancellation).await
838    }
839
840    /// Compatibility-friendly async execution for an ordinary independent
841    /// `DecisionRequest`.
842    pub async fn decide_async(
843        &self,
844        model: Arc<dyn DecisionModel>,
845        request: DecisionRequest,
846        policy: DecisionExecutionPolicy,
847        cancellation: Option<CancellationToken>,
848    ) -> RuntimeResult<PlannedDecisionResult> {
849        self.execute_decision_graph(
850            model,
851            DecisionGraphRequest::independent(request, policy),
852            cancellation,
853        )
854        .await
855    }
856
857    fn registered_model_root(&self) -> RuntimeResult<&Path> {
858        self.model_root
859            .as_deref()
860            .ok_or_else(|| RuntimeError::ModelUnavailable {
861                model: "registered models".to_owned(),
862                provider: "local".to_owned(),
863                reason: "model directory unavailable; set ML_RUNTIME_MODEL_DIR".to_owned(),
864            })
865    }
866
867    pub fn installed_models(&self) -> RuntimeResult<Vec<InstalledModelRecord>> {
868        let root = self.registered_model_root()?;
869        let installer = RegisteredModelInstaller::new(root)
870            .map_err(|error| RuntimeError::execution("model list", error.to_string()))?;
871        Ok(BuiltinModelRegistry
872            .list()
873            .into_iter()
874            .map(|model| {
875                let package = installer.installed_path(&model);
876                let (status, compiled) = if !package.exists() {
877                    (InstalledModelStatus::NotInstalled, false)
878                } else if model.backend == "coreml" && !cfg!(target_os = "macos") {
879                    (InstalledModelStatus::Unsupported, false)
880                } else {
881                    match InstallationManifest::read(&package) {
882                        Ok(manifest)
883                            if manifest.model == model.name
884                                && manifest.model_revision == model.revision
885                                && manifest.runtime_requirement == model.runtime_requirement
886                                && ml_runtime_model::runtime_requirement_satisfied(
887                                    &manifest.runtime_requirement,
888                                    VERSION,
889                                )
890                                && manifest.os == std::env::consts::OS
891                                && manifest.architecture == std::env::consts::ARCH
892                                && manifest.compiled_status == CompiledStatus::Ready
893                                && manifest
894                                    .compiled_artifact
895                                    .as_ref()
896                                    .is_some_and(|artifact| artifact.path.exists()) =>
897                        {
898                            (InstalledModelStatus::Ready, true)
899                        }
900                        Ok(manifest) if manifest.compiled_status != CompiledStatus::Ready => {
901                            (InstalledModelStatus::InstalledNotCompiled, false)
902                        }
903                        Ok(_) => (InstalledModelStatus::Incompatible, false),
904                        Err(_) => (InstalledModelStatus::Corrupt, false),
905                    }
906                };
907                InstalledModelRecord {
908                    model: model.name,
909                    revision: model.revision,
910                    backend: model.backend,
911                    platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
912                    status,
913                    compiled,
914                    package_path: package,
915                }
916            })
917            .collect())
918    }
919
920    pub async fn install_registered_model(
921        &self,
922        name: &str,
923        source_override: Option<&str>,
924        replace: bool,
925    ) -> RuntimeResult<ModelLifecycleResult> {
926        let started = Instant::now();
927        let model =
928            BuiltinModelRegistry
929                .resolve(name)
930                .map_err(|error| RuntimeError::ModelNotFound {
931                    model: error.to_string(),
932                })?;
933        if model.backend == "coreml" && !cfg!(target_os = "macos") {
934            return Err(RuntimeError::backend_unavailable(
935                "coreml",
936                "Laya installation requires macOS; Core ML is unavailable on this platform",
937            ));
938        }
939        let root = self.registered_model_root()?;
940        let installer = RegisteredModelInstaller::new(root)
941            .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
942        let installed = installer
943            .install(
944                &model,
945                source_override,
946                if replace {
947                    ml_runtime_model::InstallMode::Replace
948                } else {
949                    ml_runtime_model::InstallMode::KeepExisting
950                },
951                None,
952            )
953            .await
954            .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
955
956        let (prepared_path, model_identity, prepared_identity) = match model.backend.as_str() {
957            "onnx" => {
958                let identity = OnnxBackend::validate_laya(&installed.package_path)?;
959                (
960                    installed.package_path.join(&model.artifact_path),
961                    identity.clone(),
962                    identity,
963                )
964            }
965            "coreml" => {
966                #[cfg(feature = "coreml")]
967                {
968                    let prepared = if installed.already_installed && !replace {
969                        ml_runtime_coreml_backend::CoreMlBackend::validate_prepared_laya(
970                            &installed.package_path,
971                        )
972                        .or_else(|_| {
973                            ml_runtime_coreml_backend::CoreMlBackend::prepare_laya(
974                                &installed.package_path,
975                            )
976                        })?
977                    } else {
978                        ml_runtime_coreml_backend::CoreMlBackend::prepare_laya(
979                            &installed.package_path,
980                        )?
981                    };
982                    (
983                        prepared.path,
984                        prepared.model_identity,
985                        prepared.compiled_identity,
986                    )
987                }
988                #[cfg(not(feature = "coreml"))]
989                {
990                    return Err(RuntimeError::backend_unavailable(
991                        "coreml",
992                        "this runtime was built without Core ML support",
993                    ));
994                }
995            }
996            backend => {
997                return Err(RuntimeError::UnsupportedModel {
998                    model: model.name.clone(),
999                    backend: backend.to_owned(),
1000                    reason: "registered backend has no preparation lifecycle".to_owned(),
1001                })
1002            }
1003        };
1004
1005        {
1006            let manifest = InstallationManifest {
1007                schema_version: 1,
1008                model: model.name.clone(),
1009                model_revision: model.revision.clone(),
1010                model_checksum: model_identity,
1011                runtime_version: VERSION.to_owned(),
1012                runtime_requirement: model.runtime_requirement.clone(),
1013                backend: model.backend.clone(),
1014                os: std::env::consts::OS.to_owned(),
1015                architecture: std::env::consts::ARCH.to_owned(),
1016                installed_at_unix_seconds: std::time::SystemTime::now()
1017                    .duration_since(std::time::UNIX_EPOCH)
1018                    .unwrap_or_default()
1019                    .as_secs(),
1020                compiled_status: CompiledStatus::Ready,
1021                compiled_artifact: Some(CompiledArtifactManifest {
1022                    path: prepared_path,
1023                    identity: prepared_identity,
1024                }),
1025            };
1026            manifest
1027                .write_atomic(&installed.package_path)
1028                .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
1029            Ok(ModelLifecycleResult {
1030                model: InstalledModelRecord {
1031                    model: model.name,
1032                    revision: model.revision,
1033                    backend: model.backend,
1034                    platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1035                    status: InstalledModelStatus::Ready,
1036                    compiled: true,
1037                    package_path: installed.package_path,
1038                },
1039                already_installed: installed.already_installed,
1040                duration_ms: started.elapsed().as_millis(),
1041            })
1042        }
1043    }
1044
1045    pub fn doctor_registered_model(&self, name: &str) -> RuntimeResult<ModelDoctorReport> {
1046        let model =
1047            BuiltinModelRegistry
1048                .resolve(name)
1049                .map_err(|error| RuntimeError::ModelNotFound {
1050                    model: error.to_string(),
1051                })?;
1052        let root = self.registered_model_root()?;
1053        let installer = RegisteredModelInstaller::new(root)
1054            .map_err(|error| RuntimeError::execution("model doctor", error.to_string()))?;
1055        let package = installer.installed_path(&model);
1056        let mut diagnostics = Vec::new();
1057        if !package.is_dir() {
1058            diagnostics.push(ModelDiagnostic {
1059                check: "installation".to_owned(),
1060                ok: false,
1061                message: format!(
1062                    "model is not installed; run `ml-runtime model install {}`",
1063                    model.name
1064                ),
1065            });
1066            return Ok(ModelDoctorReport {
1067                model: InstalledModelRecord {
1068                    model: model.name,
1069                    revision: model.revision,
1070                    backend: model.backend,
1071                    platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1072                    status: InstalledModelStatus::NotInstalled,
1073                    compiled: false,
1074                    package_path: package,
1075                },
1076                diagnostics,
1077                manifest: None,
1078            });
1079        }
1080        let artifact_valid = match installer.validate(&model, &package) {
1081            Ok(()) => {
1082                diagnostics.push(ModelDiagnostic {
1083                    check: "artifact".to_owned(),
1084                    ok: true,
1085                    message: format!(
1086                        "registered files, {} artifact, tokenizer, and SHA-256 checksums are valid",
1087                        model.backend
1088                    ),
1089                });
1090                true
1091            }
1092            Err(error) => {
1093                diagnostics.push(ModelDiagnostic {
1094                    check: "artifact".to_owned(),
1095                    ok: false,
1096                    message: format!(
1097                        "{error}; reinstall with `ml-runtime model install {} --replace`",
1098                        model.name
1099                    ),
1100                });
1101                false
1102            }
1103        };
1104        let manifest = InstallationManifest::read(&package).ok();
1105        let compatible = manifest.as_ref().is_some_and(|manifest| {
1106            manifest.schema_version == 1
1107                && manifest.model == model.name
1108                && manifest.model_revision == model.revision
1109                && manifest.backend == model.backend
1110                && ml_runtime_model::runtime_requirement_satisfied(
1111                    &manifest.runtime_requirement,
1112                    VERSION,
1113                )
1114                && manifest.os == std::env::consts::OS
1115                && manifest.architecture == std::env::consts::ARCH
1116        });
1117        diagnostics.push(ModelDiagnostic {
1118            check: "metadata".to_owned(),
1119            ok: compatible,
1120            message: if compatible {
1121                "installation metadata and runtime/backend compatibility are valid".to_owned()
1122            } else {
1123                format!(
1124                    "installation metadata is missing or incompatible; run `ml-runtime model install {} --replace`",
1125                    model.name
1126                )
1127            },
1128        });
1129        let supported = model.supports_platform(std::env::consts::OS, std::env::consts::ARCH);
1130        diagnostics.push(ModelDiagnostic {
1131            check: "platform".to_owned(),
1132            ok: supported,
1133            message: if supported {
1134                format!("{} is supported on this platform", model.backend)
1135            } else {
1136                "Core ML is only supported on macOS".to_owned()
1137            },
1138        });
1139        let compiled_valid = supported
1140            && compatible
1141            && match model.backend.as_str() {
1142                "onnx" => OnnxBackend::validate_laya(&package).is_ok_and(|identity| {
1143                    manifest.as_ref().is_some_and(|manifest| {
1144                        manifest.model_checksum == identity
1145                            && manifest.compiled_artifact.as_ref().is_some_and(|artifact| {
1146                                artifact.path == package.join(&model.artifact_path)
1147                                    && artifact.identity == identity
1148                            })
1149                    })
1150                }),
1151                "coreml" => {
1152                    #[cfg(feature = "coreml")]
1153                    {
1154                        ml_runtime_coreml_backend::CoreMlBackend::validate_prepared_laya(&package)
1155                            .is_ok()
1156                    }
1157                    #[cfg(not(feature = "coreml"))]
1158                    {
1159                        false
1160                    }
1161                }
1162                _ => false,
1163            };
1164        diagnostics.push(ModelDiagnostic {
1165            check: "compiled_artifact".to_owned(),
1166            ok: compiled_valid,
1167            message: if compiled_valid {
1168                format!("prepared {} artifact identity and schema are valid", model.backend)
1169            } else {
1170                format!(
1171                    "compiled artifact is missing or invalid; run `ml-runtime model install {} --replace`",
1172                    model.name
1173                )
1174            },
1175        });
1176        let status = if !supported {
1177            InstalledModelStatus::Unsupported
1178        } else if !artifact_valid {
1179            InstalledModelStatus::Corrupt
1180        } else if !compatible {
1181            InstalledModelStatus::Incompatible
1182        } else if !compiled_valid {
1183            InstalledModelStatus::InstalledNotCompiled
1184        } else {
1185            InstalledModelStatus::Ready
1186        };
1187        Ok(ModelDoctorReport {
1188            model: InstalledModelRecord {
1189                model: model.name,
1190                revision: model.revision,
1191                backend: model.backend,
1192                platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1193                compiled: compiled_valid,
1194                package_path: package,
1195                status,
1196            },
1197            diagnostics,
1198            manifest,
1199        })
1200    }
1201
1202    pub fn remove_registered_model(&self, name: &str) -> RuntimeResult<InstalledModelRecord> {
1203        let model =
1204            BuiltinModelRegistry
1205                .resolve(name)
1206                .map_err(|error| RuntimeError::ModelNotFound {
1207                    model: error.to_string(),
1208                })?;
1209        let root = self.registered_model_root()?;
1210        let installer = RegisteredModelInstaller::new(root)
1211            .map_err(|error| RuntimeError::execution("model remove", error.to_string()))?;
1212        let package = installer.installed_path(&model);
1213        if package.exists() {
1214            #[cfg(feature = "coreml")]
1215            if model.backend == "coreml" {
1216                ml_runtime_coreml_backend::CoreMlBackend::remove_prepared_laya(&package)?;
1217            }
1218            std::fs::remove_dir_all(&package)
1219                .map_err(|error| RuntimeError::execution("model remove", error.to_string()))?;
1220        }
1221        Ok(InstalledModelRecord {
1222            model: model.name,
1223            revision: model.revision,
1224            backend: model.backend,
1225            platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1226            status: InstalledModelStatus::NotInstalled,
1227            compiled: false,
1228            package_path: package,
1229        })
1230    }
1231
1232    /// Loads a validated installed model by its registry name. Source, artifact
1233    /// layout, tokenizer, and backend selection remain runtime concerns.
1234    pub fn load_model(&self, name: &str) -> RuntimeResult<Box<dyn DecisionModel>> {
1235        let registered =
1236            BuiltinModelRegistry
1237                .resolve(name)
1238                .map_err(|error| RuntimeError::ModelNotFound {
1239                    model: error.to_string(),
1240                })?;
1241        let root = self
1242            .model_root
1243            .as_ref()
1244            .ok_or_else(|| RuntimeError::ModelUnavailable {
1245                model: name.to_owned(),
1246                provider: "local".to_owned(),
1247                reason: "model directory unavailable; set ML_RUNTIME_MODEL_DIR".to_owned(),
1248            })?;
1249        let installer = RegisteredModelInstaller::new(root).map_err(|error| {
1250            RuntimeError::execution("installed model validation", error.to_string())
1251        })?;
1252        let package = installer.installed_path(&registered);
1253        if !package.exists() {
1254            return Err(RuntimeError::ModelNotFound {
1255                model: format!("{name}; install it with `ml-runtime model install {name}`"),
1256            });
1257        }
1258        installer.validate(&registered, &package).map_err(|error| {
1259            RuntimeError::ModelIntegrity {
1260                model: name.to_owned(),
1261                reason: format!(
1262                    "{error}; reinstall with `ml-runtime model install {name} --replace`"
1263                ),
1264            }
1265        })?;
1266        let manifest =
1267            InstallationManifest::read(&package).map_err(|error| RuntimeError::ModelIntegrity {
1268                model: name.to_owned(),
1269                reason: format!("{error}; run `ml-runtime model doctor {name}`"),
1270            })?;
1271        let compatible = manifest.schema_version == 1
1272            && manifest.model == registered.name
1273            && manifest.model_revision == registered.revision
1274            && manifest.backend == registered.backend
1275            && ml_runtime_model::runtime_requirement_satisfied(
1276                &manifest.runtime_requirement,
1277                VERSION,
1278            )
1279            && manifest.os == std::env::consts::OS
1280            && manifest.architecture == std::env::consts::ARCH
1281            && manifest.compiled_status == CompiledStatus::Ready
1282            && manifest.compiled_artifact.as_ref().is_some_and(|artifact| {
1283                artifact.path.exists()
1284                    && (registered.backend != "onnx"
1285                        || (artifact.path == package.join(&registered.artifact_path)
1286                            && artifact.identity == manifest.model_checksum))
1287            });
1288        if !compatible {
1289            return Err(RuntimeError::ModelUnavailable {
1290                model: name.to_owned(),
1291                provider: "local".to_owned(),
1292                reason: format!(
1293                    "installation is incomplete, incompatible, or not compiled; run `ml-runtime model doctor {name}`"
1294                ),
1295            });
1296        }
1297        self.load_decision_model(package)
1298    }
1299
1300    pub fn set_catalog<C>(&mut self, catalog: C)
1301    where
1302        C: ModelCatalog + 'static,
1303    {
1304        self.catalog = Some(Arc::new(catalog));
1305    }
1306
1307    pub async fn catalog_models(&self) -> RuntimeResult<Vec<ModelDescriptor>> {
1308        self.catalog
1309            .as_ref()
1310            .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1311            .list()
1312            .await
1313            .map_err(catalog_error)
1314    }
1315
1316    pub async fn resolve_model(
1317        &self,
1318        reference: &ModelReference,
1319    ) -> RuntimeResult<ModelDescriptor> {
1320        self.catalog
1321            .as_ref()
1322            .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1323            .resolve(reference)
1324            .await
1325            .map_err(catalog_error)
1326    }
1327
1328    pub async fn refresh_catalog(&self) -> RuntimeResult<()> {
1329        self.catalog
1330            .as_ref()
1331            .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1332            .refresh()
1333            .await
1334            .map_err(catalog_error)
1335    }
1336
1337    pub fn models(&self) -> ModelRegistry {
1338        let models = self
1339            .loaded_models
1340            .read()
1341            .expect("loaded model registry poisoned")
1342            .values()
1343            .map(|entry| ModelCapability {
1344                id: entry.spec.id.clone(),
1345                format: entry.spec.format.clone(),
1346                location: entry.spec.location.clone(),
1347                loaded: true,
1348                provider: Some(entry.provider_name.clone()),
1349                backend: Some(entry.backend_name.clone()),
1350            })
1351            .collect();
1352        ModelRegistry { models }
1353    }
1354
1355    pub fn providers(&self) -> ProviderRegistry {
1356        let providers = self
1357            .providers
1358            .iter()
1359            .map(|(name, provider)| {
1360                ProviderCapability::from_parts(name.clone(), provider.capabilities())
1361            })
1362            .collect();
1363        ProviderRegistry { providers }
1364    }
1365
1366    pub fn backends(&self) -> BackendRegistry {
1367        let backends = self
1368            .backends
1369            .iter()
1370            .map(|(name, backend)| {
1371                BackendCapability::from_parts(name.clone(), backend.capabilities())
1372            })
1373            .collect();
1374        BackendRegistry { backends }
1375    }
1376
1377    pub fn capabilities(&self) -> RuntimeCapabilities {
1378        RuntimeCapabilities {
1379            environment: self.environment(),
1380            platforms: vec![Platform::current()],
1381            providers: self.providers().providers,
1382            backends: self.backends().backends,
1383            models: self.models().models,
1384            capabilities: self.capability_registry.list(),
1385        }
1386    }
1387
1388    pub fn capability(&self, id: &str) -> Option<Capability> {
1389        self.capability_registry.get(id)
1390    }
1391
1392    pub fn capability_registry(&self) -> &CapabilityRegistry {
1393        &self.capability_registry
1394    }
1395
1396    /// Explicitly discovers and registers local capability package metadata.
1397    ///
1398    /// Discovery never loads or executes a provider binary.
1399    pub fn load_capability_packages(
1400        &mut self,
1401        root: impl AsRef<std::path::Path>,
1402    ) -> RuntimeResult<Vec<String>> {
1403        self.capability_registry.load_local_packages(root)
1404    }
1405
1406    pub fn environment(&self) -> RuntimeEnvironment {
1407        RuntimeEnvironment::discover(
1408            self.capability_registry
1409                .providers()
1410                .iter()
1411                .map(|provider| provider.id().to_owned())
1412                .collect(),
1413        )
1414    }
1415
1416    pub fn resolve_capability(
1417        &self,
1418        request: &ExecutionRequest,
1419    ) -> RuntimeResult<CapabilityResolution> {
1420        self.resolve_capability_with_constraints(request, &ResolutionConstraints::default())
1421    }
1422
1423    pub fn resolve_capability_with_constraints(
1424        &self,
1425        request: &ExecutionRequest,
1426        constraints: &ResolutionConstraints,
1427    ) -> RuntimeResult<CapabilityResolution> {
1428        self.capability_registry
1429            .resolve(&request.capability, constraints)
1430    }
1431
1432    pub async fn execute_capability(
1433        &self,
1434        request: ExecutionRequest,
1435        authorizer: Option<&dyn CapabilityAuthorizer>,
1436    ) -> RuntimeResult<ExecutionResult> {
1437        let resolution = self
1438            .capability_registry
1439            .resolve(&request.capability, &ResolutionConstraints::default())?;
1440        let provider = self
1441            .capability_registry
1442            .provider(&resolution.provider)
1443            .ok_or_else(|| {
1444                RuntimeError::provider_unavailable(&resolution.provider, "not registered")
1445            })?;
1446        if resolution.provider == "builtin" {
1447            capability::execute(request, authorizer).await
1448        } else {
1449            if let Some(authorizer) = authorizer {
1450                if request.policy.require_authorization {
1451                    let capability = self
1452                        .capability_registry
1453                        .get(&request.capability)
1454                        .ok_or_else(|| RuntimeError::CapabilityMismatch {
1455                            reason: format!("unknown capability {}", request.capability),
1456                        })?;
1457                    authorizer.authorize(&request, &capability).await?;
1458                }
1459            } else if self
1460                .capability_registry
1461                .get(&request.capability)
1462                .is_some_and(|capability| !capability.requirements.is_empty())
1463            {
1464                return Err(RuntimeError::Execution {
1465                    operation: request.capability,
1466                    reason: "authorization required".to_owned(),
1467                });
1468            }
1469            provider.execute(request).await
1470        }
1471    }
1472
1473    pub fn status(&self) -> RuntimeStatus {
1474        RuntimeStatus {
1475            loaded_models: self
1476                .loaded_models
1477                .read()
1478                .expect("loaded model registry poisoned")
1479                .len(),
1480            max_models: self.config.max_models,
1481            max_concurrent_inferences: self.config.max_concurrent_inferences,
1482            max_concurrent_model_loads: self.config.max_concurrent_model_loads,
1483        }
1484    }
1485
1486    pub fn metrics(&self) -> RuntimeMetricsSnapshot {
1487        self.metrics.snapshot()
1488    }
1489
1490    pub fn executions(&self) -> Vec<ExecutionRecord> {
1491        self.execution_records
1492            .lock()
1493            .expect("execution records poisoned")
1494            .iter()
1495            .cloned()
1496            .collect()
1497    }
1498
1499    pub fn resources(&self) -> Vec<ModelResource> {
1500        self.loaded_models
1501            .read()
1502            .expect("loaded model registry poisoned")
1503            .values()
1504            .map(|model| ModelResource {
1505                id: model.spec.id.clone(),
1506                version: model.spec.version.clone(),
1507                target: format!("{}/{}", model.provider_name, model.backend_name),
1508                state: ModelLifecycleState::Ready,
1509                memory_bytes: model.memory_bytes,
1510                last_used: model.last_used,
1511            })
1512            .collect()
1513    }
1514
1515    pub fn snapshot(&self) -> RuntimeSnapshot {
1516        let resources = self.resources();
1517        let metrics = self.metrics();
1518        RuntimeSnapshot {
1519            active_inferences: metrics.active_inferences,
1520            active_model_loads: metrics.active_model_loads,
1521            queued_inferences: (metrics.queued_requests as usize)
1522                .saturating_sub(metrics.active_inferences),
1523            loaded_models: resources.len(),
1524            cache_entries: resources.len(),
1525            cache_memory_bytes: resources.iter().map(|resource| resource.memory_bytes).sum(),
1526            metrics,
1527        }
1528    }
1529
1530    pub fn selection_for(&self, model: &ModelSpec) -> RuntimeResult<RuntimeSelection> {
1531        let backend_name = self.select_backend(model)?;
1532        let backend = self
1533            .backends
1534            .get(&backend_name)
1535            .ok_or_else(|| RuntimeError::backend_unavailable(&backend_name, "not registered"))?;
1536        Ok(RuntimeSelection {
1537            model: model.id.clone(),
1538            provider: "local".to_owned(),
1539            backend: Some(backend_name),
1540            hardware: backend.capabilities().hardware,
1541            fallback: false,
1542            fallback_from: None,
1543            reason: "preferred local execution".to_owned(),
1544            policy: ExecutionPolicy::LocalOnly,
1545        })
1546    }
1547
1548    /// Explicitly loads a model for repeated execution with [`Self::infer_loaded`].
1549    /// Normal one-off requests should use [`Self::infer`], which loads lazily.
1550    pub async fn load(&self, model: ModelSpec) -> RuntimeResult<RuntimeModelHandle> {
1551        if self
1552            .config
1553            .selected_provider
1554            .as_deref()
1555            .is_some_and(|provider| provider != "local")
1556        {
1557            return Err(RuntimeError::provider_unavailable(
1558                self.config
1559                    .selected_provider
1560                    .clone()
1561                    .unwrap_or_else(|| "remote".to_owned()),
1562                "load is only supported for local execution paths",
1563            ));
1564        }
1565
1566        let backend_name = self.select_backend(&model)?;
1567        let key = ModelCacheKey {
1568            id: model.id.clone(),
1569            version: model.version.clone(),
1570            provider: "local".to_owned(),
1571            backend: backend_name.clone(),
1572        };
1573        if let Some(existing) = self
1574            .loaded_models
1575            .read()
1576            .expect("loaded model registry poisoned")
1577            .get(&key)
1578        {
1579            self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1580            return Ok(runtime_handle(existing));
1581        }
1582        self.metrics.cache_misses.fetch_add(1, Ordering::Relaxed);
1583        self.metrics
1584            .active_model_loads
1585            .fetch_add(1, Ordering::Relaxed);
1586        let _permit = self
1587            .load_semaphore
1588            .acquire()
1589            .await
1590            .map_err(|_| RuntimeError::execution("load", "runtime load semaphore closed"))?;
1591        if let Some(existing) = self
1592            .loaded_models
1593            .read()
1594            .expect("loaded model registry poisoned")
1595            .get(&key)
1596        {
1597            self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1598            return Ok(runtime_handle(existing));
1599        }
1600        let backend = self
1601            .backends
1602            .get(&backend_name)
1603            .ok_or_else(|| RuntimeError::backend_unavailable(&backend_name, "not registered"))?
1604            .clone();
1605        let timeout_ms = self.config.timeout_ms;
1606        let loaded = run_with_timeout(timeout_ms, "load", backend.load(&model)).await;
1607        self.metrics
1608            .active_model_loads
1609            .fetch_sub(1, Ordering::Relaxed);
1610        let handle = match loaded {
1611            Ok(handle) => handle,
1612            Err(error) => {
1613                self.metrics
1614                    .model_load_failures
1615                    .fetch_add(1, Ordering::Relaxed);
1616                return Err(error);
1617            }
1618        };
1619
1620        let mut cache = self
1621            .loaded_models
1622            .write()
1623            .expect("loaded model registry poisoned");
1624        if cache.len() >= self.config.max_models {
1625            if let Some(oldest) = cache
1626                .iter()
1627                .min_by_key(|(_, model)| model.last_used)
1628                .map(|(key, _)| key.clone())
1629            {
1630                cache.remove(&oldest);
1631                self.metrics.evictions.fetch_add(1, Ordering::Relaxed);
1632            }
1633        }
1634        self.metrics.model_loads.fetch_add(1, Ordering::Relaxed);
1635        let loaded = LoadedModel {
1636            memory_bytes: model_memory_bytes(&model),
1637            spec: model,
1638            handle,
1639            backend_name,
1640            provider_name: "local".to_owned(),
1641            last_used: self.cache_clock.fetch_add(1, Ordering::Relaxed),
1642        };
1643        let runtime_handle = runtime_handle(&loaded);
1644        cache.insert(key, loaded);
1645        Ok(runtime_handle)
1646    }
1647
1648    pub async fn load_reference(
1649        &self,
1650        reference: ModelReference,
1651    ) -> RuntimeResult<RuntimeModelHandle> {
1652        match reference {
1653            ModelReference::Spec(spec) => self.load(spec).await,
1654            ModelReference::Id { id, version } => self
1655                .find_loaded(&id, version.as_ref())
1656                .map(|loaded| runtime_handle(&loaded)),
1657        }
1658    }
1659
1660    /// Resolves, loads, routes, and executes one request.
1661    pub async fn infer(&self, request: InferenceRequest) -> RuntimeResult<InferenceResult> {
1662        self.infer_with_cancellation(request, CancellationToken::new())
1663            .await
1664    }
1665
1666    /// Executes against a previously loaded handle without exposing cache internals.
1667    pub async fn infer_loaded(
1668        &self,
1669        model: &RuntimeModelHandle,
1670        input: Input,
1671        options: InferenceOptions,
1672    ) -> RuntimeResult<InferenceResult> {
1673        self.infer(InferenceRequest {
1674            model: ModelReference::id(model.id().to_owned(), model.version().map(str::to_owned)),
1675            input,
1676            options,
1677        })
1678        .await
1679    }
1680
1681    pub async fn infer_loaded_stream(
1682        &self,
1683        model: &RuntimeModelHandle,
1684        input: Input,
1685        mut options: InferenceOptions,
1686    ) -> RuntimeResult<InferenceStream> {
1687        options.stream = true;
1688        self.infer_stream(InferenceRequest {
1689            model: ModelReference::id(model.id().to_owned(), model.version().map(str::to_owned)),
1690            input,
1691            options,
1692        })
1693        .await
1694    }
1695
1696    /// Executes requests in stable input order.
1697    ///
1698    /// The call fails as a whole on a request error. Backends without native
1699    /// batching are invoked individually behind the same API.
1700    pub async fn infer_batch(
1701        &self,
1702        mut requests: Vec<InferenceRequest>,
1703    ) -> RuntimeResult<Vec<InferenceResult>> {
1704        if requests.is_empty() {
1705            return Ok(Vec::new());
1706        }
1707        if let Some(limit) = self.config.max_batch_size {
1708            if requests.len() > limit {
1709                return Err(RuntimeError::ResourceLimit {
1710                    reason: format!(
1711                        "batch size {} exceeds configured limit {limit}",
1712                        requests.len()
1713                    ),
1714                });
1715            }
1716        }
1717        for request in &requests {
1718            self.validate_request(request)?;
1719        }
1720        for request in &mut requests {
1721            self.resolve_catalog_reference_for_local(request).await?;
1722        }
1723        let selection = self.select_for_request(&requests[0])?;
1724        let same_model = requests
1725            .iter()
1726            .all(|request| model_id(&request.model) == model_id(&requests[0].model));
1727        if selection.provider != "local" || !same_model {
1728            return futures_util::future::try_join_all(
1729                requests.into_iter().map(|request| self.infer(request)),
1730            )
1731            .await;
1732        }
1733        let loaded = self.resolve_loaded_model(&requests[0].model).await?;
1734        let backend = self
1735            .backends
1736            .get(&loaded.backend_name)
1737            .ok_or_else(|| {
1738                RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
1739            })?
1740            .clone();
1741        if !backend.capabilities().batching {
1742            return futures_util::future::try_join_all(
1743                requests.into_iter().map(|request| self.infer(request)),
1744            )
1745            .await;
1746        }
1747        let batch_size = requests.len();
1748        self.metrics
1749            .total_requests
1750            .fetch_add(batch_size as u64, Ordering::Relaxed);
1751        self.metrics
1752            .batch_executions
1753            .fetch_add(1, Ordering::Relaxed);
1754        self.metrics
1755            .total_requests_batched
1756            .fetch_add(batch_size as u64, Ordering::Relaxed);
1757        self.metrics
1758            .batch_sizes
1759            .lock()
1760            .expect("metrics poisoned")
1761            .push(batch_size);
1762        let _permit = self.inference_semaphore.acquire().await.map_err(|_| {
1763            RuntimeError::execution("infer_batch", "runtime inference semaphore closed")
1764        })?;
1765        let mut results = run_with_timeout(
1766            self.config.timeout_ms,
1767            "infer_batch",
1768            backend.infer_batch(&loaded.handle, &requests, Some(CancellationToken::new())),
1769        )
1770        .await?;
1771        if results.len() != batch_size {
1772            return Err(RuntimeError::execution(
1773                "infer_batch",
1774                format!(
1775                    "backend returned {} results for {batch_size} requests",
1776                    results.len()
1777                ),
1778            ));
1779        }
1780        for (result, request) in results.iter_mut().zip(requests.iter()) {
1781            let output = result.output.clone();
1782            decorate_metadata(
1783                &mut result.metadata,
1784                &output,
1785                &loaded.spec.id,
1786                "local",
1787                &loaded.backend_name,
1788                backend.capabilities().hardware.clone(),
1789                Duration::ZERO,
1790                estimate_input_tokens(&request.input),
1791                loaded.spec.version.clone(),
1792                selection.policy.clone(),
1793                false,
1794                None,
1795                Some(selection.reason.clone()),
1796            );
1797            result.metadata.batch_size = batch_size;
1798            result.metadata.cache_hit = true;
1799        }
1800        self.metrics
1801            .successful_requests
1802            .fetch_add(results.len() as u64, Ordering::Relaxed);
1803        Ok(results)
1804    }
1805
1806    /// Executes one request while propagating cancellation through scheduling,
1807    /// provider transport, and backend execution.
1808    pub async fn infer_with_cancellation(
1809        &self,
1810        request: InferenceRequest,
1811        cancellation: CancellationToken,
1812    ) -> RuntimeResult<InferenceResult> {
1813        let started = Instant::now();
1814        let request_id = format!(
1815            "request-{}",
1816            self.execution_clock.fetch_add(1, Ordering::Relaxed) + 1
1817        );
1818        self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
1819        self.metrics
1820            .active_inferences
1821            .fetch_add(1, Ordering::Relaxed);
1822        let model = model_details(&request.model);
1823        let result = self.infer_inner(request.clone(), cancellation).await;
1824        self.metrics
1825            .active_inferences
1826            .fetch_sub(1, Ordering::Relaxed);
1827        match result {
1828            Ok(mut result) => {
1829                result.metadata.request_id = Some(request_id.clone());
1830                self.record_execution(
1831                    &request_id,
1832                    started.elapsed(),
1833                    &result.metadata,
1834                    ExecutionCompletion::Succeeded,
1835                );
1836                self.metrics
1837                    .successful_requests
1838                    .fetch_add(1, Ordering::Relaxed);
1839                Ok(result)
1840            }
1841            Err(error) => {
1842                let completion = if matches!(error, RuntimeError::Cancelled) {
1843                    self.metrics
1844                        .cancelled_requests
1845                        .fetch_add(1, Ordering::Relaxed);
1846                    ExecutionCompletion::Cancelled
1847                } else {
1848                    self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
1849                    ExecutionCompletion::Failed
1850                };
1851                let metadata = ExecutionMetadata {
1852                    request_id: Some(request_id.clone()),
1853                    model: model.0,
1854                    model_version: model.1,
1855                    routing_policy: request.options.execution.clone(),
1856                    streaming_requested: request.options.stream,
1857                    ..ExecutionMetadata::default()
1858                };
1859                self.record_execution(&request_id, started.elapsed(), &metadata, completion);
1860                Err(error)
1861            }
1862        }
1863    }
1864
1865    async fn infer_inner(
1866        &self,
1867        mut request: InferenceRequest,
1868        cancellation: CancellationToken,
1869    ) -> RuntimeResult<InferenceResult> {
1870        self.validate_request(&request)?;
1871        if cancellation.is_cancelled() {
1872            return Err(RuntimeError::Cancelled);
1873        }
1874
1875        let _permit = tokio::select! {
1876            permit = self.inference_semaphore.acquire() => permit
1877                .map_err(|_| RuntimeError::execution("infer", "runtime inference semaphore closed"))?,
1878            _ = cancellation.cancelled() => return Err(RuntimeError::Cancelled),
1879        };
1880
1881        self.resolve_catalog_reference_for_local(&mut request)
1882            .await?;
1883
1884        let mut selection = self.select_for_request(&request)?;
1885        let timeout_ms = request.options.timeout_ms.or(self.config.timeout_ms);
1886        let input_tokens = estimate_input_tokens(&request.input);
1887
1888        match selection.provider.as_str() {
1889            "local" => {
1890                let cache_hit = self.is_loaded(&request.model);
1891                let model_load_started = Instant::now();
1892                if cache_hit {
1893                    self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1894                } else {
1895                    self.metrics.cache_misses.fetch_add(1, Ordering::Relaxed);
1896                }
1897
1898                let loaded = match self.resolve_loaded_model(&request.model).await {
1899                    Ok(loaded) => loaded,
1900                    Err(error) if self.can_fallback(&request, &selection, &error) => {
1901                        selection = self.remote_selection(
1902                            model_id(&request.model),
1903                            true,
1904                            Some("local model unavailable".to_owned()),
1905                            &request.options,
1906                        )?;
1907                        return self
1908                            .infer_remote(
1909                                request,
1910                                cancellation,
1911                                timeout_ms,
1912                                input_tokens,
1913                                selection,
1914                            )
1915                            .await;
1916                    }
1917                    Err(error) => return Err(error),
1918                };
1919                let backend = self
1920                    .backends
1921                    .get(&loaded.backend_name)
1922                    .ok_or_else(|| {
1923                        RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
1924                    })?
1925                    .clone();
1926                let started = Instant::now();
1927                let result = run_with_timeout(
1928                    timeout_ms,
1929                    "infer",
1930                    backend.infer(&loaded.handle, &request, Some(cancellation.clone())),
1931                )
1932                .await;
1933                let mut result = match result {
1934                    Ok(result) => result,
1935                    Err(error) if self.can_fallback(&request, &selection, &error) => {
1936                        selection = self.remote_selection(
1937                            model_id(&request.model),
1938                            true,
1939                            Some(error.to_string()),
1940                            &request.options,
1941                        )?;
1942                        return self
1943                            .infer_remote(
1944                                request,
1945                                cancellation,
1946                                timeout_ms,
1947                                input_tokens,
1948                                selection,
1949                            )
1950                            .await;
1951                    }
1952                    Err(error) => return Err(error),
1953                };
1954                result.metadata.model_load_ms = Some(if cache_hit {
1955                    0.0
1956                } else {
1957                    model_load_started.elapsed().as_secs_f64() * 1000.0
1958                });
1959                let capabilities = backend.capabilities();
1960                let output = result.output.clone();
1961                decorate_metadata(
1962                    &mut result.metadata,
1963                    &output,
1964                    &loaded.spec.id,
1965                    "local",
1966                    &loaded.backend_name,
1967                    capabilities.hardware,
1968                    started.elapsed(),
1969                    input_tokens,
1970                    loaded.spec.version.clone(),
1971                    selection.policy.clone(),
1972                    selection.fallback,
1973                    selection.fallback_from.clone(),
1974                    Some(selection.reason.clone()),
1975                );
1976                result.metadata.cache_hit = cache_hit;
1977                Ok(result)
1978            }
1979            _ => {
1980                self.infer_remote(request, cancellation, timeout_ms, input_tokens, selection)
1981                    .await
1982            }
1983        }
1984    }
1985
1986    fn record_execution(
1987        &self,
1988        request_id: &str,
1989        total_duration: Duration,
1990        metadata: &ExecutionMetadata,
1991        completion: ExecutionCompletion,
1992    ) {
1993        let execution_id = format!(
1994            "execution-{}",
1995            self.execution_clock.fetch_add(1, Ordering::Relaxed) + 1
1996        );
1997        let execution_duration =
1998            Duration::from_secs_f64(metadata.execution_ms.unwrap_or(0.0) / 1000.0);
1999        let queued_duration =
2000            Duration::from_secs_f64(metadata.queue_wait_ms.unwrap_or(0.0) / 1000.0);
2001        let model_load_duration =
2002            Duration::from_secs_f64(metadata.model_load_ms.unwrap_or(0.0) / 1000.0);
2003        self.metrics
2004            .queue_wait_micros
2005            .lock()
2006            .expect("metrics poisoned")
2007            .push(queued_duration.as_micros() as u64);
2008        self.metrics
2009            .model_load_micros
2010            .lock()
2011            .expect("metrics poisoned")
2012            .push(model_load_duration.as_micros() as u64);
2013        self.metrics
2014            .execution_micros
2015            .lock()
2016            .expect("metrics poisoned")
2017            .push(execution_duration.as_micros() as u64);
2018        self.metrics
2019            .total_duration_micros
2020            .lock()
2021            .expect("metrics poisoned")
2022            .push(total_duration.as_micros() as u64);
2023        if metadata.provider == "local" {
2024            self.metrics
2025                .local_executions
2026                .fetch_add(1, Ordering::Relaxed);
2027        } else {
2028            self.metrics
2029                .remote_executions
2030                .fetch_add(1, Ordering::Relaxed);
2031        }
2032        if metadata.fallback {
2033            self.metrics
2034                .fallback_executions
2035                .fetch_add(1, Ordering::Relaxed);
2036        }
2037        if metadata.streaming_requested {
2038            self.metrics
2039                .streaming_requests
2040                .fetch_add(1, Ordering::Relaxed);
2041        }
2042        let record = ExecutionRecord {
2043            execution_id,
2044            request_id: request_id.to_owned(),
2045            model_id: metadata.model.clone(),
2046            model_version: metadata.model_version.clone().unwrap_or_default(),
2047            provider: metadata.provider.clone(),
2048            backend: metadata.backend.clone(),
2049            target: metadata.execution_target.clone(),
2050            routing_policy: metadata.routing_policy.clone(),
2051            fallback: metadata.fallback_from.clone(),
2052            cache_hit: metadata.cache_hit,
2053            batch_size: metadata.batch_size.max(1),
2054            queued_duration,
2055            model_load_duration,
2056            execution_duration,
2057            total_duration,
2058            streaming: metadata.streaming_requested,
2059            output_event_count: metadata.output_event_count,
2060            completion,
2061        };
2062        let mut records = self
2063            .execution_records
2064            .lock()
2065            .expect("execution records poisoned");
2066        records.push_back(record);
2067        while records.len() > 256 {
2068            records.pop_front();
2069        }
2070    }
2071
2072    async fn infer_remote(
2073        &self,
2074        request: InferenceRequest,
2075        cancellation: CancellationToken,
2076        timeout_ms: Option<u64>,
2077        input_tokens: Option<u64>,
2078        selection: RuntimeSelection,
2079    ) -> RuntimeResult<InferenceResult> {
2080        let provider_name = selection.provider.as_str();
2081        let provider = self
2082            .providers
2083            .get(provider_name)
2084            .ok_or_else(|| RuntimeError::provider_unavailable(provider_name, "not registered"))?
2085            .clone();
2086        let started = Instant::now();
2087        let mut result = run_with_timeout(
2088            timeout_ms,
2089            "infer",
2090            provider.infer(request.clone(), Some(cancellation)),
2091        )
2092        .await?;
2093        let backend_name = result.metadata.backend.clone();
2094        let hardware = result.metadata.hardware.clone();
2095        let output = result.output.clone();
2096        let (model, version) = model_details(&request.model);
2097        decorate_metadata(
2098            &mut result.metadata,
2099            &output,
2100            &model,
2101            provider_name,
2102            backend_name.as_str(),
2103            hardware,
2104            started.elapsed(),
2105            input_tokens,
2106            version,
2107            selection.policy,
2108            selection.fallback,
2109            selection.fallback_from,
2110            Some(selection.reason),
2111        );
2112        Ok(result)
2113    }
2114
2115    /// Starts normalized `Started` → `Output*` → `Completed` streaming.
2116    pub async fn infer_stream(&self, request: InferenceRequest) -> RuntimeResult<InferenceStream> {
2117        self.infer_stream_with_cancellation(request, CancellationToken::new())
2118            .await
2119    }
2120
2121    /// Starts a bounded stream with caller-controlled cancellation.
2122    pub async fn infer_stream_with_cancellation(
2123        &self,
2124        mut request: InferenceRequest,
2125        cancellation: CancellationToken,
2126    ) -> RuntimeResult<InferenceStream> {
2127        self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
2128        self.metrics
2129            .streaming_requests
2130            .fetch_add(1, Ordering::Relaxed);
2131        self.validate_request(&request)?;
2132        if !request.options.stream {
2133            return Err(RuntimeError::CapabilityMismatch {
2134                reason: "infer_stream requires InferenceOptions::stream to be true".to_owned(),
2135            });
2136        }
2137        self.resolve_catalog_reference_for_local(&mut request)
2138            .await?;
2139        let selection = self.select_for_request(&request)?;
2140        let buffer = self.config.max_stream_buffer;
2141        if buffer == 0 {
2142            return Err(RuntimeError::ResourceLimit {
2143                reason: "max_stream_buffer must be greater than zero".to_owned(),
2144            });
2145        }
2146        match selection.provider.as_str() {
2147            "local" => {
2148                let loaded = self.resolve_loaded_model(&request.model).await?;
2149                let backend = self
2150                    .backends
2151                    .get(&loaded.backend_name)
2152                    .ok_or_else(|| {
2153                        RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
2154                    })?
2155                    .clone();
2156                let capabilities = backend.capabilities();
2157                let stream = if capabilities.streaming {
2158                    backend
2159                        .infer_stream(&loaded.handle, &request, Some(cancellation.clone()))
2160                        .await?
2161                } else {
2162                    self.metrics
2163                        .compatibility_streams
2164                        .fetch_add(1, Ordering::Relaxed);
2165                    let result = backend
2166                        .infer(&loaded.handle, &request, Some(cancellation.clone()))
2167                        .await?;
2168                    Box::pin(futures_util::stream::iter(vec![Ok(InferenceChunk {
2169                        output: result.output,
2170                        done: true,
2171                        metadata: Some(result.metadata),
2172                    })])) as BoxStream<_>
2173                };
2174                Ok(normalize_stream(
2175                    stream,
2176                    request,
2177                    selection,
2178                    capabilities.streaming,
2179                    true,
2180                    buffer,
2181                    cancellation,
2182                    self.metrics.clone(),
2183                ))
2184            }
2185            provider_name => {
2186                let provider = self
2187                    .providers
2188                    .get(provider_name)
2189                    .ok_or_else(|| {
2190                        RuntimeError::provider_unavailable(provider_name, "not registered")
2191                    })?
2192                    .clone();
2193                let capabilities = provider.capabilities();
2194                let stream = if capabilities.streaming {
2195                    provider
2196                        .infer_stream(request.clone(), Some(cancellation.clone()))
2197                        .await?
2198                } else {
2199                    self.metrics
2200                        .compatibility_streams
2201                        .fetch_add(1, Ordering::Relaxed);
2202                    let result = provider
2203                        .infer(request.clone(), Some(cancellation.clone()))
2204                        .await?;
2205                    Box::pin(futures_util::stream::iter(vec![Ok(InferenceChunk {
2206                        output: result.output,
2207                        done: true,
2208                        metadata: Some(result.metadata),
2209                    })])) as BoxStream<_>
2210                };
2211                Ok(normalize_stream(
2212                    stream,
2213                    request,
2214                    selection,
2215                    capabilities.streaming,
2216                    false,
2217                    buffer,
2218                    cancellation,
2219                    self.metrics.clone(),
2220                ))
2221            }
2222        }
2223    }
2224
2225    fn validate_request(&self, request: &InferenceRequest) -> RuntimeResult<()> {
2226        if request.options.require_local && request.options.require_remote {
2227            return Err(RuntimeError::InvalidRequest {
2228                reason: "request cannot require both local and remote execution".to_owned(),
2229            });
2230        }
2231        if let Some(limit) = self.config.max_batch_size {
2232            let batch_size = request
2233                .options
2234                .batch_size
2235                .unwrap_or_else(|| request.input.batch_size());
2236            if batch_size > limit {
2237                return Err(RuntimeError::ResourceLimit {
2238                    reason: format!("batch size {batch_size} exceeds configured limit {limit}"),
2239                });
2240            }
2241        }
2242        Ok(())
2243    }
2244
2245    async fn resolve_catalog_reference_for_local(
2246        &self,
2247        request: &mut InferenceRequest,
2248    ) -> RuntimeResult<()> {
2249        let local_resolution = request.options.require_local
2250            || (!request.options.require_remote
2251                && matches!(
2252                    request.options.execution,
2253                    ExecutionPolicy::LocalOnly
2254                        | ExecutionPolicy::PreferLocal
2255                        | ExecutionPolicy::LocalThenRemote
2256                ));
2257        if local_resolution && !self.is_loaded(&request.model) {
2258            if let (Some(catalog), ModelReference::Id { .. }) = (&self.catalog, &request.model) {
2259                let descriptor = catalog
2260                    .resolve(&request.model)
2261                    .await
2262                    .map_err(catalog_error)?;
2263                catalog
2264                    .validate(&descriptor.id)
2265                    .await
2266                    .map_err(catalog_error)?;
2267                request.model = ModelReference::Spec(descriptor.spec());
2268            }
2269        }
2270        Ok(())
2271    }
2272
2273    async fn resolve_loaded_model(&self, reference: &ModelReference) -> RuntimeResult<LoadedModel> {
2274        match reference {
2275            ModelReference::Id { id, version } => self.find_loaded(id, version.as_ref()),
2276            ModelReference::Spec(spec) => {
2277                if let Ok(existing) = self.find_loaded(&spec.id, spec.version.as_ref()) {
2278                    return Ok(existing);
2279                }
2280                self.load(spec.clone()).await?;
2281                self.find_loaded(&spec.id, spec.version.as_ref())
2282            }
2283        }
2284    }
2285
2286    fn find_loaded(&self, id: &str, version: Option<&String>) -> RuntimeResult<LoadedModel> {
2287        let mut cache = self
2288            .loaded_models
2289            .write()
2290            .expect("loaded model registry poisoned");
2291        let found = cache.iter_mut().find(|(key, loaded)| {
2292            key.id == id
2293                && version.is_none_or(|version| loaded.spec.version.as_ref() == Some(version))
2294        });
2295        if let Some((_, loaded)) = found {
2296            loaded.last_used = self.cache_clock.fetch_add(1, Ordering::Relaxed);
2297            return Ok(loaded.clone());
2298        }
2299
2300        Err(RuntimeError::ModelNotFound {
2301            model: id.to_owned(),
2302        })
2303    }
2304
2305    fn is_loaded(&self, reference: &ModelReference) -> bool {
2306        let (id, version) = model_details(reference);
2307        self.loaded_models
2308            .read()
2309            .expect("loaded model registry poisoned")
2310            .keys()
2311            .any(|key| {
2312                key.id == id
2313                    && version
2314                        .as_ref()
2315                        .is_none_or(|v| key.version.as_ref() == Some(v))
2316            })
2317    }
2318
2319    fn select_for_request(&self, request: &InferenceRequest) -> RuntimeResult<RuntimeSelection> {
2320        let model_id = model_id(&request.model);
2321        let policy = if request.options.require_remote {
2322            ExecutionPolicy::RemoteOnly
2323        } else if request.options.require_local {
2324            ExecutionPolicy::LocalOnly
2325        } else if self.config.allow_remote_fallback {
2326            ExecutionPolicy::LocalThenRemote
2327        } else {
2328            request.options.execution.clone()
2329        };
2330        match policy {
2331            ExecutionPolicy::RemoteOnly => self.remote_selection(
2332                model_id,
2333                false,
2334                Some("remote-only policy".to_owned()),
2335                &request.options,
2336            ),
2337            ExecutionPolicy::LocalOnly => self.local_selection(&request.model, policy),
2338            ExecutionPolicy::PreferLocal => match self
2339                .local_selection(&request.model, policy.clone())
2340            {
2341                Ok(selection) => Ok(selection),
2342                Err(error) => {
2343                    self.remote_selection(model_id, true, Some(error.to_string()), &request.options)
2344                }
2345            },
2346            ExecutionPolicy::LocalThenRemote => match self
2347                .local_selection(&request.model, policy.clone())
2348            {
2349                Ok(selection) => Ok(selection),
2350                Err(error) => {
2351                    self.remote_selection(model_id, true, Some(error.to_string()), &request.options)
2352                }
2353            },
2354            ExecutionPolicy::PreferRemote | ExecutionPolicy::RemoteThenLocal => match self
2355                .remote_selection(
2356                    model_id.clone(),
2357                    false,
2358                    Some("preferred remote execution".to_owned()),
2359                    &request.options,
2360                ) {
2361                Ok(selection) => Ok(selection),
2362                Err(_) => self.local_selection(&request.model, policy),
2363            },
2364        }
2365    }
2366
2367    fn local_selection(
2368        &self,
2369        reference: &ModelReference,
2370        policy: ExecutionPolicy,
2371    ) -> RuntimeResult<RuntimeSelection> {
2372        match reference {
2373            ModelReference::Id { id, version } => {
2374                let loaded = self.find_loaded(id, version.as_ref())?;
2375                Ok(RuntimeSelection {
2376                    model: loaded.spec.id,
2377                    provider: "local".to_owned(),
2378                    backend: Some(loaded.backend_name.clone()),
2379                    hardware: self
2380                        .backends
2381                        .get(&loaded.backend_name)
2382                        .and_then(|backend| backend.capabilities().hardware),
2383                    fallback: false,
2384                    fallback_from: None,
2385                    reason: "preferred local execution".to_owned(),
2386                    policy,
2387                })
2388            }
2389            ModelReference::Spec(spec) => {
2390                let backend = self.select_backend(spec)?;
2391                let hardware = self
2392                    .backends
2393                    .get(&backend)
2394                    .and_then(|b| b.capabilities().hardware);
2395                Ok(RuntimeSelection {
2396                    model: spec.id.clone(),
2397                    provider: "local".to_owned(),
2398                    backend: Some(backend),
2399                    hardware,
2400                    fallback: false,
2401                    fallback_from: None,
2402                    reason: "preferred local execution".to_owned(),
2403                    policy,
2404                })
2405            }
2406        }
2407    }
2408
2409    fn remote_selection(
2410        &self,
2411        model: String,
2412        fallback: bool,
2413        reason: Option<String>,
2414        options: &InferenceOptions,
2415    ) -> RuntimeResult<RuntimeSelection> {
2416        let selected = if let Some(provider_name) = self.config.selected_provider.as_deref() {
2417            if provider_name != "local" {
2418                Some(provider_name.to_owned())
2419            } else {
2420                None
2421            }
2422        } else {
2423            None
2424        }
2425        .or_else(|| {
2426            self.providers.iter().find_map(|(name, provider)| {
2427                let capabilities = provider.capabilities();
2428                if capabilities.available && capabilities.remote {
2429                    Some(name.clone())
2430                } else {
2431                    None
2432                }
2433            })
2434        })
2435        .ok_or_else(|| {
2436            RuntimeError::provider_unavailable("remote", "no remote provider registered")
2437        })?;
2438
2439        Ok(RuntimeSelection {
2440            model,
2441            provider: selected,
2442            backend: None,
2443            hardware: None,
2444            fallback,
2445            fallback_from: fallback.then(|| "local".to_owned()),
2446            reason: reason.unwrap_or_else(|| "remote execution selected by policy".to_owned()),
2447            policy: if fallback && options.execution == ExecutionPolicy::LocalOnly {
2448                ExecutionPolicy::LocalThenRemote
2449            } else {
2450                options.execution.clone()
2451            },
2452        })
2453    }
2454
2455    fn can_fallback(
2456        &self,
2457        request: &InferenceRequest,
2458        selection: &RuntimeSelection,
2459        error: &RuntimeError,
2460    ) -> bool {
2461        matches!(
2462            selection.policy,
2463            ExecutionPolicy::LocalThenRemote | ExecutionPolicy::PreferLocal
2464        ) && !request.options.require_local
2465            && matches!(
2466                error,
2467                RuntimeError::ProviderUnavailable { .. }
2468                    | RuntimeError::ModelUnavailable { .. }
2469                    | RuntimeError::BackendUnavailable { .. }
2470                    | RuntimeError::ModelNotFound { .. }
2471                    | RuntimeError::CapabilityMismatch { .. }
2472                    | RuntimeError::Timeout { .. }
2473                    | RuntimeError::Transport { .. }
2474                    | RuntimeError::Protocol { .. }
2475            )
2476    }
2477
2478    fn select_backend(&self, model: &ModelSpec) -> RuntimeResult<String> {
2479        if let Some(selected) = self.config.selected_backend.as_deref() {
2480            let backend = self
2481                .backends
2482                .get(selected)
2483                .ok_or_else(|| RuntimeError::backend_unavailable(selected, "not registered"))?;
2484            let capabilities = backend.capabilities();
2485            if !capabilities.available {
2486                return Err(RuntimeError::backend_unavailable(
2487                    selected,
2488                    "backend is not available on this platform",
2489                ));
2490            }
2491            if !backend.supports(model) {
2492                return Err(RuntimeError::UnsupportedModel {
2493                    model: model.id.clone(),
2494                    backend: selected.to_owned(),
2495                    reason: "model format is not supported".to_owned(),
2496                });
2497            }
2498            return Ok(selected.to_owned());
2499        }
2500
2501        let mut candidates: Vec<_> = self
2502            .backends
2503            .iter()
2504            .filter_map(|(name, backend)| {
2505                let capabilities = backend.capabilities();
2506                if capabilities.available && backend.supports(model) {
2507                    Some((
2508                        name.clone(),
2509                        !capabilities.accelerators.is_empty(),
2510                        capabilities.hardware.clone(),
2511                    ))
2512                } else {
2513                    None
2514                }
2515            })
2516            .collect();
2517
2518        candidates.sort_by(|left, right| left.0.cmp(&right.0));
2519        if self.config.prefer_acceleration {
2520            candidates
2521                .sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0)));
2522        }
2523
2524        candidates
2525            .into_iter()
2526            .next()
2527            .map(|candidate| candidate.0)
2528            .ok_or_else(|| {
2529                RuntimeError::backend_unavailable("local", "no compatible backend registered")
2530            })
2531    }
2532}
2533
2534async fn run_with_timeout<F, T>(
2535    timeout_ms: Option<u64>,
2536    operation: &str,
2537    future: F,
2538) -> RuntimeResult<T>
2539where
2540    F: std::future::Future<Output = RuntimeResult<T>>,
2541{
2542    if let Some(timeout_ms) = timeout_ms {
2543        timeout(Duration::from_millis(timeout_ms), future)
2544            .await
2545            .map_err(|_| RuntimeError::Timeout {
2546                operation: operation.to_owned(),
2547            })?
2548    } else {
2549        future.await
2550    }
2551}
2552
2553fn model_id(reference: &ModelReference) -> String {
2554    match reference {
2555        ModelReference::Id { id, .. } => id.clone(),
2556        ModelReference::Spec(spec) => spec.id.clone(),
2557    }
2558}
2559
2560fn runtime_handle(loaded: &LoadedModel) -> RuntimeModelHandle {
2561    RuntimeModelHandle {
2562        model_id: loaded.spec.id.clone(),
2563        version: loaded.spec.version.clone(),
2564        provider: loaded.provider_name.clone(),
2565        backend: loaded.backend_name.clone(),
2566        state: ModelLifecycleState::Ready,
2567    }
2568}
2569
2570fn model_details(reference: &ModelReference) -> (String, Option<String>) {
2571    match reference {
2572        ModelReference::Id { id, version } => (id.clone(), version.clone()),
2573        ModelReference::Spec(spec) => (spec.id.clone(), spec.version.clone()),
2574    }
2575}
2576
2577fn catalog_error(error: CatalogError) -> RuntimeError {
2578    match error {
2579        CatalogError::ModelNotFound(model) => RuntimeError::ModelNotFound { model },
2580        CatalogError::InvalidIdentity(model) => RuntimeError::InvalidRequest {
2581            reason: format!("invalid model identity: {model}"),
2582        },
2583        CatalogError::ArtifactIntegrity { model, reason } => {
2584            RuntimeError::ModelIntegrity { model, reason }
2585        }
2586        other => RuntimeError::Execution {
2587            operation: "catalog".to_owned(),
2588            reason: other.to_string(),
2589        },
2590    }
2591}
2592
2593fn estimate_input_tokens(input: &Input) -> Option<u64> {
2594    match input {
2595        Input::Text(text) => Some(text.split_whitespace().count() as u64),
2596        Input::Tokens(tokens) => Some(tokens.len() as u64),
2597        _ => None,
2598    }
2599}
2600
2601fn estimate_output_tokens(output: &Output) -> Option<u64> {
2602    match output {
2603        Output::Text(text) => Some(text.split_whitespace().count() as u64),
2604        Output::Tokens(tokens) => Some(tokens.len() as u64),
2605        _ => None,
2606    }
2607}
2608
2609fn model_memory_bytes(model: &ModelSpec) -> u64 {
2610    match &model.location {
2611        ModelLocation::Path(path) => std::fs::metadata(path)
2612            .map(|metadata| metadata.len())
2613            .unwrap_or(0),
2614        _ => 0,
2615    }
2616}
2617
2618#[allow(clippy::too_many_arguments)]
2619fn decorate_metadata(
2620    metadata: &mut ExecutionMetadata,
2621    output: &Output,
2622    model: &str,
2623    provider: &str,
2624    backend: &str,
2625    hardware: Option<String>,
2626    elapsed: Duration,
2627    input_tokens: Option<u64>,
2628    model_version: Option<String>,
2629    routing_policy: ExecutionPolicy,
2630    fallback: bool,
2631    fallback_from: Option<String>,
2632    fallback_reason: Option<String>,
2633) {
2634    metadata.model = model.to_owned();
2635    metadata.model_version = model_version;
2636    metadata.provider = provider.to_owned();
2637    metadata.backend = backend.to_owned();
2638    metadata.execution_target = provider.to_owned();
2639    metadata.routing_policy = routing_policy;
2640    metadata.fallback = fallback;
2641    metadata.fallback_from = fallback_from;
2642    metadata.fallback_reason = fallback_reason;
2643    metadata.hardware = hardware;
2644    metadata.latency_ms = Some(elapsed.as_secs_f64() * 1000.0);
2645    metadata.execution_ms = metadata.latency_ms;
2646    metadata.input_tokens = input_tokens;
2647    metadata.output_tokens = estimate_output_tokens(output);
2648}
2649
2650#[allow(clippy::too_many_arguments)]
2651fn normalize_stream(
2652    mut source: BoxStream<RuntimeResult<InferenceChunk>>,
2653    request: InferenceRequest,
2654    selection: RuntimeSelection,
2655    streaming_supported: bool,
2656    cache_hit: bool,
2657    buffer: usize,
2658    cancellation: CancellationToken,
2659    metrics: Arc<RuntimeMetrics>,
2660) -> InferenceStream {
2661    let (sender, receiver) = mpsc::channel(buffer);
2662    tokio::spawn(async move {
2663        let (model, version) = model_details(&request.model);
2664        let mut metadata = ExecutionMetadata {
2665            model,
2666            model_version: version,
2667            provider: selection.provider.clone(),
2668            execution_target: selection.provider.clone(),
2669            routing_policy: selection.policy.clone(),
2670            fallback: selection.fallback,
2671            fallback_from: selection.fallback_from.clone(),
2672            fallback_reason: Some(selection.reason.clone()),
2673            streaming_requested: true,
2674            streaming_supported,
2675            cache_hit,
2676            completion_state: Some("started".to_owned()),
2677            ..ExecutionMetadata::default()
2678        };
2679        if sender
2680            .send(Ok(InferenceStreamEvent::Started(metadata.clone())))
2681            .await
2682            .is_err()
2683        {
2684            return;
2685        }
2686
2687        let mut output_count = 0;
2688        while let Some(item) = tokio::select! {
2689            item = source.next() => item,
2690            _ = cancellation.cancelled() => {
2691                metrics.cancelled_streams.fetch_add(1, Ordering::Relaxed);
2692                let _ = sender.send(Err(RuntimeError::Cancelled)).await;
2693                return;
2694            }
2695        } {
2696            match item {
2697                Ok(chunk) => {
2698                    if let Some(chunk_metadata) = chunk.metadata {
2699                        metadata = chunk_metadata;
2700                    }
2701                    metadata.streaming_requested = true;
2702                    metadata.streaming_supported = streaming_supported;
2703                    metadata.cache_hit = cache_hit;
2704                    metadata.output_event_count = output_count + 1;
2705                    output_count += 1;
2706                    if sender
2707                        .send(Ok(InferenceStreamEvent::Output(chunk.output)))
2708                        .await
2709                        .is_err()
2710                    {
2711                        return;
2712                    }
2713                    metrics
2714                        .streamed_output_events
2715                        .fetch_add(1, Ordering::Relaxed);
2716                    if chunk.done {
2717                        metadata.output_event_count = output_count;
2718                        metadata.completion_state = Some("completed".to_owned());
2719                        let _ = sender
2720                            .send(Ok(InferenceStreamEvent::Completed(metadata)))
2721                            .await;
2722                        return;
2723                    }
2724                }
2725                Err(error) => {
2726                    let _ = sender.send(Err(error)).await;
2727                    return;
2728                }
2729            }
2730        }
2731        metadata.output_event_count = output_count;
2732        metadata.completion_state = Some("completed".to_owned());
2733        let _ = sender
2734            .send(Ok(InferenceStreamEvent::Completed(metadata)))
2735            .await;
2736    });
2737    Box::pin(ReceiverStream::new(receiver))
2738}
2739
2740#[cfg(test)]
2741mod tests {
2742    use super::*;
2743    use async_trait::async_trait;
2744    use futures_util::StreamExt;
2745    use ml_runtime_backend::BackendCapabilities;
2746    use ml_runtime_common::BoxStream;
2747    use ml_runtime_cpu_backend::CpuBackend;
2748    use ml_runtime_inference::InferenceOptions;
2749    use ml_runtime_provider::{Provider, ProviderCapabilities};
2750    use serde_json::json;
2751    use tokio_stream::iter;
2752
2753    #[derive(Clone)]
2754    struct BlockingBackend {
2755        started: Arc<Semaphore>,
2756    }
2757
2758    #[async_trait]
2759    impl Backend for BlockingBackend {
2760        fn name(&self) -> &str {
2761            "blocking"
2762        }
2763
2764        fn capabilities(&self) -> BackendCapabilities {
2765            BackendCapabilities {
2766                available: true,
2767                local: true,
2768                streaming: true,
2769                cancellation: true,
2770                supported_formats: vec![ModelFormat::Unknown],
2771                ..BackendCapabilities::default()
2772            }
2773        }
2774
2775        fn supports(&self, model: &ModelSpec) -> bool {
2776            model.format == ModelFormat::Unknown
2777        }
2778
2779        async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle> {
2780            Ok(ModelHandle::new(
2781                model.id.clone(),
2782                self.name(),
2783                model.format.clone(),
2784                Arc::new(()),
2785            ))
2786        }
2787
2788        async fn infer(
2789            &self,
2790            _model: &ModelHandle,
2791            _request: &InferenceRequest,
2792            cancellation: Option<CancellationToken>,
2793        ) -> RuntimeResult<InferenceResult> {
2794            self.started.add_permits(1);
2795            cancellation
2796                .expect("runtime supplies cancellation")
2797                .cancelled()
2798                .await;
2799            Err(RuntimeError::Cancelled)
2800        }
2801
2802        async fn infer_stream(
2803            &self,
2804            _model: &ModelHandle,
2805            _request: &InferenceRequest,
2806            _cancellation: Option<CancellationToken>,
2807        ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
2808            Ok(Box::pin(futures_util::stream::pending()))
2809        }
2810    }
2811
2812    #[test]
2813    fn runtime_version_is_workspace_version() {
2814        assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
2815    }
2816
2817    #[derive(Clone)]
2818    struct MockRemoteProvider;
2819
2820    #[async_trait]
2821    impl Provider for MockRemoteProvider {
2822        fn name(&self) -> &str {
2823            "mock-remote"
2824        }
2825
2826        fn capabilities(&self) -> ProviderCapabilities {
2827            ProviderCapabilities {
2828                available: true,
2829                local: false,
2830                remote: true,
2831                streaming: true,
2832                cancellation: true,
2833                structured_output: true,
2834                batching: true,
2835                endpoint: Some("mock://remote".to_owned()),
2836                notes: vec!["test remote provider".to_owned()],
2837            }
2838        }
2839
2840        async fn infer(
2841            &self,
2842            request: InferenceRequest,
2843            cancellation: Option<CancellationToken>,
2844        ) -> RuntimeResult<InferenceResult> {
2845            if model_id(&request.model) == "blocking-remote" {
2846                cancellation
2847                    .expect("runtime supplies cancellation")
2848                    .cancelled()
2849                    .await;
2850                return Err(RuntimeError::Cancelled);
2851            }
2852            if cancellation
2853                .as_ref()
2854                .is_some_and(CancellationToken::is_cancelled)
2855            {
2856                return Err(RuntimeError::Cancelled);
2857            }
2858            Ok(InferenceResult {
2859                output: Output::Structured(json!({"remote": model_id(&request.model)})),
2860                metadata: ExecutionMetadata {
2861                    backend: "remote-api".to_owned(),
2862                    hardware: Some("network".to_owned()),
2863                    ..ExecutionMetadata::default()
2864                },
2865            })
2866        }
2867
2868        async fn infer_stream(
2869            &self,
2870            request: InferenceRequest,
2871            _cancellation: Option<CancellationToken>,
2872        ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
2873            let stream = iter(vec![Ok(InferenceChunk {
2874                output: Output::Text(model_id(&request.model)),
2875                done: true,
2876                metadata: None,
2877            })]);
2878            Ok(Box::pin(stream))
2879        }
2880    }
2881
2882    #[tokio::test]
2883    async fn loads_and_runs_local_cpu_models() {
2884        let runtime = Runtime::builder()
2885            .register_backend(CpuBackend::default())
2886            .build();
2887        let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
2888        runtime.load(model.clone()).await.unwrap();
2889
2890        let result = runtime
2891            .infer(InferenceRequest {
2892                model: model.into(),
2893                input: Input::Text("hello runtime".to_owned()),
2894                options: InferenceOptions::default(),
2895            })
2896            .await
2897            .unwrap();
2898
2899        assert_eq!(result.output, Output::Text("hello runtime".to_owned()));
2900        assert_eq!(result.metadata.provider, "local");
2901        assert_eq!(result.metadata.backend, "cpu");
2902    }
2903
2904    #[tokio::test]
2905    async fn supports_remote_provider_selection() {
2906        let runtime = Runtime::builder()
2907            .register_backend(CpuBackend::default())
2908            .register_provider(MockRemoteProvider)
2909            .provider("mock-remote")
2910            .build();
2911
2912        let result = runtime
2913            .infer(InferenceRequest {
2914                model: ModelReference::id("remote-model", None),
2915                input: Input::Text("hello".to_owned()),
2916                options: InferenceOptions {
2917                    require_remote: true,
2918                    ..InferenceOptions::default()
2919                },
2920            })
2921            .await
2922            .unwrap();
2923
2924        assert_eq!(result.metadata.provider, "mock-remote");
2925        assert_eq!(result.metadata.backend, "remote-api");
2926    }
2927
2928    #[tokio::test]
2929    async fn only_falls_back_to_remote_when_enabled() {
2930        let request = InferenceRequest {
2931            model: ModelReference::id("missing", None),
2932            input: Input::Text("hello".to_owned()),
2933            options: InferenceOptions::default(),
2934        };
2935
2936        let local_only = Runtime::builder()
2937            .register_backend(CpuBackend::default())
2938            .register_provider(MockRemoteProvider)
2939            .build();
2940        assert!(matches!(
2941            local_only.infer(request.clone()).await,
2942            Err(RuntimeError::ModelNotFound { .. })
2943        ));
2944
2945        let hybrid = Runtime::builder()
2946            .register_backend(CpuBackend::default())
2947            .register_provider(MockRemoteProvider)
2948            .allow_remote_fallback(true)
2949            .build();
2950        let result = hybrid.infer(request).await.unwrap();
2951        assert_eq!(result.metadata.provider, "mock-remote");
2952        assert!(result.metadata.fallback);
2953        assert_eq!(result.metadata.fallback_from.as_deref(), Some("local"));
2954    }
2955
2956    #[tokio::test]
2957    async fn explicit_policies_choose_deterministic_targets() {
2958        let local = Runtime::builder()
2959            .register_backend(CpuBackend::default())
2960            .register_provider(MockRemoteProvider)
2961            .build();
2962        let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
2963        local.load(model.clone()).await.unwrap();
2964
2965        let local_result = local
2966            .infer(InferenceRequest {
2967                model: model.clone().into(),
2968                input: Input::Text("hello".to_owned()),
2969                options: InferenceOptions {
2970                    execution: ExecutionPolicy::PreferLocal,
2971                    ..InferenceOptions::default()
2972                },
2973            })
2974            .await
2975            .unwrap();
2976        assert_eq!(local_result.metadata.provider, "local");
2977
2978        let remote_result = local
2979            .infer(InferenceRequest {
2980                model: ModelReference::id("remote-model", None),
2981                input: Input::Text("hello".to_owned()),
2982                options: InferenceOptions {
2983                    execution: ExecutionPolicy::PreferRemote,
2984                    ..InferenceOptions::default()
2985                },
2986            })
2987            .await
2988            .unwrap();
2989        assert_eq!(remote_result.metadata.provider, "mock-remote");
2990        assert_eq!(
2991            remote_result.metadata.routing_policy,
2992            ExecutionPolicy::PreferRemote
2993        );
2994    }
2995
2996    #[tokio::test]
2997    async fn rejects_requests_above_batch_limit() {
2998        let runtime = Runtime::builder()
2999            .register_backend(CpuBackend::default())
3000            .max_batch_size(2)
3001            .build();
3002
3003        let err = runtime
3004            .infer(InferenceRequest {
3005                model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3006                input: Input::Tokens(vec![1, 2, 3]),
3007                options: InferenceOptions {
3008                    batch_size: Some(3),
3009                    ..InferenceOptions::default()
3010                },
3011            })
3012            .await
3013            .unwrap_err();
3014
3015        assert!(matches!(err, RuntimeError::ResourceLimit { .. }));
3016    }
3017
3018    #[tokio::test]
3019    async fn propagates_cancellation() {
3020        let runtime = Runtime::builder()
3021            .register_backend(CpuBackend::default())
3022            .build();
3023        let token = CancellationToken::new();
3024        token.cancel();
3025
3026        let err = runtime
3027            .infer_with_cancellation(
3028                InferenceRequest {
3029                    model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory)
3030                        .into(),
3031                    input: Input::Text("hello".to_owned()),
3032                    options: InferenceOptions::default(),
3033                },
3034                token,
3035            )
3036            .await
3037            .unwrap_err();
3038
3039        assert!(matches!(err, RuntimeError::Cancelled));
3040    }
3041
3042    #[tokio::test]
3043    async fn cancels_active_and_queued_inference() {
3044        let started = Arc::new(Semaphore::new(0));
3045        let runtime = Arc::new(
3046            Runtime::builder()
3047                .register_backend(BlockingBackend {
3048                    started: Arc::clone(&started),
3049                })
3050                .backend("blocking")
3051                .concurrency(1)
3052                .build(),
3053        );
3054        let make_request = || {
3055            InferenceRequest::new(
3056                ModelSpec::new("blocking", ModelFormat::Unknown, ModelLocation::Memory),
3057                "input",
3058            )
3059        };
3060
3061        let active_token = CancellationToken::new();
3062        let active = {
3063            let runtime = Arc::clone(&runtime);
3064            let token = active_token.clone();
3065            tokio::spawn(
3066                async move { runtime.infer_with_cancellation(make_request(), token).await },
3067            )
3068        };
3069        let permit = started.acquire().await.unwrap();
3070        permit.forget();
3071
3072        let queued_token = CancellationToken::new();
3073        let queued = {
3074            let runtime = Arc::clone(&runtime);
3075            let token = queued_token.clone();
3076            tokio::spawn(
3077                async move { runtime.infer_with_cancellation(make_request(), token).await },
3078            )
3079        };
3080        tokio::task::yield_now().await;
3081        queued_token.cancel();
3082        assert!(matches!(
3083            queued.await.unwrap(),
3084            Err(RuntimeError::Cancelled)
3085        ));
3086
3087        active_token.cancel();
3088        assert!(matches!(
3089            active.await.unwrap(),
3090            Err(RuntimeError::Cancelled)
3091        ));
3092    }
3093
3094    #[tokio::test]
3095    async fn cancels_an_active_stream() {
3096        let runtime = Runtime::builder()
3097            .register_backend(BlockingBackend {
3098                started: Arc::new(Semaphore::new(0)),
3099            })
3100            .backend("blocking")
3101            .build();
3102        let token = CancellationToken::new();
3103        let mut stream = runtime
3104            .infer_stream_with_cancellation(
3105                InferenceRequest::new(
3106                    ModelSpec::new("blocking", ModelFormat::Unknown, ModelLocation::Memory),
3107                    "input",
3108                )
3109                .with_options(InferenceOptions {
3110                    stream: true,
3111                    ..InferenceOptions::default()
3112                }),
3113                token.clone(),
3114            )
3115            .await
3116            .unwrap();
3117        assert!(matches!(
3118            stream.next().await.unwrap().unwrap(),
3119            InferenceStreamEvent::Started(_)
3120        ));
3121        token.cancel();
3122        assert!(matches!(
3123            stream.next().await.unwrap(),
3124            Err(RuntimeError::Cancelled)
3125        ));
3126    }
3127
3128    #[tokio::test]
3129    async fn propagates_remote_cancellation() {
3130        let runtime = Arc::new(
3131            Runtime::builder()
3132                .register_provider(MockRemoteProvider)
3133                .provider("mock-remote")
3134                .build(),
3135        );
3136        let token = CancellationToken::new();
3137        let task = {
3138            let runtime = Arc::clone(&runtime);
3139            let token = token.clone();
3140            tokio::spawn(async move {
3141                runtime
3142                    .infer_with_cancellation(
3143                        InferenceRequest::new(ModelReference::latest("blocking-remote"), "input")
3144                            .with_options(InferenceOptions {
3145                                execution: ExecutionPolicy::RemoteOnly,
3146                                ..InferenceOptions::default()
3147                            }),
3148                        token,
3149                    )
3150                    .await
3151            })
3152        };
3153        tokio::task::yield_now().await;
3154        token.cancel();
3155        assert!(matches!(task.await.unwrap(), Err(RuntimeError::Cancelled)));
3156    }
3157
3158    #[tokio::test]
3159    async fn streams_from_local_backend() {
3160        let runtime = Runtime::builder()
3161            .register_backend(CpuBackend::default())
3162            .build();
3163
3164        let mut stream = runtime
3165            .infer_stream(InferenceRequest {
3166                model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3167                input: Input::Text("hello runtime".to_owned()),
3168                options: InferenceOptions {
3169                    stream: true,
3170                    ..InferenceOptions::default()
3171                },
3172            })
3173            .await
3174            .unwrap();
3175
3176        assert!(matches!(
3177            stream.next().await.unwrap().unwrap(),
3178            InferenceStreamEvent::Started(_)
3179        ));
3180        assert!(matches!(
3181            stream.next().await.unwrap().unwrap(),
3182            InferenceStreamEvent::Output(Output::Text(text)) if text == "hello"
3183        ));
3184        assert!(matches!(
3185            stream.next().await.unwrap().unwrap(),
3186            InferenceStreamEvent::Output(Output::Text(text)) if text == "runtime"
3187        ));
3188        assert!(matches!(
3189            stream.next().await.unwrap().unwrap(),
3190            InferenceStreamEvent::Completed(metadata)
3191                if metadata.output_event_count == 2 && metadata.streaming_supported
3192        ));
3193    }
3194
3195    #[tokio::test]
3196    async fn rejects_stream_calls_without_stream_flag() {
3197        let runtime = Runtime::builder()
3198            .register_backend(CpuBackend::default())
3199            .build();
3200        let result = runtime
3201            .infer_stream(InferenceRequest {
3202                model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3203                input: Input::Text("hello".to_owned()),
3204                options: InferenceOptions::default(),
3205            })
3206            .await;
3207
3208        assert!(matches!(
3209            result,
3210            Err(RuntimeError::CapabilityMismatch { .. })
3211        ));
3212    }
3213
3214    #[tokio::test]
3215    async fn streams_from_remote_provider() {
3216        let runtime = Runtime::builder()
3217            .register_backend(CpuBackend::default())
3218            .register_provider(MockRemoteProvider)
3219            .provider("mock-remote")
3220            .build();
3221
3222        let mut stream = runtime
3223            .infer_stream(InferenceRequest {
3224                model: ModelReference::id("remote-model", None),
3225                input: Input::Text("hello".to_owned()),
3226                options: InferenceOptions {
3227                    stream: true,
3228                    require_remote: true,
3229                    ..InferenceOptions::default()
3230                },
3231            })
3232            .await
3233            .unwrap();
3234
3235        assert!(matches!(
3236            stream.next().await.unwrap().unwrap(),
3237            InferenceStreamEvent::Started(_)
3238        ));
3239        assert!(matches!(
3240            stream.next().await.unwrap().unwrap(),
3241            InferenceStreamEvent::Output(Output::Text(text)) if text == "remote-model"
3242        ));
3243        assert!(matches!(
3244            stream.next().await.unwrap().unwrap(),
3245            InferenceStreamEvent::Completed(metadata)
3246                if metadata.output_event_count == 1
3247        ));
3248    }
3249
3250    #[tokio::test]
3251    async fn explicit_handles_reuse_loaded_models_and_report_cache_hits() {
3252        let runtime = Runtime::builder()
3253            .register_backend(CpuBackend::default())
3254            .build();
3255        let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
3256        let handle = runtime.load(model).await.unwrap();
3257        assert_eq!(handle.id(), "echo");
3258        let result = runtime
3259            .infer_loaded(
3260                &handle,
3261                Input::Text("cached".to_owned()),
3262                Default::default(),
3263            )
3264            .await
3265            .unwrap();
3266        assert!(result.metadata.cache_hit);
3267        assert_eq!(runtime.models().list().len(), 1);
3268    }
3269
3270    #[tokio::test]
3271    async fn loaded_stream_reuses_the_cached_model() {
3272        let runtime = Runtime::builder()
3273            .register_backend(CpuBackend::default())
3274            .build();
3275        let handle = runtime
3276            .load(ModelSpec::new(
3277                "echo",
3278                ModelFormat::Unknown,
3279                ModelLocation::Memory,
3280            ))
3281            .await
3282            .unwrap();
3283        let mut stream = runtime
3284            .infer_loaded_stream(
3285                &handle,
3286                Input::Text("cached stream".to_owned()),
3287                InferenceOptions::default(),
3288            )
3289            .await
3290            .unwrap();
3291        let events = futures_util::StreamExt::collect::<Vec<_>>(&mut stream).await;
3292        assert!(matches!(
3293            events.first().and_then(|event| event.as_ref().ok()),
3294            Some(InferenceStreamEvent::Started(_))
3295        ));
3296        assert!(matches!(
3297            events.last().and_then(|event| event.as_ref().ok()),
3298            Some(InferenceStreamEvent::Completed(metadata)) if metadata.cache_hit
3299        ));
3300        assert_eq!(runtime.models().list().len(), 1);
3301    }
3302
3303    #[tokio::test]
3304    async fn cache_evicts_the_least_recently_used_target() {
3305        let runtime = Runtime::builder()
3306            .register_backend(CpuBackend::default())
3307            .max_models(1)
3308            .build();
3309        runtime
3310            .load(ModelSpec::new(
3311                "first",
3312                ModelFormat::Unknown,
3313                ModelLocation::Memory,
3314            ))
3315            .await
3316            .unwrap();
3317        runtime
3318            .load(ModelSpec::new(
3319                "second",
3320                ModelFormat::Unknown,
3321                ModelLocation::Memory,
3322            ))
3323            .await
3324            .unwrap();
3325        assert_eq!(runtime.models().list().len(), 1);
3326        assert_eq!(runtime.models().list()[0].id, "second");
3327    }
3328
3329    #[tokio::test]
3330    async fn batch_inference_preserves_result_order_and_identity() {
3331        let runtime = Runtime::builder()
3332            .register_backend(CpuBackend::default())
3333            .build();
3334        let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
3335        let results = runtime
3336            .infer_batch(vec![
3337                InferenceRequest {
3338                    model: model.clone().into(),
3339                    input: Input::Text("a".to_owned()),
3340                    options: Default::default(),
3341                },
3342                InferenceRequest {
3343                    model: model.into(),
3344                    input: Input::Text("b".to_owned()),
3345                    options: Default::default(),
3346                },
3347            ])
3348            .await
3349            .unwrap();
3350        assert_eq!(results.len(), 2);
3351        assert_eq!(results[0].output, Output::Text("a".to_owned()));
3352        assert_eq!(results[1].output, Output::Text("b".to_owned()));
3353        assert_eq!(results[0].metadata.batch_size, 2);
3354        assert_eq!(results[1].metadata.batch_size, 2);
3355    }
3356
3357    #[tokio::test]
3358    async fn records_execution_metrics_and_snapshot() {
3359        let runtime = Runtime::builder()
3360            .register_backend(CpuBackend::default())
3361            .build();
3362        let model = ModelSpec::new("observed", ModelFormat::Unknown, ModelLocation::Memory);
3363        let result = runtime
3364            .infer(InferenceRequest {
3365                model: model.into(),
3366                input: Input::Text("hello".to_owned()),
3367                options: Default::default(),
3368            })
3369            .await
3370            .unwrap();
3371
3372        let metrics = runtime.metrics();
3373        assert_eq!(metrics.total_requests, 1);
3374        assert_eq!(metrics.successful_requests, 1);
3375        assert_eq!(runtime.executions().len(), 1);
3376        assert_eq!(
3377            runtime.executions()[0].completion,
3378            ExecutionCompletion::Succeeded
3379        );
3380        assert_eq!(
3381            runtime.executions()[0].request_id,
3382            result.metadata.request_id.clone().unwrap()
3383        );
3384        assert_eq!(runtime.snapshot().loaded_models, 1);
3385    }
3386}