Skip to main content

rust_ml_runtime/
lib.rs

1#![cfg_attr(test, allow(clippy::default_constructed_unit_structs))]
2
3use futures_util::StreamExt;
4use ml_runtime_backend::{Backend, BackendCapability};
5use ml_runtime_common::BoxStream;
6use ml_runtime_cpu_backend::CpuBackend;
7use ml_runtime_http_provider::HttpProvider;
8use ml_runtime_inference::InferenceChunk;
9use ml_runtime_model::{CatalogError, ModelCatalog, ModelHandle};
10use ml_runtime_onnx_backend::OnnxBackend;
11use ml_runtime_provider::{Provider, ProviderCapability};
12use serde::{Deserialize, Serialize};
13use std::{
14    collections::{BTreeMap, HashMap, VecDeque},
15    hash::{Hash, Hasher},
16    path::{Path, PathBuf},
17    sync::{
18        atomic::{AtomicU64, AtomicUsize, Ordering},
19        Arc, Mutex, RwLock,
20    },
21    time::{Duration, Instant},
22};
23use tokio::{
24    sync::{mpsc, Semaphore},
25    time::timeout,
26};
27use tokio_stream::wrappers::ReceiverStream;
28
29mod capability;
30mod decision;
31mod generation;
32
33pub use capability::{
34    AvailabilityState, BuiltinCapabilityProvider, Capability, CapabilityAuthorizer,
35    CapabilityAvailability, CapabilityExecutionMetadata, CapabilityLimits,
36    CapabilityPackageManifest, CapabilityPolicy, CapabilityProvider, CapabilityRegistry,
37    CapabilityResolution, ExecutionRequest, ExecutionResult, LocalProviderPackage,
38    ManifestCapabilityProvider, PackageCapability, PackageManifest, ProviderDescriptor,
39    ProviderManifest, ResolutionCandidate, ResolutionConstraints,
40};
41pub use decision::DecisionPlanner;
42
43pub use ml_runtime_backend;
44pub use ml_runtime_backend::{DecisionModel, DecisionModelProvider};
45pub use ml_runtime_common;
46pub use ml_runtime_common::{CancellationToken, RuntimeError, RuntimeResult};
47pub use ml_runtime_http_provider::openai::{
48    ApiKey, OpenAiCompatibleConfig, OpenAiCompatibleProvider,
49};
50pub use ml_runtime_inference;
51pub use ml_runtime_inference::{
52    DecisionDependency, DecisionDependencyCondition, DecisionExecution,
53    DecisionExecutionDiagnostics, DecisionExecutionPlan, DecisionExecutionPolicy,
54    DecisionExecutionStrategy, DecisionGraphRequest, DecisionModelCapabilities, DecisionNode,
55    DecisionOption, DecisionPlanStage, DecisionProvenance, DecisionQuestion, DecisionRequest,
56    DecisionResult, DecisionType, DecisionValue, DeviceKind, ExecutionMetadata, ExecutionPolicy,
57    InferenceOptions, InferenceRequest, InferenceResult, InferenceStreamEvent, Input,
58    ModelArchitecture, ModelDescription, ModelIdentity, Output, PlannedDecisionResult, Tensor,
59    TypedDecision,
60};
61pub use ml_runtime_model;
62pub use ml_runtime_model::{
63    default_model_root, BuiltinModelRegistry, CompiledArtifactManifest, CompiledStatus,
64    FilesystemModelCatalog, InstallationManifest, ModelDescriptor, ModelFormat, ModelId,
65    ModelLocation, ModelReference, ModelSpec, RegisteredModelInstaller,
66};
67pub use ml_runtime_provider;
68pub use ml_runtime_provider::{
69    ChatMessage, ChatMetadata, ChatRequest, ChatResponse, ChatRole, ChatStreamEvent,
70    ChatStreamSummary, ExecutionLocation, FinishReason, GenerationCapabilities,
71    GenerationModelInfo, GenerationOptions, GenerationProvider, ResponseFormat, StructuredOutput,
72    ToolCall, ToolChoice, ToolDefinition, Usage,
73};
74
75pub const VERSION: &str = env!("CARGO_PKG_VERSION");
76
77#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
78#[serde(rename_all = "snake_case")]
79pub enum InstalledModelStatus {
80    NotInstalled,
81    InstalledNotCompiled,
82    Ready,
83    Corrupt,
84    Incompatible,
85    Unsupported,
86}
87
88impl std::fmt::Display for InstalledModelStatus {
89    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
90        formatter.write_str(match self {
91            Self::NotInstalled => "not_installed",
92            Self::InstalledNotCompiled => "installed_not_compiled",
93            Self::Ready => "ready",
94            Self::Corrupt => "corrupt",
95            Self::Incompatible => "incompatible",
96            Self::Unsupported => "unsupported",
97        })
98    }
99}
100
101#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
102pub struct InstalledModelRecord {
103    pub model: String,
104    pub revision: String,
105    pub backend: String,
106    pub platform: String,
107    pub status: InstalledModelStatus,
108    pub compiled: bool,
109    pub package_path: PathBuf,
110}
111
112#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
113pub struct ModelDiagnostic {
114    pub check: String,
115    pub ok: bool,
116    pub message: String,
117}
118
119#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
120pub struct ModelDoctorReport {
121    pub model: InstalledModelRecord,
122    pub diagnostics: Vec<ModelDiagnostic>,
123    pub manifest: Option<InstallationManifest>,
124}
125
126#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
127pub struct ModelLifecycleResult {
128    pub model: InstalledModelRecord,
129    pub already_installed: bool,
130    pub duration_ms: u128,
131}
132
133/// Bounded asynchronous stream of normalized runtime-owned inference events.
134pub type InferenceStream = BoxStream<RuntimeResult<InferenceStreamEvent>>;
135
136#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
137pub enum Platform {
138    Linux,
139    Macos,
140    Windows,
141    Unknown,
142}
143
144impl Platform {
145    fn current() -> Self {
146        if cfg!(target_os = "linux") {
147            Self::Linux
148        } else if cfg!(target_os = "macos") {
149            Self::Macos
150        } else if cfg!(target_os = "windows") {
151            Self::Windows
152        } else {
153            Self::Unknown
154        }
155    }
156}
157
158#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
159pub struct ModelCapability {
160    pub id: String,
161    pub format: ModelFormat,
162    pub location: ModelLocation,
163    pub loaded: bool,
164    pub provider: Option<String>,
165    pub backend: Option<String>,
166}
167
168#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
169pub struct RuntimeCapabilities {
170    pub environment: RuntimeEnvironment,
171    pub platforms: Vec<Platform>,
172    pub providers: Vec<ProviderCapability>,
173    pub backends: Vec<BackendCapability>,
174    pub models: Vec<ModelCapability>,
175    pub capabilities: Vec<Capability>,
176}
177
178#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
179pub struct RuntimeEnvironment {
180    pub platform: String,
181    pub architecture: String,
182    pub operating_system: String,
183    pub cpu: String,
184    pub memory_bytes: Option<u64>,
185    pub accelerators: Vec<String>,
186    pub filesystem: String,
187    pub network: String,
188    pub developer_tools: Vec<String>,
189    pub installed_providers: Vec<String>,
190}
191
192impl RuntimeEnvironment {
193    pub fn discover(provider_ids: Vec<String>) -> Self {
194        Self {
195            platform: std::env::consts::OS.to_owned(),
196            architecture: std::env::consts::ARCH.to_owned(),
197            operating_system: std::env::consts::OS.to_owned(),
198            cpu: std::thread::available_parallelism()
199                .map(|count| format!("{} logical cores", count.get()))
200                .unwrap_or_else(|_| "unknown".to_owned()),
201            memory_bytes: available_memory_bytes(),
202            accelerators: Vec::new(),
203            filesystem: std::env::current_dir()
204                .map(|path| path.display().to_string())
205                .unwrap_or_else(|_| "unavailable".to_owned()),
206            network: "available by explicit policy".to_owned(),
207            developer_tools: ["git"]
208                .iter()
209                .filter(|tool| command_available(tool))
210                .map(|tool| (*tool).to_owned())
211                .collect(),
212            installed_providers: provider_ids,
213        }
214    }
215}
216
217fn command_available(command: &str) -> bool {
218    std::process::Command::new(command)
219        .arg("--version")
220        .output()
221        .map(|output| output.status.success())
222        .unwrap_or(false)
223}
224
225fn available_memory_bytes() -> Option<u64> {
226    let contents = std::fs::read_to_string("/proc/meminfo").ok()?;
227    contents.lines().find_map(|line| {
228        let value = line
229            .strip_prefix("MemTotal:")?
230            .split_whitespace()
231            .next()?
232            .parse::<u64>()
233            .ok()?;
234        Some(value * 1024)
235    })
236}
237
238#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
239pub struct RuntimeStatus {
240    pub loaded_models: usize,
241    pub max_models: usize,
242    pub max_concurrent_inferences: usize,
243    pub max_concurrent_model_loads: usize,
244}
245
246pub type ExecutionId = String;
247pub type RequestId = String;
248
249#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
250pub enum ExecutionCompletion {
251    Succeeded,
252    Failed,
253    Cancelled,
254}
255
256#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
257pub struct ExecutionRecord {
258    pub execution_id: ExecutionId,
259    pub request_id: RequestId,
260    pub model_id: String,
261    pub model_version: String,
262    pub provider: String,
263    pub backend: String,
264    pub target: String,
265    pub routing_policy: ExecutionPolicy,
266    pub fallback: Option<String>,
267    pub cache_hit: bool,
268    pub batch_size: usize,
269    pub queued_duration: Duration,
270    pub model_load_duration: Duration,
271    pub execution_duration: Duration,
272    pub total_duration: Duration,
273    pub streaming: bool,
274    pub output_event_count: usize,
275    pub completion: ExecutionCompletion,
276}
277
278#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
279pub struct RuntimeMetricsSnapshot {
280    pub total_requests: u64,
281    pub successful_requests: u64,
282    pub failed_requests: u64,
283    pub cancelled_requests: u64,
284    pub local_executions: u64,
285    pub remote_executions: u64,
286    pub fallback_executions: u64,
287    pub fallback_failures: u64,
288    pub model_loads: u64,
289    pub model_load_failures: u64,
290    pub cache_hits: u64,
291    pub cache_misses: u64,
292    pub evictions: u64,
293    pub queued_requests: u64,
294    pub active_inferences: usize,
295    pub active_model_loads: usize,
296    pub batch_executions: u64,
297    pub individual_executions: u64,
298    pub total_requests_batched: u64,
299    pub streaming_requests: u64,
300    pub streamed_output_events: u64,
301    pub cancelled_streams: u64,
302    pub compatibility_streams: u64,
303    pub queue_wait_micros: Vec<u64>,
304    pub model_load_micros: Vec<u64>,
305    pub execution_micros: Vec<u64>,
306    pub total_duration_micros: Vec<u64>,
307    pub batch_sizes: Vec<usize>,
308}
309
310#[derive(Default)]
311pub struct RuntimeMetrics {
312    total_requests: AtomicU64,
313    successful_requests: AtomicU64,
314    failed_requests: AtomicU64,
315    cancelled_requests: AtomicU64,
316    local_executions: AtomicU64,
317    remote_executions: AtomicU64,
318    fallback_executions: AtomicU64,
319    fallback_failures: AtomicU64,
320    model_loads: AtomicU64,
321    model_load_failures: AtomicU64,
322    cache_hits: AtomicU64,
323    cache_misses: AtomicU64,
324    evictions: AtomicU64,
325    queued_requests: AtomicU64,
326    active_inferences: AtomicUsize,
327    active_model_loads: AtomicUsize,
328    batch_executions: AtomicU64,
329    individual_executions: AtomicU64,
330    total_requests_batched: AtomicU64,
331    streaming_requests: AtomicU64,
332    streamed_output_events: AtomicU64,
333    cancelled_streams: AtomicU64,
334    compatibility_streams: AtomicU64,
335    queue_wait_micros: Mutex<Vec<u64>>,
336    model_load_micros: Mutex<Vec<u64>>,
337    execution_micros: Mutex<Vec<u64>>,
338    total_duration_micros: Mutex<Vec<u64>>,
339    batch_sizes: Mutex<Vec<usize>>,
340}
341
342impl RuntimeMetrics {
343    pub fn snapshot(&self) -> RuntimeMetricsSnapshot {
344        let values = |values: &Mutex<Vec<u64>>| values.lock().expect("metrics poisoned").clone();
345        RuntimeMetricsSnapshot {
346            total_requests: self.total_requests.load(Ordering::Relaxed),
347            successful_requests: self.successful_requests.load(Ordering::Relaxed),
348            failed_requests: self.failed_requests.load(Ordering::Relaxed),
349            cancelled_requests: self.cancelled_requests.load(Ordering::Relaxed),
350            local_executions: self.local_executions.load(Ordering::Relaxed),
351            remote_executions: self.remote_executions.load(Ordering::Relaxed),
352            fallback_executions: self.fallback_executions.load(Ordering::Relaxed),
353            fallback_failures: self.fallback_failures.load(Ordering::Relaxed),
354            model_loads: self.model_loads.load(Ordering::Relaxed),
355            model_load_failures: self.model_load_failures.load(Ordering::Relaxed),
356            cache_hits: self.cache_hits.load(Ordering::Relaxed),
357            cache_misses: self.cache_misses.load(Ordering::Relaxed),
358            evictions: self.evictions.load(Ordering::Relaxed),
359            queued_requests: self.queued_requests.load(Ordering::Relaxed),
360            active_inferences: self.active_inferences.load(Ordering::Relaxed),
361            active_model_loads: self.active_model_loads.load(Ordering::Relaxed),
362            batch_executions: self.batch_executions.load(Ordering::Relaxed),
363            individual_executions: self.individual_executions.load(Ordering::Relaxed),
364            total_requests_batched: self.total_requests_batched.load(Ordering::Relaxed),
365            streaming_requests: self.streaming_requests.load(Ordering::Relaxed),
366            streamed_output_events: self.streamed_output_events.load(Ordering::Relaxed),
367            cancelled_streams: self.cancelled_streams.load(Ordering::Relaxed),
368            compatibility_streams: self.compatibility_streams.load(Ordering::Relaxed),
369            queue_wait_micros: values(&self.queue_wait_micros),
370            model_load_micros: values(&self.model_load_micros),
371            execution_micros: values(&self.execution_micros),
372            total_duration_micros: values(&self.total_duration_micros),
373            batch_sizes: self.batch_sizes.lock().expect("metrics poisoned").clone(),
374        }
375    }
376}
377
378#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
379pub struct ModelResource {
380    pub id: String,
381    pub version: Option<String>,
382    pub target: String,
383    pub state: ModelLifecycleState,
384    pub memory_bytes: u64,
385    pub last_used: u64,
386}
387
388#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
389pub struct RuntimeSnapshot {
390    pub active_inferences: usize,
391    pub active_model_loads: usize,
392    pub queued_inferences: usize,
393    pub loaded_models: usize,
394    pub cache_entries: usize,
395    pub cache_memory_bytes: u64,
396    pub metrics: RuntimeMetricsSnapshot,
397}
398
399#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
400pub enum ModelLifecycleState {
401    Discovered,
402    Validated,
403    Loaded,
404    Ready,
405    InUse,
406    Evicted,
407}
408
409#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
410pub struct RuntimeConfig {
411    pub selected_backend: Option<String>,
412    pub selected_provider: Option<String>,
413    pub prefer_acceleration: bool,
414    pub allow_remote_fallback: bool,
415    pub timeout_ms: Option<u64>,
416    pub max_concurrency: usize,
417    pub max_batch_size: Option<usize>,
418    pub max_concurrent_inferences: usize,
419    pub max_concurrent_model_loads: usize,
420    pub max_models: usize,
421    pub max_memory_bytes: Option<u64>,
422    pub max_stream_buffer: usize,
423}
424
425impl Default for RuntimeConfig {
426    fn default() -> Self {
427        Self {
428            selected_backend: None,
429            selected_provider: Some("local".to_owned()),
430            prefer_acceleration: false,
431            allow_remote_fallback: false,
432            timeout_ms: None,
433            max_concurrency: 4,
434            max_batch_size: Some(16),
435            max_concurrent_inferences: 4,
436            max_concurrent_model_loads: 1,
437            max_models: 8,
438            max_memory_bytes: None,
439            max_stream_buffer: 32,
440        }
441    }
442}
443
444#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
445pub struct RuntimeSelection {
446    pub model: String,
447    pub provider: String,
448    pub backend: Option<String>,
449    pub hardware: Option<String>,
450    pub fallback: bool,
451    pub fallback_from: Option<String>,
452    pub reason: String,
453    pub policy: ExecutionPolicy,
454}
455
456#[derive(Clone, Debug)]
457pub struct ModelRegistry {
458    models: Vec<ModelCapability>,
459}
460
461impl ModelRegistry {
462    pub fn list(&self) -> &[ModelCapability] {
463        &self.models
464    }
465}
466
467#[derive(Clone, Debug)]
468pub struct ProviderRegistry {
469    providers: Vec<ProviderCapability>,
470}
471
472impl ProviderRegistry {
473    pub fn list(&self) -> &[ProviderCapability] {
474        &self.providers
475    }
476}
477
478#[derive(Clone, Debug)]
479pub struct BackendRegistry {
480    backends: Vec<BackendCapability>,
481}
482
483impl BackendRegistry {
484    pub fn list(&self) -> &[BackendCapability] {
485        &self.backends
486    }
487}
488
489#[derive(Clone)]
490struct LoadedModel {
491    spec: ModelSpec,
492    handle: ModelHandle,
493    backend_name: String,
494    provider_name: String,
495    last_used: u64,
496    memory_bytes: u64,
497}
498
499#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
500/// Opaque runtime-owned reference to a loaded model resource.
501pub struct RuntimeModelHandle {
502    model_id: String,
503    version: Option<String>,
504    provider: String,
505    backend: String,
506    state: ModelLifecycleState,
507}
508
509impl RuntimeModelHandle {
510    pub fn id(&self) -> &str {
511        &self.model_id
512    }
513    pub fn version(&self) -> Option<&str> {
514        self.version.as_deref()
515    }
516    pub fn provider(&self) -> &str {
517        &self.provider
518    }
519    pub fn backend(&self) -> &str {
520        &self.backend
521    }
522    pub fn state(&self) -> &ModelLifecycleState {
523        &self.state
524    }
525}
526
527#[derive(Clone, Debug, Eq)]
528struct ModelCacheKey {
529    id: String,
530    version: Option<String>,
531    provider: String,
532    backend: String,
533}
534
535impl PartialEq for ModelCacheKey {
536    fn eq(&self, other: &Self) -> bool {
537        self.id == other.id
538            && self.version == other.version
539            && self.provider == other.provider
540            && self.backend == other.backend
541    }
542}
543
544impl Hash for ModelCacheKey {
545    fn hash<H: Hasher>(&self, state: &mut H) {
546        self.id.hash(state);
547        self.version.hash(state);
548        self.provider.hash(state);
549        self.backend.hash(state);
550    }
551}
552
553/// Explicit construction and extension surface for an application-owned [`Runtime`].
554pub struct RuntimeBuilder {
555    config: RuntimeConfig,
556    backends: BTreeMap<String, Arc<dyn Backend>>,
557    providers: BTreeMap<String, Arc<dyn Provider>>,
558    generation_providers: BTreeMap<String, Arc<dyn GenerationProvider>>,
559    decision_providers: BTreeMap<String, Arc<dyn DecisionModelProvider>>,
560    catalog: Option<Arc<dyn ModelCatalog>>,
561    capability_registry: CapabilityRegistry,
562    model_root: Option<std::path::PathBuf>,
563}
564
565impl Default for RuntimeBuilder {
566    fn default() -> Self {
567        let mut builder = Self {
568            config: RuntimeConfig::default(),
569            backends: BTreeMap::new(),
570            providers: BTreeMap::new(),
571            generation_providers: BTreeMap::new(),
572            decision_providers: BTreeMap::new(),
573            catalog: None,
574            capability_registry: CapabilityRegistry::with_builtins(),
575            model_root: default_model_root().ok(),
576        };
577        builder
578            .backends
579            .insert("cpu".to_owned(), Arc::new(CpuBackend));
580        builder
581            .backends
582            .insert("onnx".to_owned(), Arc::new(OnnxBackend));
583        builder
584            .decision_providers
585            .insert("onnx".to_owned(), Arc::new(OnnxBackend));
586        #[cfg(feature = "coreml")]
587        builder.decision_providers.insert(
588            "coreml".to_owned(),
589            Arc::new(ml_runtime_coreml_backend::CoreMlBackend),
590        );
591        builder
592    }
593}
594
595impl RuntimeBuilder {
596    pub fn backend(mut self, name: impl Into<String>) -> Self {
597        self.config.selected_backend = Some(name.into());
598        self
599    }
600
601    pub fn provider(mut self, name: impl Into<String>) -> Self {
602        self.config.selected_provider = Some(name.into());
603        self
604    }
605
606    /// Configures the built-in HTTP provider as the selected remote target.
607    pub fn remote(mut self, endpoint: impl Into<String>) -> Self {
608        let provider = HttpProvider::new(endpoint);
609        self.providers
610            .insert(provider.name().to_owned(), Arc::new(provider));
611        self
612    }
613
614    pub fn catalog<C>(mut self, catalog: C) -> Self
615    where
616        C: ModelCatalog + 'static,
617    {
618        self.catalog = Some(Arc::new(catalog));
619        self
620    }
621
622    pub fn model_root(mut self, root: impl Into<std::path::PathBuf>) -> Self {
623        self.model_root = Some(root.into());
624        self
625    }
626
627    pub fn prefer_acceleration(mut self, enabled: bool) -> Self {
628        self.config.prefer_acceleration = enabled;
629        self
630    }
631
632    pub fn allow_remote_fallback(mut self, enabled: bool) -> Self {
633        self.config.allow_remote_fallback = enabled;
634        self
635    }
636
637    pub fn timeout_ms(mut self, timeout_ms: u64) -> Self {
638        self.config.timeout_ms = Some(timeout_ms);
639        self
640    }
641
642    pub fn concurrency(mut self, max_concurrency: usize) -> Self {
643        self.config.max_concurrency = max_concurrency.max(1);
644        self.config.max_concurrent_inferences = max_concurrency.max(1);
645        self
646    }
647
648    pub fn max_concurrent_inferences(mut self, limit: usize) -> Self {
649        self.config.max_concurrent_inferences = limit.max(1);
650        self.config.max_concurrency = limit.max(1);
651        self
652    }
653
654    pub fn max_concurrent_model_loads(mut self, limit: usize) -> Self {
655        self.config.max_concurrent_model_loads = limit.max(1);
656        self
657    }
658
659    pub fn max_models(mut self, limit: usize) -> Self {
660        self.config.max_models = limit.max(1);
661        self
662    }
663
664    pub fn max_memory_bytes(mut self, limit: u64) -> Self {
665        self.config.max_memory_bytes = Some(limit);
666        self
667    }
668
669    pub fn max_batch_size(mut self, max_batch_size: usize) -> Self {
670        self.config.max_batch_size = Some(max_batch_size.max(1));
671        self
672    }
673
674    /// Registers an experimental/custom backend extension by its declared name.
675    pub fn register_backend<B>(mut self, backend: B) -> Self
676    where
677        B: Backend + 'static,
678    {
679        let name = backend.name().to_owned();
680        self.backends.insert(name, Arc::new(backend));
681        self
682    }
683
684    pub fn register_backend_arc(mut self, backend: Arc<dyn Backend>) -> Self {
685        self.backends.insert(backend.name().to_owned(), backend);
686        self
687    }
688
689    /// Registers an experimental/custom provider extension by its declared name.
690    pub fn register_provider<P>(mut self, provider: P) -> Self
691    where
692        P: Provider + 'static,
693    {
694        let name = provider.name().to_owned();
695        self.providers.insert(name, Arc::new(provider));
696        self
697    }
698
699    pub fn register_provider_arc(mut self, provider: Arc<dyn Provider>) -> Self {
700        self.providers.insert(provider.name().to_owned(), provider);
701        self
702    }
703
704    /// Registers a generative-inference provider by its declared name. The
705    /// caller selects it explicitly per request; it is never used as a
706    /// fallback for typed inference.
707    pub fn register_generation_provider<P>(mut self, provider: P) -> Self
708    where
709        P: GenerationProvider + 'static,
710    {
711        self.generation_providers
712            .insert(provider.name().to_owned(), Arc::new(provider));
713        self
714    }
715
716    pub fn register_decision_provider<P>(mut self, provider: P) -> Self
717    where
718        P: DecisionModelProvider + 'static,
719    {
720        self.decision_providers
721            .insert(provider.name().to_owned(), Arc::new(provider));
722        self
723    }
724
725    pub fn register_capability_provider<P>(mut self, provider: P) -> Self
726    where
727        P: CapabilityProvider + 'static,
728    {
729        self.capability_registry.register(provider);
730        self
731    }
732
733    pub fn build(self) -> Runtime {
734        Runtime {
735            config: self.config.clone(),
736            backends: self.backends,
737            providers: self.providers,
738            generation_providers: self.generation_providers,
739            decision_providers: self.decision_providers,
740            catalog: self.catalog,
741            capability_registry: self.capability_registry,
742            model_root: self.model_root,
743            loaded_models: RwLock::new(HashMap::new()),
744            inference_semaphore: Arc::new(Semaphore::new(
745                self.config.max_concurrent_inferences.max(1),
746            )),
747            load_semaphore: Arc::new(Semaphore::new(
748                self.config.max_concurrent_model_loads.max(1),
749            )),
750            cache_clock: AtomicU64::new(0),
751            metrics: Arc::new(RuntimeMetrics::default()),
752            execution_clock: AtomicU64::new(0),
753            execution_records: Mutex::new(VecDeque::new()),
754        }
755    }
756}
757
758/// Coordinates catalog resolution, routing, model lifecycle, and inference.
759///
760/// A runtime is explicit application state; it does not create global state or
761/// background services. Use [`Runtime::builder`] to obtain built-in CPU and
762/// ONNX execution.
763pub struct Runtime {
764    config: RuntimeConfig,
765    backends: BTreeMap<String, Arc<dyn Backend>>,
766    providers: BTreeMap<String, Arc<dyn Provider>>,
767    generation_providers: BTreeMap<String, Arc<dyn GenerationProvider>>,
768    decision_providers: BTreeMap<String, Arc<dyn DecisionModelProvider>>,
769    catalog: Option<Arc<dyn ModelCatalog>>,
770    capability_registry: CapabilityRegistry,
771    model_root: Option<std::path::PathBuf>,
772    loaded_models: RwLock<HashMap<ModelCacheKey, LoadedModel>>,
773    inference_semaphore: Arc<Semaphore>,
774    load_semaphore: Arc<Semaphore>,
775    cache_clock: AtomicU64,
776    metrics: Arc<RuntimeMetrics>,
777    execution_clock: AtomicU64,
778    execution_records: Mutex<VecDeque<ExecutionRecord>>,
779}
780
781impl Default for Runtime {
782    fn default() -> Self {
783        Self::new()
784    }
785}
786
787impl Runtime {
788    pub fn new() -> Self {
789        Self::builder().build()
790    }
791
792    pub fn builder() -> RuntimeBuilder {
793        RuntimeBuilder::default()
794    }
795
796    pub fn config(&self) -> &RuntimeConfig {
797        &self.config
798    }
799
800    /// Loads an immutable local typed-decision artifact through the single matching backend.
801    /// There is no fallback: missing, ambiguous, or invalid artifacts fail closed.
802    pub fn load_decision_model(
803        &self,
804        artifact: impl AsRef<Path>,
805    ) -> RuntimeResult<Box<dyn DecisionModel>> {
806        let artifact = artifact.as_ref();
807        if !artifact.exists() {
808            return Err(RuntimeError::ModelNotFound {
809                model: artifact.display().to_string(),
810            });
811        }
812        if !artifact.is_dir() {
813            return Err(RuntimeError::ModelUnavailable {
814                model: artifact.display().to_string(),
815                provider: "local".to_owned(),
816                reason: "typed-decision artifact must be a directory".to_owned(),
817            });
818        }
819        let candidates: Vec<_> = self
820            .decision_providers
821            .values()
822            .filter(|provider| provider.supports_artifact(artifact))
823            .collect();
824        match candidates.as_slice() {
825            [] => Err(RuntimeError::UnsupportedModel {
826                model: artifact.display().to_string(),
827                backend: "none".to_owned(),
828                reason: "no registered typed-decision provider accepts this artifact".to_owned(),
829            }),
830            [provider] => provider.load_decision_model(artifact),
831            _ => Err(RuntimeError::CapabilityMismatch {
832                reason: format!(
833                    "multiple typed-decision providers accept {}",
834                    artifact.display()
835                ),
836            }),
837        }
838    }
839
840    /// Returns model/backend execution facts without requiring applications to
841    /// inspect artifact layouts or backend names.
842    pub fn decision_capabilities(&self, model: &dyn DecisionModel) -> DecisionModelCapabilities {
843        model.capabilities()
844    }
845
846    /// Builds the deterministic execution plan for a loaded model and graph.
847    pub fn explain_decision(
848        &self,
849        model: &dyn DecisionModel,
850        request: &DecisionGraphRequest,
851    ) -> RuntimeResult<DecisionExecutionPlan> {
852        DecisionPlanner.plan(request, &model.capabilities())
853    }
854
855    /// Executes a graph asynchronously. Synchronous native model calls are
856    /// moved off the caller's async executor thread.
857    pub async fn execute_decision_graph(
858        &self,
859        model: Arc<dyn DecisionModel>,
860        request: DecisionGraphRequest,
861        cancellation: Option<CancellationToken>,
862    ) -> RuntimeResult<PlannedDecisionResult> {
863        decision::execute_graph(model, request, cancellation).await
864    }
865
866    /// Compatibility-friendly async execution for an ordinary independent
867    /// `DecisionRequest`.
868    pub async fn decide_async(
869        &self,
870        model: Arc<dyn DecisionModel>,
871        request: DecisionRequest,
872        policy: DecisionExecutionPolicy,
873        cancellation: Option<CancellationToken>,
874    ) -> RuntimeResult<PlannedDecisionResult> {
875        self.execute_decision_graph(
876            model,
877            DecisionGraphRequest::independent(request, policy),
878            cancellation,
879        )
880        .await
881    }
882
883    fn registered_model_root(&self) -> RuntimeResult<&Path> {
884        self.model_root
885            .as_deref()
886            .ok_or_else(|| RuntimeError::ModelUnavailable {
887                model: "registered models".to_owned(),
888                provider: "local".to_owned(),
889                reason: "model directory unavailable; set ML_RUNTIME_MODEL_DIR".to_owned(),
890            })
891    }
892
893    pub fn installed_models(&self) -> RuntimeResult<Vec<InstalledModelRecord>> {
894        let root = self.registered_model_root()?;
895        let installer = RegisteredModelInstaller::new(root)
896            .map_err(|error| RuntimeError::execution("model list", error.to_string()))?;
897        Ok(BuiltinModelRegistry
898            .list()
899            .into_iter()
900            .map(|model| {
901                let package = installer.installed_path(&model);
902                let (status, compiled) = if !package.exists() {
903                    (InstalledModelStatus::NotInstalled, false)
904                } else if model.backend == "coreml" && !cfg!(target_os = "macos") {
905                    (InstalledModelStatus::Unsupported, false)
906                } else {
907                    match InstallationManifest::read(&package) {
908                        Ok(manifest)
909                            if manifest.model == model.name
910                                && manifest.model_revision == model.revision
911                                && manifest.runtime_requirement == model.runtime_requirement
912                                && ml_runtime_model::runtime_requirement_satisfied(
913                                    &manifest.runtime_requirement,
914                                    VERSION,
915                                )
916                                && manifest.os == std::env::consts::OS
917                                && manifest.architecture == std::env::consts::ARCH
918                                && manifest.compiled_status == CompiledStatus::Ready
919                                && manifest
920                                    .compiled_artifact
921                                    .as_ref()
922                                    .is_some_and(|artifact| artifact.path.exists()) =>
923                        {
924                            (InstalledModelStatus::Ready, true)
925                        }
926                        Ok(manifest) if manifest.compiled_status != CompiledStatus::Ready => {
927                            (InstalledModelStatus::InstalledNotCompiled, false)
928                        }
929                        Ok(_) => (InstalledModelStatus::Incompatible, false),
930                        Err(_) => (InstalledModelStatus::Corrupt, false),
931                    }
932                };
933                InstalledModelRecord {
934                    model: model.name,
935                    revision: model.revision,
936                    backend: model.backend,
937                    platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
938                    status,
939                    compiled,
940                    package_path: package,
941                }
942            })
943            .collect())
944    }
945
946    pub async fn install_registered_model(
947        &self,
948        name: &str,
949        source_override: Option<&str>,
950        replace: bool,
951    ) -> RuntimeResult<ModelLifecycleResult> {
952        let started = Instant::now();
953        let model =
954            BuiltinModelRegistry
955                .resolve(name)
956                .map_err(|error| RuntimeError::ModelNotFound {
957                    model: error.to_string(),
958                })?;
959        if model.backend == "coreml" && !cfg!(target_os = "macos") {
960            return Err(RuntimeError::backend_unavailable(
961                "coreml",
962                "Laya installation requires macOS; Core ML is unavailable on this platform",
963            ));
964        }
965        let root = self.registered_model_root()?;
966        let installer = RegisteredModelInstaller::new(root)
967            .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
968        let installed = installer
969            .install(
970                &model,
971                source_override,
972                if replace {
973                    ml_runtime_model::InstallMode::Replace
974                } else {
975                    ml_runtime_model::InstallMode::KeepExisting
976                },
977                None,
978            )
979            .await
980            .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
981
982        let (prepared_path, model_identity, prepared_identity) = match model.backend.as_str() {
983            "onnx" => {
984                let identity = OnnxBackend::validate_laya(&installed.package_path)?;
985                (
986                    installed.package_path.join(&model.artifact_path),
987                    identity.clone(),
988                    identity,
989                )
990            }
991            "coreml" => {
992                #[cfg(feature = "coreml")]
993                {
994                    let prepared = if installed.already_installed && !replace {
995                        ml_runtime_coreml_backend::CoreMlBackend::validate_prepared_laya(
996                            &installed.package_path,
997                        )
998                        .or_else(|_| {
999                            ml_runtime_coreml_backend::CoreMlBackend::prepare_laya(
1000                                &installed.package_path,
1001                            )
1002                        })?
1003                    } else {
1004                        ml_runtime_coreml_backend::CoreMlBackend::prepare_laya(
1005                            &installed.package_path,
1006                        )?
1007                    };
1008                    (
1009                        prepared.path,
1010                        prepared.model_identity,
1011                        prepared.compiled_identity,
1012                    )
1013                }
1014                #[cfg(not(feature = "coreml"))]
1015                {
1016                    return Err(RuntimeError::backend_unavailable(
1017                        "coreml",
1018                        "this runtime was built without Core ML support",
1019                    ));
1020                }
1021            }
1022            backend => {
1023                return Err(RuntimeError::UnsupportedModel {
1024                    model: model.name.clone(),
1025                    backend: backend.to_owned(),
1026                    reason: "registered backend has no preparation lifecycle".to_owned(),
1027                })
1028            }
1029        };
1030
1031        {
1032            let manifest = InstallationManifest {
1033                schema_version: 1,
1034                model: model.name.clone(),
1035                model_revision: model.revision.clone(),
1036                model_checksum: model_identity,
1037                runtime_version: VERSION.to_owned(),
1038                runtime_requirement: model.runtime_requirement.clone(),
1039                backend: model.backend.clone(),
1040                os: std::env::consts::OS.to_owned(),
1041                architecture: std::env::consts::ARCH.to_owned(),
1042                installed_at_unix_seconds: std::time::SystemTime::now()
1043                    .duration_since(std::time::UNIX_EPOCH)
1044                    .unwrap_or_default()
1045                    .as_secs(),
1046                compiled_status: CompiledStatus::Ready,
1047                compiled_artifact: Some(CompiledArtifactManifest {
1048                    path: prepared_path,
1049                    identity: prepared_identity,
1050                }),
1051            };
1052            manifest
1053                .write_atomic(&installed.package_path)
1054                .map_err(|error| RuntimeError::execution("model install", error.to_string()))?;
1055            Ok(ModelLifecycleResult {
1056                model: InstalledModelRecord {
1057                    model: model.name,
1058                    revision: model.revision,
1059                    backend: model.backend,
1060                    platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1061                    status: InstalledModelStatus::Ready,
1062                    compiled: true,
1063                    package_path: installed.package_path,
1064                },
1065                already_installed: installed.already_installed,
1066                duration_ms: started.elapsed().as_millis(),
1067            })
1068        }
1069    }
1070
1071    pub fn doctor_registered_model(&self, name: &str) -> RuntimeResult<ModelDoctorReport> {
1072        let model =
1073            BuiltinModelRegistry
1074                .resolve(name)
1075                .map_err(|error| RuntimeError::ModelNotFound {
1076                    model: error.to_string(),
1077                })?;
1078        let root = self.registered_model_root()?;
1079        let installer = RegisteredModelInstaller::new(root)
1080            .map_err(|error| RuntimeError::execution("model doctor", error.to_string()))?;
1081        let package = installer.installed_path(&model);
1082        let mut diagnostics = Vec::new();
1083        if !package.is_dir() {
1084            diagnostics.push(ModelDiagnostic {
1085                check: "installation".to_owned(),
1086                ok: false,
1087                message: format!(
1088                    "model is not installed; run `ml-runtime model install {}`",
1089                    model.name
1090                ),
1091            });
1092            return Ok(ModelDoctorReport {
1093                model: InstalledModelRecord {
1094                    model: model.name,
1095                    revision: model.revision,
1096                    backend: model.backend,
1097                    platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1098                    status: InstalledModelStatus::NotInstalled,
1099                    compiled: false,
1100                    package_path: package,
1101                },
1102                diagnostics,
1103                manifest: None,
1104            });
1105        }
1106        let artifact_valid = match installer.validate(&model, &package) {
1107            Ok(()) => {
1108                diagnostics.push(ModelDiagnostic {
1109                    check: "artifact".to_owned(),
1110                    ok: true,
1111                    message: format!(
1112                        "registered files, {} artifact, tokenizer, and SHA-256 checksums are valid",
1113                        model.backend
1114                    ),
1115                });
1116                true
1117            }
1118            Err(error) => {
1119                diagnostics.push(ModelDiagnostic {
1120                    check: "artifact".to_owned(),
1121                    ok: false,
1122                    message: format!(
1123                        "{error}; reinstall with `ml-runtime model install {} --replace`",
1124                        model.name
1125                    ),
1126                });
1127                false
1128            }
1129        };
1130        let manifest = InstallationManifest::read(&package).ok();
1131        let compatible = manifest.as_ref().is_some_and(|manifest| {
1132            manifest.schema_version == 1
1133                && manifest.model == model.name
1134                && manifest.model_revision == model.revision
1135                && manifest.backend == model.backend
1136                && ml_runtime_model::runtime_requirement_satisfied(
1137                    &manifest.runtime_requirement,
1138                    VERSION,
1139                )
1140                && manifest.os == std::env::consts::OS
1141                && manifest.architecture == std::env::consts::ARCH
1142        });
1143        diagnostics.push(ModelDiagnostic {
1144            check: "metadata".to_owned(),
1145            ok: compatible,
1146            message: if compatible {
1147                "installation metadata and runtime/backend compatibility are valid".to_owned()
1148            } else {
1149                format!(
1150                    "installation metadata is missing or incompatible; run `ml-runtime model install {} --replace`",
1151                    model.name
1152                )
1153            },
1154        });
1155        let supported = model.supports_platform(std::env::consts::OS, std::env::consts::ARCH);
1156        diagnostics.push(ModelDiagnostic {
1157            check: "platform".to_owned(),
1158            ok: supported,
1159            message: if supported {
1160                format!("{} is supported on this platform", model.backend)
1161            } else {
1162                "Core ML is only supported on macOS".to_owned()
1163            },
1164        });
1165        let compiled_valid = supported
1166            && compatible
1167            && match model.backend.as_str() {
1168                "onnx" => OnnxBackend::validate_laya(&package).is_ok_and(|identity| {
1169                    manifest.as_ref().is_some_and(|manifest| {
1170                        manifest.model_checksum == identity
1171                            && manifest.compiled_artifact.as_ref().is_some_and(|artifact| {
1172                                artifact.path == package.join(&model.artifact_path)
1173                                    && artifact.identity == identity
1174                            })
1175                    })
1176                }),
1177                "coreml" => {
1178                    #[cfg(feature = "coreml")]
1179                    {
1180                        ml_runtime_coreml_backend::CoreMlBackend::validate_prepared_laya(&package)
1181                            .is_ok()
1182                    }
1183                    #[cfg(not(feature = "coreml"))]
1184                    {
1185                        false
1186                    }
1187                }
1188                _ => false,
1189            };
1190        diagnostics.push(ModelDiagnostic {
1191            check: "compiled_artifact".to_owned(),
1192            ok: compiled_valid,
1193            message: if compiled_valid {
1194                format!("prepared {} artifact identity and schema are valid", model.backend)
1195            } else {
1196                format!(
1197                    "compiled artifact is missing or invalid; run `ml-runtime model install {} --replace`",
1198                    model.name
1199                )
1200            },
1201        });
1202        let status = if !supported {
1203            InstalledModelStatus::Unsupported
1204        } else if !artifact_valid {
1205            InstalledModelStatus::Corrupt
1206        } else if !compatible {
1207            InstalledModelStatus::Incompatible
1208        } else if !compiled_valid {
1209            InstalledModelStatus::InstalledNotCompiled
1210        } else {
1211            InstalledModelStatus::Ready
1212        };
1213        Ok(ModelDoctorReport {
1214            model: InstalledModelRecord {
1215                model: model.name,
1216                revision: model.revision,
1217                backend: model.backend,
1218                platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1219                compiled: compiled_valid,
1220                package_path: package,
1221                status,
1222            },
1223            diagnostics,
1224            manifest,
1225        })
1226    }
1227
1228    pub fn remove_registered_model(&self, name: &str) -> RuntimeResult<InstalledModelRecord> {
1229        let model =
1230            BuiltinModelRegistry
1231                .resolve(name)
1232                .map_err(|error| RuntimeError::ModelNotFound {
1233                    model: error.to_string(),
1234                })?;
1235        let root = self.registered_model_root()?;
1236        let installer = RegisteredModelInstaller::new(root)
1237            .map_err(|error| RuntimeError::execution("model remove", error.to_string()))?;
1238        let package = installer.installed_path(&model);
1239        if package.exists() {
1240            #[cfg(feature = "coreml")]
1241            if model.backend == "coreml" {
1242                ml_runtime_coreml_backend::CoreMlBackend::remove_prepared_laya(&package)?;
1243            }
1244            std::fs::remove_dir_all(&package)
1245                .map_err(|error| RuntimeError::execution("model remove", error.to_string()))?;
1246        }
1247        Ok(InstalledModelRecord {
1248            model: model.name,
1249            revision: model.revision,
1250            backend: model.backend,
1251            platform: format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH),
1252            status: InstalledModelStatus::NotInstalled,
1253            compiled: false,
1254            package_path: package,
1255        })
1256    }
1257
1258    /// Loads a validated installed model by its registry name. Source, artifact
1259    /// layout, tokenizer, and backend selection remain runtime concerns.
1260    pub fn load_model(&self, name: &str) -> RuntimeResult<Box<dyn DecisionModel>> {
1261        let registered =
1262            BuiltinModelRegistry
1263                .resolve(name)
1264                .map_err(|error| RuntimeError::ModelNotFound {
1265                    model: error.to_string(),
1266                })?;
1267        let root = self
1268            .model_root
1269            .as_ref()
1270            .ok_or_else(|| RuntimeError::ModelUnavailable {
1271                model: name.to_owned(),
1272                provider: "local".to_owned(),
1273                reason: "model directory unavailable; set ML_RUNTIME_MODEL_DIR".to_owned(),
1274            })?;
1275        let installer = RegisteredModelInstaller::new(root).map_err(|error| {
1276            RuntimeError::execution("installed model validation", error.to_string())
1277        })?;
1278        let package = installer.installed_path(&registered);
1279        if !package.exists() {
1280            return Err(RuntimeError::ModelNotFound {
1281                model: format!("{name}; install it with `ml-runtime model install {name}`"),
1282            });
1283        }
1284        installer.validate(&registered, &package).map_err(|error| {
1285            RuntimeError::ModelIntegrity {
1286                model: name.to_owned(),
1287                reason: format!(
1288                    "{error}; reinstall with `ml-runtime model install {name} --replace`"
1289                ),
1290            }
1291        })?;
1292        let manifest =
1293            InstallationManifest::read(&package).map_err(|error| RuntimeError::ModelIntegrity {
1294                model: name.to_owned(),
1295                reason: format!("{error}; run `ml-runtime model doctor {name}`"),
1296            })?;
1297        let compatible = manifest.schema_version == 1
1298            && manifest.model == registered.name
1299            && manifest.model_revision == registered.revision
1300            && manifest.backend == registered.backend
1301            && ml_runtime_model::runtime_requirement_satisfied(
1302                &manifest.runtime_requirement,
1303                VERSION,
1304            )
1305            && manifest.os == std::env::consts::OS
1306            && manifest.architecture == std::env::consts::ARCH
1307            && manifest.compiled_status == CompiledStatus::Ready
1308            && manifest.compiled_artifact.as_ref().is_some_and(|artifact| {
1309                artifact.path.exists()
1310                    && (registered.backend != "onnx"
1311                        || (artifact.path == package.join(&registered.artifact_path)
1312                            && artifact.identity == manifest.model_checksum))
1313            });
1314        if !compatible {
1315            return Err(RuntimeError::ModelUnavailable {
1316                model: name.to_owned(),
1317                provider: "local".to_owned(),
1318                reason: format!(
1319                    "installation is incomplete, incompatible, or not compiled; run `ml-runtime model doctor {name}`"
1320                ),
1321            });
1322        }
1323        self.load_decision_model(package)
1324    }
1325
1326    pub fn set_catalog<C>(&mut self, catalog: C)
1327    where
1328        C: ModelCatalog + 'static,
1329    {
1330        self.catalog = Some(Arc::new(catalog));
1331    }
1332
1333    pub async fn catalog_models(&self) -> RuntimeResult<Vec<ModelDescriptor>> {
1334        self.catalog
1335            .as_ref()
1336            .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1337            .list()
1338            .await
1339            .map_err(catalog_error)
1340    }
1341
1342    pub async fn resolve_model(
1343        &self,
1344        reference: &ModelReference,
1345    ) -> RuntimeResult<ModelDescriptor> {
1346        self.catalog
1347            .as_ref()
1348            .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1349            .resolve(reference)
1350            .await
1351            .map_err(catalog_error)
1352    }
1353
1354    pub async fn refresh_catalog(&self) -> RuntimeResult<()> {
1355        self.catalog
1356            .as_ref()
1357            .ok_or_else(|| RuntimeError::execution("catalog", "no model catalog configured"))?
1358            .refresh()
1359            .await
1360            .map_err(catalog_error)
1361    }
1362
1363    pub fn models(&self) -> ModelRegistry {
1364        let models = self
1365            .loaded_models
1366            .read()
1367            .expect("loaded model registry poisoned")
1368            .values()
1369            .map(|entry| ModelCapability {
1370                id: entry.spec.id.clone(),
1371                format: entry.spec.format.clone(),
1372                location: entry.spec.location.clone(),
1373                loaded: true,
1374                provider: Some(entry.provider_name.clone()),
1375                backend: Some(entry.backend_name.clone()),
1376            })
1377            .collect();
1378        ModelRegistry { models }
1379    }
1380
1381    pub fn providers(&self) -> ProviderRegistry {
1382        let providers = self
1383            .providers
1384            .iter()
1385            .map(|(name, provider)| {
1386                ProviderCapability::from_parts(name.clone(), provider.capabilities())
1387            })
1388            .collect();
1389        ProviderRegistry { providers }
1390    }
1391
1392    pub fn backends(&self) -> BackendRegistry {
1393        let backends = self
1394            .backends
1395            .iter()
1396            .map(|(name, backend)| {
1397                BackendCapability::from_parts(name.clone(), backend.capabilities())
1398            })
1399            .collect();
1400        BackendRegistry { backends }
1401    }
1402
1403    pub fn capabilities(&self) -> RuntimeCapabilities {
1404        RuntimeCapabilities {
1405            environment: self.environment(),
1406            platforms: vec![Platform::current()],
1407            providers: self.providers().providers,
1408            backends: self.backends().backends,
1409            models: self.models().models,
1410            capabilities: self.capability_registry.list(),
1411        }
1412    }
1413
1414    pub fn capability(&self, id: &str) -> Option<Capability> {
1415        self.capability_registry.get(id)
1416    }
1417
1418    pub fn capability_registry(&self) -> &CapabilityRegistry {
1419        &self.capability_registry
1420    }
1421
1422    /// Explicitly discovers and registers local capability package metadata.
1423    ///
1424    /// Discovery never loads or executes a provider binary.
1425    pub fn load_capability_packages(
1426        &mut self,
1427        root: impl AsRef<std::path::Path>,
1428    ) -> RuntimeResult<Vec<String>> {
1429        self.capability_registry.load_local_packages(root)
1430    }
1431
1432    pub fn environment(&self) -> RuntimeEnvironment {
1433        RuntimeEnvironment::discover(
1434            self.capability_registry
1435                .providers()
1436                .iter()
1437                .map(|provider| provider.id().to_owned())
1438                .collect(),
1439        )
1440    }
1441
1442    pub fn resolve_capability(
1443        &self,
1444        request: &ExecutionRequest,
1445    ) -> RuntimeResult<CapabilityResolution> {
1446        self.resolve_capability_with_constraints(request, &ResolutionConstraints::default())
1447    }
1448
1449    pub fn resolve_capability_with_constraints(
1450        &self,
1451        request: &ExecutionRequest,
1452        constraints: &ResolutionConstraints,
1453    ) -> RuntimeResult<CapabilityResolution> {
1454        self.capability_registry
1455            .resolve(&request.capability, constraints)
1456    }
1457
1458    pub async fn execute_capability(
1459        &self,
1460        request: ExecutionRequest,
1461        authorizer: Option<&dyn CapabilityAuthorizer>,
1462    ) -> RuntimeResult<ExecutionResult> {
1463        let resolution = self
1464            .capability_registry
1465            .resolve(&request.capability, &ResolutionConstraints::default())?;
1466        let provider = self
1467            .capability_registry
1468            .provider(&resolution.provider)
1469            .ok_or_else(|| {
1470                RuntimeError::provider_unavailable(&resolution.provider, "not registered")
1471            })?;
1472        if resolution.provider == "builtin" {
1473            capability::execute(request, authorizer).await
1474        } else {
1475            if let Some(authorizer) = authorizer {
1476                if request.policy.require_authorization {
1477                    let capability = self
1478                        .capability_registry
1479                        .get(&request.capability)
1480                        .ok_or_else(|| RuntimeError::CapabilityMismatch {
1481                            reason: format!("unknown capability {}", request.capability),
1482                        })?;
1483                    authorizer.authorize(&request, &capability).await?;
1484                }
1485            } else if self
1486                .capability_registry
1487                .get(&request.capability)
1488                .is_some_and(|capability| !capability.requirements.is_empty())
1489            {
1490                return Err(RuntimeError::Execution {
1491                    operation: request.capability,
1492                    reason: "authorization required".to_owned(),
1493                });
1494            }
1495            provider.execute(request).await
1496        }
1497    }
1498
1499    pub fn status(&self) -> RuntimeStatus {
1500        RuntimeStatus {
1501            loaded_models: self
1502                .loaded_models
1503                .read()
1504                .expect("loaded model registry poisoned")
1505                .len(),
1506            max_models: self.config.max_models,
1507            max_concurrent_inferences: self.config.max_concurrent_inferences,
1508            max_concurrent_model_loads: self.config.max_concurrent_model_loads,
1509        }
1510    }
1511
1512    pub fn metrics(&self) -> RuntimeMetricsSnapshot {
1513        self.metrics.snapshot()
1514    }
1515
1516    pub fn executions(&self) -> Vec<ExecutionRecord> {
1517        self.execution_records
1518            .lock()
1519            .expect("execution records poisoned")
1520            .iter()
1521            .cloned()
1522            .collect()
1523    }
1524
1525    pub fn resources(&self) -> Vec<ModelResource> {
1526        self.loaded_models
1527            .read()
1528            .expect("loaded model registry poisoned")
1529            .values()
1530            .map(|model| ModelResource {
1531                id: model.spec.id.clone(),
1532                version: model.spec.version.clone(),
1533                target: format!("{}/{}", model.provider_name, model.backend_name),
1534                state: ModelLifecycleState::Ready,
1535                memory_bytes: model.memory_bytes,
1536                last_used: model.last_used,
1537            })
1538            .collect()
1539    }
1540
1541    pub fn snapshot(&self) -> RuntimeSnapshot {
1542        let resources = self.resources();
1543        let metrics = self.metrics();
1544        RuntimeSnapshot {
1545            active_inferences: metrics.active_inferences,
1546            active_model_loads: metrics.active_model_loads,
1547            queued_inferences: (metrics.queued_requests as usize)
1548                .saturating_sub(metrics.active_inferences),
1549            loaded_models: resources.len(),
1550            cache_entries: resources.len(),
1551            cache_memory_bytes: resources.iter().map(|resource| resource.memory_bytes).sum(),
1552            metrics,
1553        }
1554    }
1555
1556    pub fn selection_for(&self, model: &ModelSpec) -> RuntimeResult<RuntimeSelection> {
1557        let backend_name = self.select_backend(model)?;
1558        let backend = self
1559            .backends
1560            .get(&backend_name)
1561            .ok_or_else(|| RuntimeError::backend_unavailable(&backend_name, "not registered"))?;
1562        Ok(RuntimeSelection {
1563            model: model.id.clone(),
1564            provider: "local".to_owned(),
1565            backend: Some(backend_name),
1566            hardware: backend.capabilities().hardware,
1567            fallback: false,
1568            fallback_from: None,
1569            reason: "preferred local execution".to_owned(),
1570            policy: ExecutionPolicy::LocalOnly,
1571        })
1572    }
1573
1574    /// Explicitly loads a model for repeated execution with [`Self::infer_loaded`].
1575    /// Normal one-off requests should use [`Self::infer`], which loads lazily.
1576    pub async fn load(&self, model: ModelSpec) -> RuntimeResult<RuntimeModelHandle> {
1577        if self
1578            .config
1579            .selected_provider
1580            .as_deref()
1581            .is_some_and(|provider| provider != "local")
1582        {
1583            return Err(RuntimeError::provider_unavailable(
1584                self.config
1585                    .selected_provider
1586                    .clone()
1587                    .unwrap_or_else(|| "remote".to_owned()),
1588                "load is only supported for local execution paths",
1589            ));
1590        }
1591
1592        let backend_name = self.select_backend(&model)?;
1593        let key = ModelCacheKey {
1594            id: model.id.clone(),
1595            version: model.version.clone(),
1596            provider: "local".to_owned(),
1597            backend: backend_name.clone(),
1598        };
1599        if let Some(existing) = self
1600            .loaded_models
1601            .read()
1602            .expect("loaded model registry poisoned")
1603            .get(&key)
1604        {
1605            self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1606            return Ok(runtime_handle(existing));
1607        }
1608        self.metrics.cache_misses.fetch_add(1, Ordering::Relaxed);
1609        self.metrics
1610            .active_model_loads
1611            .fetch_add(1, Ordering::Relaxed);
1612        let _permit = self
1613            .load_semaphore
1614            .acquire()
1615            .await
1616            .map_err(|_| RuntimeError::execution("load", "runtime load semaphore closed"))?;
1617        if let Some(existing) = self
1618            .loaded_models
1619            .read()
1620            .expect("loaded model registry poisoned")
1621            .get(&key)
1622        {
1623            self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1624            return Ok(runtime_handle(existing));
1625        }
1626        let backend = self
1627            .backends
1628            .get(&backend_name)
1629            .ok_or_else(|| RuntimeError::backend_unavailable(&backend_name, "not registered"))?
1630            .clone();
1631        let timeout_ms = self.config.timeout_ms;
1632        let loaded = run_with_timeout(timeout_ms, "load", backend.load(&model)).await;
1633        self.metrics
1634            .active_model_loads
1635            .fetch_sub(1, Ordering::Relaxed);
1636        let handle = match loaded {
1637            Ok(handle) => handle,
1638            Err(error) => {
1639                self.metrics
1640                    .model_load_failures
1641                    .fetch_add(1, Ordering::Relaxed);
1642                return Err(error);
1643            }
1644        };
1645
1646        let mut cache = self
1647            .loaded_models
1648            .write()
1649            .expect("loaded model registry poisoned");
1650        if cache.len() >= self.config.max_models {
1651            if let Some(oldest) = cache
1652                .iter()
1653                .min_by_key(|(_, model)| model.last_used)
1654                .map(|(key, _)| key.clone())
1655            {
1656                cache.remove(&oldest);
1657                self.metrics.evictions.fetch_add(1, Ordering::Relaxed);
1658            }
1659        }
1660        self.metrics.model_loads.fetch_add(1, Ordering::Relaxed);
1661        let loaded = LoadedModel {
1662            memory_bytes: model_memory_bytes(&model),
1663            spec: model,
1664            handle,
1665            backend_name,
1666            provider_name: "local".to_owned(),
1667            last_used: self.cache_clock.fetch_add(1, Ordering::Relaxed),
1668        };
1669        let runtime_handle = runtime_handle(&loaded);
1670        cache.insert(key, loaded);
1671        Ok(runtime_handle)
1672    }
1673
1674    pub async fn load_reference(
1675        &self,
1676        reference: ModelReference,
1677    ) -> RuntimeResult<RuntimeModelHandle> {
1678        match reference {
1679            ModelReference::Spec(spec) => self.load(spec).await,
1680            ModelReference::Id { id, version } => self
1681                .find_loaded(&id, version.as_ref())
1682                .map(|loaded| runtime_handle(&loaded)),
1683        }
1684    }
1685
1686    /// Resolves, loads, routes, and executes one request.
1687    pub async fn infer(&self, request: InferenceRequest) -> RuntimeResult<InferenceResult> {
1688        self.infer_with_cancellation(request, CancellationToken::new())
1689            .await
1690    }
1691
1692    /// Executes against a previously loaded handle without exposing cache internals.
1693    pub async fn infer_loaded(
1694        &self,
1695        model: &RuntimeModelHandle,
1696        input: Input,
1697        options: InferenceOptions,
1698    ) -> RuntimeResult<InferenceResult> {
1699        self.infer(InferenceRequest {
1700            model: ModelReference::id(model.id().to_owned(), model.version().map(str::to_owned)),
1701            input,
1702            options,
1703        })
1704        .await
1705    }
1706
1707    pub async fn infer_loaded_stream(
1708        &self,
1709        model: &RuntimeModelHandle,
1710        input: Input,
1711        mut options: InferenceOptions,
1712    ) -> RuntimeResult<InferenceStream> {
1713        options.stream = true;
1714        self.infer_stream(InferenceRequest {
1715            model: ModelReference::id(model.id().to_owned(), model.version().map(str::to_owned)),
1716            input,
1717            options,
1718        })
1719        .await
1720    }
1721
1722    /// Executes requests in stable input order.
1723    ///
1724    /// The call fails as a whole on a request error. Backends without native
1725    /// batching are invoked individually behind the same API.
1726    pub async fn infer_batch(
1727        &self,
1728        mut requests: Vec<InferenceRequest>,
1729    ) -> RuntimeResult<Vec<InferenceResult>> {
1730        if requests.is_empty() {
1731            return Ok(Vec::new());
1732        }
1733        if let Some(limit) = self.config.max_batch_size {
1734            if requests.len() > limit {
1735                return Err(RuntimeError::ResourceLimit {
1736                    reason: format!(
1737                        "batch size {} exceeds configured limit {limit}",
1738                        requests.len()
1739                    ),
1740                });
1741            }
1742        }
1743        for request in &requests {
1744            self.validate_request(request)?;
1745        }
1746        for request in &mut requests {
1747            self.resolve_catalog_reference_for_local(request).await?;
1748        }
1749        let selection = self.select_for_request(&requests[0])?;
1750        let same_model = requests
1751            .iter()
1752            .all(|request| model_id(&request.model) == model_id(&requests[0].model));
1753        if selection.provider != "local" || !same_model {
1754            return futures_util::future::try_join_all(
1755                requests.into_iter().map(|request| self.infer(request)),
1756            )
1757            .await;
1758        }
1759        let loaded = self.resolve_loaded_model(&requests[0].model).await?;
1760        let backend = self
1761            .backends
1762            .get(&loaded.backend_name)
1763            .ok_or_else(|| {
1764                RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
1765            })?
1766            .clone();
1767        if !backend.capabilities().batching {
1768            return futures_util::future::try_join_all(
1769                requests.into_iter().map(|request| self.infer(request)),
1770            )
1771            .await;
1772        }
1773        let batch_size = requests.len();
1774        self.metrics
1775            .total_requests
1776            .fetch_add(batch_size as u64, Ordering::Relaxed);
1777        self.metrics
1778            .batch_executions
1779            .fetch_add(1, Ordering::Relaxed);
1780        self.metrics
1781            .total_requests_batched
1782            .fetch_add(batch_size as u64, Ordering::Relaxed);
1783        self.metrics
1784            .batch_sizes
1785            .lock()
1786            .expect("metrics poisoned")
1787            .push(batch_size);
1788        let _permit = self.inference_semaphore.acquire().await.map_err(|_| {
1789            RuntimeError::execution("infer_batch", "runtime inference semaphore closed")
1790        })?;
1791        let mut results = run_with_timeout(
1792            self.config.timeout_ms,
1793            "infer_batch",
1794            backend.infer_batch(&loaded.handle, &requests, Some(CancellationToken::new())),
1795        )
1796        .await?;
1797        if results.len() != batch_size {
1798            return Err(RuntimeError::execution(
1799                "infer_batch",
1800                format!(
1801                    "backend returned {} results for {batch_size} requests",
1802                    results.len()
1803                ),
1804            ));
1805        }
1806        for (result, request) in results.iter_mut().zip(requests.iter()) {
1807            let output = result.output.clone();
1808            decorate_metadata(
1809                &mut result.metadata,
1810                &output,
1811                &loaded.spec.id,
1812                "local",
1813                &loaded.backend_name,
1814                backend.capabilities().hardware.clone(),
1815                Duration::ZERO,
1816                estimate_input_tokens(&request.input),
1817                loaded.spec.version.clone(),
1818                selection.policy.clone(),
1819                false,
1820                None,
1821                Some(selection.reason.clone()),
1822            );
1823            result.metadata.batch_size = batch_size;
1824            result.metadata.cache_hit = true;
1825        }
1826        self.metrics
1827            .successful_requests
1828            .fetch_add(results.len() as u64, Ordering::Relaxed);
1829        Ok(results)
1830    }
1831
1832    /// Executes one request while propagating cancellation through scheduling,
1833    /// provider transport, and backend execution.
1834    pub async fn infer_with_cancellation(
1835        &self,
1836        request: InferenceRequest,
1837        cancellation: CancellationToken,
1838    ) -> RuntimeResult<InferenceResult> {
1839        let started = Instant::now();
1840        let request_id = format!(
1841            "request-{}",
1842            self.execution_clock.fetch_add(1, Ordering::Relaxed) + 1
1843        );
1844        self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
1845        self.metrics
1846            .active_inferences
1847            .fetch_add(1, Ordering::Relaxed);
1848        let model = model_details(&request.model);
1849        let result = self.infer_inner(request.clone(), cancellation).await;
1850        self.metrics
1851            .active_inferences
1852            .fetch_sub(1, Ordering::Relaxed);
1853        match result {
1854            Ok(mut result) => {
1855                result.metadata.request_id = Some(request_id.clone());
1856                self.record_execution(
1857                    &request_id,
1858                    started.elapsed(),
1859                    &result.metadata,
1860                    ExecutionCompletion::Succeeded,
1861                );
1862                self.metrics
1863                    .successful_requests
1864                    .fetch_add(1, Ordering::Relaxed);
1865                Ok(result)
1866            }
1867            Err(error) => {
1868                let completion = if matches!(error, RuntimeError::Cancelled) {
1869                    self.metrics
1870                        .cancelled_requests
1871                        .fetch_add(1, Ordering::Relaxed);
1872                    ExecutionCompletion::Cancelled
1873                } else {
1874                    self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
1875                    ExecutionCompletion::Failed
1876                };
1877                let metadata = ExecutionMetadata {
1878                    request_id: Some(request_id.clone()),
1879                    model: model.0,
1880                    model_version: model.1,
1881                    routing_policy: request.options.execution.clone(),
1882                    streaming_requested: request.options.stream,
1883                    ..ExecutionMetadata::default()
1884                };
1885                self.record_execution(&request_id, started.elapsed(), &metadata, completion);
1886                Err(error)
1887            }
1888        }
1889    }
1890
1891    async fn infer_inner(
1892        &self,
1893        mut request: InferenceRequest,
1894        cancellation: CancellationToken,
1895    ) -> RuntimeResult<InferenceResult> {
1896        self.validate_request(&request)?;
1897        if cancellation.is_cancelled() {
1898            return Err(RuntimeError::Cancelled);
1899        }
1900
1901        let _permit = tokio::select! {
1902            permit = self.inference_semaphore.acquire() => permit
1903                .map_err(|_| RuntimeError::execution("infer", "runtime inference semaphore closed"))?,
1904            _ = cancellation.cancelled() => return Err(RuntimeError::Cancelled),
1905        };
1906
1907        self.resolve_catalog_reference_for_local(&mut request)
1908            .await?;
1909
1910        let mut selection = self.select_for_request(&request)?;
1911        let timeout_ms = request.options.timeout_ms.or(self.config.timeout_ms);
1912        let input_tokens = estimate_input_tokens(&request.input);
1913
1914        match selection.provider.as_str() {
1915            "local" => {
1916                let cache_hit = self.is_loaded(&request.model);
1917                let model_load_started = Instant::now();
1918                if cache_hit {
1919                    self.metrics.cache_hits.fetch_add(1, Ordering::Relaxed);
1920                } else {
1921                    self.metrics.cache_misses.fetch_add(1, Ordering::Relaxed);
1922                }
1923
1924                let loaded = match self.resolve_loaded_model(&request.model).await {
1925                    Ok(loaded) => loaded,
1926                    Err(error) if self.can_fallback(&request, &selection, &error) => {
1927                        selection = self.remote_selection(
1928                            model_id(&request.model),
1929                            true,
1930                            Some("local model unavailable".to_owned()),
1931                            &request.options,
1932                        )?;
1933                        return self
1934                            .infer_remote(
1935                                request,
1936                                cancellation,
1937                                timeout_ms,
1938                                input_tokens,
1939                                selection,
1940                            )
1941                            .await;
1942                    }
1943                    Err(error) => return Err(error),
1944                };
1945                let backend = self
1946                    .backends
1947                    .get(&loaded.backend_name)
1948                    .ok_or_else(|| {
1949                        RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
1950                    })?
1951                    .clone();
1952                let started = Instant::now();
1953                let result = run_with_timeout(
1954                    timeout_ms,
1955                    "infer",
1956                    backend.infer(&loaded.handle, &request, Some(cancellation.clone())),
1957                )
1958                .await;
1959                let mut result = match result {
1960                    Ok(result) => result,
1961                    Err(error) if self.can_fallback(&request, &selection, &error) => {
1962                        selection = self.remote_selection(
1963                            model_id(&request.model),
1964                            true,
1965                            Some(error.to_string()),
1966                            &request.options,
1967                        )?;
1968                        return self
1969                            .infer_remote(
1970                                request,
1971                                cancellation,
1972                                timeout_ms,
1973                                input_tokens,
1974                                selection,
1975                            )
1976                            .await;
1977                    }
1978                    Err(error) => return Err(error),
1979                };
1980                result.metadata.model_load_ms = Some(if cache_hit {
1981                    0.0
1982                } else {
1983                    model_load_started.elapsed().as_secs_f64() * 1000.0
1984                });
1985                let capabilities = backend.capabilities();
1986                let output = result.output.clone();
1987                decorate_metadata(
1988                    &mut result.metadata,
1989                    &output,
1990                    &loaded.spec.id,
1991                    "local",
1992                    &loaded.backend_name,
1993                    capabilities.hardware,
1994                    started.elapsed(),
1995                    input_tokens,
1996                    loaded.spec.version.clone(),
1997                    selection.policy.clone(),
1998                    selection.fallback,
1999                    selection.fallback_from.clone(),
2000                    Some(selection.reason.clone()),
2001                );
2002                result.metadata.cache_hit = cache_hit;
2003                Ok(result)
2004            }
2005            _ => {
2006                self.infer_remote(request, cancellation, timeout_ms, input_tokens, selection)
2007                    .await
2008            }
2009        }
2010    }
2011
2012    fn record_execution(
2013        &self,
2014        request_id: &str,
2015        total_duration: Duration,
2016        metadata: &ExecutionMetadata,
2017        completion: ExecutionCompletion,
2018    ) {
2019        let execution_id = format!(
2020            "execution-{}",
2021            self.execution_clock.fetch_add(1, Ordering::Relaxed) + 1
2022        );
2023        let execution_duration =
2024            Duration::from_secs_f64(metadata.execution_ms.unwrap_or(0.0) / 1000.0);
2025        let queued_duration =
2026            Duration::from_secs_f64(metadata.queue_wait_ms.unwrap_or(0.0) / 1000.0);
2027        let model_load_duration =
2028            Duration::from_secs_f64(metadata.model_load_ms.unwrap_or(0.0) / 1000.0);
2029        self.metrics
2030            .queue_wait_micros
2031            .lock()
2032            .expect("metrics poisoned")
2033            .push(queued_duration.as_micros() as u64);
2034        self.metrics
2035            .model_load_micros
2036            .lock()
2037            .expect("metrics poisoned")
2038            .push(model_load_duration.as_micros() as u64);
2039        self.metrics
2040            .execution_micros
2041            .lock()
2042            .expect("metrics poisoned")
2043            .push(execution_duration.as_micros() as u64);
2044        self.metrics
2045            .total_duration_micros
2046            .lock()
2047            .expect("metrics poisoned")
2048            .push(total_duration.as_micros() as u64);
2049        if metadata.provider == "local" {
2050            self.metrics
2051                .local_executions
2052                .fetch_add(1, Ordering::Relaxed);
2053        } else {
2054            self.metrics
2055                .remote_executions
2056                .fetch_add(1, Ordering::Relaxed);
2057        }
2058        if metadata.fallback {
2059            self.metrics
2060                .fallback_executions
2061                .fetch_add(1, Ordering::Relaxed);
2062        }
2063        if metadata.streaming_requested {
2064            self.metrics
2065                .streaming_requests
2066                .fetch_add(1, Ordering::Relaxed);
2067        }
2068        let record = ExecutionRecord {
2069            execution_id,
2070            request_id: request_id.to_owned(),
2071            model_id: metadata.model.clone(),
2072            model_version: metadata.model_version.clone().unwrap_or_default(),
2073            provider: metadata.provider.clone(),
2074            backend: metadata.backend.clone(),
2075            target: metadata.execution_target.clone(),
2076            routing_policy: metadata.routing_policy.clone(),
2077            fallback: metadata.fallback_from.clone(),
2078            cache_hit: metadata.cache_hit,
2079            batch_size: metadata.batch_size.max(1),
2080            queued_duration,
2081            model_load_duration,
2082            execution_duration,
2083            total_duration,
2084            streaming: metadata.streaming_requested,
2085            output_event_count: metadata.output_event_count,
2086            completion,
2087        };
2088        let mut records = self
2089            .execution_records
2090            .lock()
2091            .expect("execution records poisoned");
2092        records.push_back(record);
2093        while records.len() > 256 {
2094            records.pop_front();
2095        }
2096    }
2097
2098    async fn infer_remote(
2099        &self,
2100        request: InferenceRequest,
2101        cancellation: CancellationToken,
2102        timeout_ms: Option<u64>,
2103        input_tokens: Option<u64>,
2104        selection: RuntimeSelection,
2105    ) -> RuntimeResult<InferenceResult> {
2106        let provider_name = selection.provider.as_str();
2107        let provider = self
2108            .providers
2109            .get(provider_name)
2110            .ok_or_else(|| RuntimeError::provider_unavailable(provider_name, "not registered"))?
2111            .clone();
2112        let started = Instant::now();
2113        let mut result = run_with_timeout(
2114            timeout_ms,
2115            "infer",
2116            provider.infer(request.clone(), Some(cancellation)),
2117        )
2118        .await?;
2119        let backend_name = result.metadata.backend.clone();
2120        let hardware = result.metadata.hardware.clone();
2121        let output = result.output.clone();
2122        let (model, version) = model_details(&request.model);
2123        decorate_metadata(
2124            &mut result.metadata,
2125            &output,
2126            &model,
2127            provider_name,
2128            backend_name.as_str(),
2129            hardware,
2130            started.elapsed(),
2131            input_tokens,
2132            version,
2133            selection.policy,
2134            selection.fallback,
2135            selection.fallback_from,
2136            Some(selection.reason),
2137        );
2138        Ok(result)
2139    }
2140
2141    /// Starts normalized `Started` → `Output*` → `Completed` streaming.
2142    pub async fn infer_stream(&self, request: InferenceRequest) -> RuntimeResult<InferenceStream> {
2143        self.infer_stream_with_cancellation(request, CancellationToken::new())
2144            .await
2145    }
2146
2147    /// Starts a bounded stream with caller-controlled cancellation.
2148    pub async fn infer_stream_with_cancellation(
2149        &self,
2150        mut request: InferenceRequest,
2151        cancellation: CancellationToken,
2152    ) -> RuntimeResult<InferenceStream> {
2153        self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
2154        self.metrics
2155            .streaming_requests
2156            .fetch_add(1, Ordering::Relaxed);
2157        self.validate_request(&request)?;
2158        if !request.options.stream {
2159            return Err(RuntimeError::CapabilityMismatch {
2160                reason: "infer_stream requires InferenceOptions::stream to be true".to_owned(),
2161            });
2162        }
2163        self.resolve_catalog_reference_for_local(&mut request)
2164            .await?;
2165        let selection = self.select_for_request(&request)?;
2166        let buffer = self.config.max_stream_buffer;
2167        if buffer == 0 {
2168            return Err(RuntimeError::ResourceLimit {
2169                reason: "max_stream_buffer must be greater than zero".to_owned(),
2170            });
2171        }
2172        match selection.provider.as_str() {
2173            "local" => {
2174                let loaded = self.resolve_loaded_model(&request.model).await?;
2175                let backend = self
2176                    .backends
2177                    .get(&loaded.backend_name)
2178                    .ok_or_else(|| {
2179                        RuntimeError::backend_unavailable(&loaded.backend_name, "not registered")
2180                    })?
2181                    .clone();
2182                let capabilities = backend.capabilities();
2183                let stream = if capabilities.streaming {
2184                    backend
2185                        .infer_stream(&loaded.handle, &request, Some(cancellation.clone()))
2186                        .await?
2187                } else {
2188                    self.metrics
2189                        .compatibility_streams
2190                        .fetch_add(1, Ordering::Relaxed);
2191                    let result = backend
2192                        .infer(&loaded.handle, &request, Some(cancellation.clone()))
2193                        .await?;
2194                    Box::pin(futures_util::stream::iter(vec![Ok(InferenceChunk {
2195                        output: result.output,
2196                        done: true,
2197                        metadata: Some(result.metadata),
2198                    })])) as BoxStream<_>
2199                };
2200                Ok(normalize_stream(
2201                    stream,
2202                    request,
2203                    selection,
2204                    capabilities.streaming,
2205                    true,
2206                    buffer,
2207                    cancellation,
2208                    self.metrics.clone(),
2209                ))
2210            }
2211            provider_name => {
2212                let provider = self
2213                    .providers
2214                    .get(provider_name)
2215                    .ok_or_else(|| {
2216                        RuntimeError::provider_unavailable(provider_name, "not registered")
2217                    })?
2218                    .clone();
2219                let capabilities = provider.capabilities();
2220                let stream = if capabilities.streaming {
2221                    provider
2222                        .infer_stream(request.clone(), Some(cancellation.clone()))
2223                        .await?
2224                } else {
2225                    self.metrics
2226                        .compatibility_streams
2227                        .fetch_add(1, Ordering::Relaxed);
2228                    let result = provider
2229                        .infer(request.clone(), Some(cancellation.clone()))
2230                        .await?;
2231                    Box::pin(futures_util::stream::iter(vec![Ok(InferenceChunk {
2232                        output: result.output,
2233                        done: true,
2234                        metadata: Some(result.metadata),
2235                    })])) as BoxStream<_>
2236                };
2237                Ok(normalize_stream(
2238                    stream,
2239                    request,
2240                    selection,
2241                    capabilities.streaming,
2242                    false,
2243                    buffer,
2244                    cancellation,
2245                    self.metrics.clone(),
2246                ))
2247            }
2248        }
2249    }
2250
2251    fn validate_request(&self, request: &InferenceRequest) -> RuntimeResult<()> {
2252        if request.options.require_local && request.options.require_remote {
2253            return Err(RuntimeError::InvalidRequest {
2254                reason: "request cannot require both local and remote execution".to_owned(),
2255            });
2256        }
2257        if let Some(limit) = self.config.max_batch_size {
2258            let batch_size = request
2259                .options
2260                .batch_size
2261                .unwrap_or_else(|| request.input.batch_size());
2262            if batch_size > limit {
2263                return Err(RuntimeError::ResourceLimit {
2264                    reason: format!("batch size {batch_size} exceeds configured limit {limit}"),
2265                });
2266            }
2267        }
2268        Ok(())
2269    }
2270
2271    async fn resolve_catalog_reference_for_local(
2272        &self,
2273        request: &mut InferenceRequest,
2274    ) -> RuntimeResult<()> {
2275        let local_resolution = request.options.require_local
2276            || (!request.options.require_remote
2277                && matches!(
2278                    request.options.execution,
2279                    ExecutionPolicy::LocalOnly
2280                        | ExecutionPolicy::PreferLocal
2281                        | ExecutionPolicy::LocalThenRemote
2282                ));
2283        if local_resolution && !self.is_loaded(&request.model) {
2284            if let (Some(catalog), ModelReference::Id { .. }) = (&self.catalog, &request.model) {
2285                let descriptor = catalog
2286                    .resolve(&request.model)
2287                    .await
2288                    .map_err(catalog_error)?;
2289                catalog
2290                    .validate(&descriptor.id)
2291                    .await
2292                    .map_err(catalog_error)?;
2293                request.model = ModelReference::Spec(descriptor.spec());
2294            }
2295        }
2296        Ok(())
2297    }
2298
2299    async fn resolve_loaded_model(&self, reference: &ModelReference) -> RuntimeResult<LoadedModel> {
2300        match reference {
2301            ModelReference::Id { id, version } => self.find_loaded(id, version.as_ref()),
2302            ModelReference::Spec(spec) => {
2303                if let Ok(existing) = self.find_loaded(&spec.id, spec.version.as_ref()) {
2304                    return Ok(existing);
2305                }
2306                self.load(spec.clone()).await?;
2307                self.find_loaded(&spec.id, spec.version.as_ref())
2308            }
2309        }
2310    }
2311
2312    fn find_loaded(&self, id: &str, version: Option<&String>) -> RuntimeResult<LoadedModel> {
2313        let mut cache = self
2314            .loaded_models
2315            .write()
2316            .expect("loaded model registry poisoned");
2317        let found = cache.iter_mut().find(|(key, loaded)| {
2318            key.id == id
2319                && version.is_none_or(|version| loaded.spec.version.as_ref() == Some(version))
2320        });
2321        if let Some((_, loaded)) = found {
2322            loaded.last_used = self.cache_clock.fetch_add(1, Ordering::Relaxed);
2323            return Ok(loaded.clone());
2324        }
2325
2326        Err(RuntimeError::ModelNotFound {
2327            model: id.to_owned(),
2328        })
2329    }
2330
2331    fn is_loaded(&self, reference: &ModelReference) -> bool {
2332        let (id, version) = model_details(reference);
2333        self.loaded_models
2334            .read()
2335            .expect("loaded model registry poisoned")
2336            .keys()
2337            .any(|key| {
2338                key.id == id
2339                    && version
2340                        .as_ref()
2341                        .is_none_or(|v| key.version.as_ref() == Some(v))
2342            })
2343    }
2344
2345    fn select_for_request(&self, request: &InferenceRequest) -> RuntimeResult<RuntimeSelection> {
2346        let model_id = model_id(&request.model);
2347        let policy = if request.options.require_remote {
2348            ExecutionPolicy::RemoteOnly
2349        } else if request.options.require_local {
2350            ExecutionPolicy::LocalOnly
2351        } else if self.config.allow_remote_fallback {
2352            ExecutionPolicy::LocalThenRemote
2353        } else {
2354            request.options.execution.clone()
2355        };
2356        match policy {
2357            ExecutionPolicy::RemoteOnly => self.remote_selection(
2358                model_id,
2359                false,
2360                Some("remote-only policy".to_owned()),
2361                &request.options,
2362            ),
2363            ExecutionPolicy::LocalOnly => self.local_selection(&request.model, policy),
2364            ExecutionPolicy::PreferLocal => match self
2365                .local_selection(&request.model, policy.clone())
2366            {
2367                Ok(selection) => Ok(selection),
2368                Err(error) => {
2369                    self.remote_selection(model_id, true, Some(error.to_string()), &request.options)
2370                }
2371            },
2372            ExecutionPolicy::LocalThenRemote => match self
2373                .local_selection(&request.model, policy.clone())
2374            {
2375                Ok(selection) => Ok(selection),
2376                Err(error) => {
2377                    self.remote_selection(model_id, true, Some(error.to_string()), &request.options)
2378                }
2379            },
2380            ExecutionPolicy::PreferRemote | ExecutionPolicy::RemoteThenLocal => match self
2381                .remote_selection(
2382                    model_id.clone(),
2383                    false,
2384                    Some("preferred remote execution".to_owned()),
2385                    &request.options,
2386                ) {
2387                Ok(selection) => Ok(selection),
2388                Err(_) => self.local_selection(&request.model, policy),
2389            },
2390        }
2391    }
2392
2393    fn local_selection(
2394        &self,
2395        reference: &ModelReference,
2396        policy: ExecutionPolicy,
2397    ) -> RuntimeResult<RuntimeSelection> {
2398        match reference {
2399            ModelReference::Id { id, version } => {
2400                let loaded = self.find_loaded(id, version.as_ref())?;
2401                Ok(RuntimeSelection {
2402                    model: loaded.spec.id,
2403                    provider: "local".to_owned(),
2404                    backend: Some(loaded.backend_name.clone()),
2405                    hardware: self
2406                        .backends
2407                        .get(&loaded.backend_name)
2408                        .and_then(|backend| backend.capabilities().hardware),
2409                    fallback: false,
2410                    fallback_from: None,
2411                    reason: "preferred local execution".to_owned(),
2412                    policy,
2413                })
2414            }
2415            ModelReference::Spec(spec) => {
2416                let backend = self.select_backend(spec)?;
2417                let hardware = self
2418                    .backends
2419                    .get(&backend)
2420                    .and_then(|b| b.capabilities().hardware);
2421                Ok(RuntimeSelection {
2422                    model: spec.id.clone(),
2423                    provider: "local".to_owned(),
2424                    backend: Some(backend),
2425                    hardware,
2426                    fallback: false,
2427                    fallback_from: None,
2428                    reason: "preferred local execution".to_owned(),
2429                    policy,
2430                })
2431            }
2432        }
2433    }
2434
2435    fn remote_selection(
2436        &self,
2437        model: String,
2438        fallback: bool,
2439        reason: Option<String>,
2440        options: &InferenceOptions,
2441    ) -> RuntimeResult<RuntimeSelection> {
2442        let selected = if let Some(provider_name) = self.config.selected_provider.as_deref() {
2443            if provider_name != "local" {
2444                Some(provider_name.to_owned())
2445            } else {
2446                None
2447            }
2448        } else {
2449            None
2450        }
2451        .or_else(|| {
2452            self.providers.iter().find_map(|(name, provider)| {
2453                let capabilities = provider.capabilities();
2454                if capabilities.available && capabilities.remote {
2455                    Some(name.clone())
2456                } else {
2457                    None
2458                }
2459            })
2460        })
2461        .ok_or_else(|| {
2462            RuntimeError::provider_unavailable("remote", "no remote provider registered")
2463        })?;
2464
2465        Ok(RuntimeSelection {
2466            model,
2467            provider: selected,
2468            backend: None,
2469            hardware: None,
2470            fallback,
2471            fallback_from: fallback.then(|| "local".to_owned()),
2472            reason: reason.unwrap_or_else(|| "remote execution selected by policy".to_owned()),
2473            policy: if fallback && options.execution == ExecutionPolicy::LocalOnly {
2474                ExecutionPolicy::LocalThenRemote
2475            } else {
2476                options.execution.clone()
2477            },
2478        })
2479    }
2480
2481    fn can_fallback(
2482        &self,
2483        request: &InferenceRequest,
2484        selection: &RuntimeSelection,
2485        error: &RuntimeError,
2486    ) -> bool {
2487        matches!(
2488            selection.policy,
2489            ExecutionPolicy::LocalThenRemote | ExecutionPolicy::PreferLocal
2490        ) && !request.options.require_local
2491            && matches!(
2492                error,
2493                RuntimeError::ProviderUnavailable { .. }
2494                    | RuntimeError::ModelUnavailable { .. }
2495                    | RuntimeError::BackendUnavailable { .. }
2496                    | RuntimeError::ModelNotFound { .. }
2497                    | RuntimeError::CapabilityMismatch { .. }
2498                    | RuntimeError::Timeout { .. }
2499                    | RuntimeError::Transport { .. }
2500                    | RuntimeError::Protocol { .. }
2501            )
2502    }
2503
2504    fn select_backend(&self, model: &ModelSpec) -> RuntimeResult<String> {
2505        if let Some(selected) = self.config.selected_backend.as_deref() {
2506            let backend = self
2507                .backends
2508                .get(selected)
2509                .ok_or_else(|| RuntimeError::backend_unavailable(selected, "not registered"))?;
2510            let capabilities = backend.capabilities();
2511            if !capabilities.available {
2512                return Err(RuntimeError::backend_unavailable(
2513                    selected,
2514                    "backend is not available on this platform",
2515                ));
2516            }
2517            if !backend.supports(model) {
2518                return Err(RuntimeError::UnsupportedModel {
2519                    model: model.id.clone(),
2520                    backend: selected.to_owned(),
2521                    reason: "model format is not supported".to_owned(),
2522                });
2523            }
2524            return Ok(selected.to_owned());
2525        }
2526
2527        let mut candidates: Vec<_> = self
2528            .backends
2529            .iter()
2530            .filter_map(|(name, backend)| {
2531                let capabilities = backend.capabilities();
2532                if capabilities.available && backend.supports(model) {
2533                    Some((
2534                        name.clone(),
2535                        !capabilities.accelerators.is_empty(),
2536                        capabilities.hardware.clone(),
2537                    ))
2538                } else {
2539                    None
2540                }
2541            })
2542            .collect();
2543
2544        candidates.sort_by(|left, right| left.0.cmp(&right.0));
2545        if self.config.prefer_acceleration {
2546            candidates
2547                .sort_by(|left, right| right.1.cmp(&left.1).then_with(|| left.0.cmp(&right.0)));
2548        }
2549
2550        candidates
2551            .into_iter()
2552            .next()
2553            .map(|candidate| candidate.0)
2554            .ok_or_else(|| {
2555                RuntimeError::backend_unavailable("local", "no compatible backend registered")
2556            })
2557    }
2558}
2559
2560async fn run_with_timeout<F, T>(
2561    timeout_ms: Option<u64>,
2562    operation: &str,
2563    future: F,
2564) -> RuntimeResult<T>
2565where
2566    F: std::future::Future<Output = RuntimeResult<T>>,
2567{
2568    if let Some(timeout_ms) = timeout_ms {
2569        timeout(Duration::from_millis(timeout_ms), future)
2570            .await
2571            .map_err(|_| RuntimeError::Timeout {
2572                operation: operation.to_owned(),
2573            })?
2574    } else {
2575        future.await
2576    }
2577}
2578
2579fn model_id(reference: &ModelReference) -> String {
2580    match reference {
2581        ModelReference::Id { id, .. } => id.clone(),
2582        ModelReference::Spec(spec) => spec.id.clone(),
2583    }
2584}
2585
2586fn runtime_handle(loaded: &LoadedModel) -> RuntimeModelHandle {
2587    RuntimeModelHandle {
2588        model_id: loaded.spec.id.clone(),
2589        version: loaded.spec.version.clone(),
2590        provider: loaded.provider_name.clone(),
2591        backend: loaded.backend_name.clone(),
2592        state: ModelLifecycleState::Ready,
2593    }
2594}
2595
2596fn model_details(reference: &ModelReference) -> (String, Option<String>) {
2597    match reference {
2598        ModelReference::Id { id, version } => (id.clone(), version.clone()),
2599        ModelReference::Spec(spec) => (spec.id.clone(), spec.version.clone()),
2600    }
2601}
2602
2603fn catalog_error(error: CatalogError) -> RuntimeError {
2604    match error {
2605        CatalogError::ModelNotFound(model) => RuntimeError::ModelNotFound { model },
2606        CatalogError::InvalidIdentity(model) => RuntimeError::InvalidRequest {
2607            reason: format!("invalid model identity: {model}"),
2608        },
2609        CatalogError::ArtifactIntegrity { model, reason } => {
2610            RuntimeError::ModelIntegrity { model, reason }
2611        }
2612        other => RuntimeError::Execution {
2613            operation: "catalog".to_owned(),
2614            reason: other.to_string(),
2615        },
2616    }
2617}
2618
2619fn estimate_input_tokens(input: &Input) -> Option<u64> {
2620    match input {
2621        Input::Text(text) => Some(text.split_whitespace().count() as u64),
2622        Input::Tokens(tokens) => Some(tokens.len() as u64),
2623        _ => None,
2624    }
2625}
2626
2627fn estimate_output_tokens(output: &Output) -> Option<u64> {
2628    match output {
2629        Output::Text(text) => Some(text.split_whitespace().count() as u64),
2630        Output::Tokens(tokens) => Some(tokens.len() as u64),
2631        _ => None,
2632    }
2633}
2634
2635fn model_memory_bytes(model: &ModelSpec) -> u64 {
2636    match &model.location {
2637        ModelLocation::Path(path) => std::fs::metadata(path)
2638            .map(|metadata| metadata.len())
2639            .unwrap_or(0),
2640        _ => 0,
2641    }
2642}
2643
2644#[allow(clippy::too_many_arguments)]
2645fn decorate_metadata(
2646    metadata: &mut ExecutionMetadata,
2647    output: &Output,
2648    model: &str,
2649    provider: &str,
2650    backend: &str,
2651    hardware: Option<String>,
2652    elapsed: Duration,
2653    input_tokens: Option<u64>,
2654    model_version: Option<String>,
2655    routing_policy: ExecutionPolicy,
2656    fallback: bool,
2657    fallback_from: Option<String>,
2658    fallback_reason: Option<String>,
2659) {
2660    metadata.model = model.to_owned();
2661    metadata.model_version = model_version;
2662    metadata.provider = provider.to_owned();
2663    metadata.backend = backend.to_owned();
2664    metadata.execution_target = provider.to_owned();
2665    metadata.routing_policy = routing_policy;
2666    metadata.fallback = fallback;
2667    metadata.fallback_from = fallback_from;
2668    metadata.fallback_reason = fallback_reason;
2669    metadata.hardware = hardware;
2670    metadata.latency_ms = Some(elapsed.as_secs_f64() * 1000.0);
2671    metadata.execution_ms = metadata.latency_ms;
2672    metadata.input_tokens = input_tokens;
2673    metadata.output_tokens = estimate_output_tokens(output);
2674}
2675
2676#[allow(clippy::too_many_arguments)]
2677fn normalize_stream(
2678    mut source: BoxStream<RuntimeResult<InferenceChunk>>,
2679    request: InferenceRequest,
2680    selection: RuntimeSelection,
2681    streaming_supported: bool,
2682    cache_hit: bool,
2683    buffer: usize,
2684    cancellation: CancellationToken,
2685    metrics: Arc<RuntimeMetrics>,
2686) -> InferenceStream {
2687    let (sender, receiver) = mpsc::channel(buffer);
2688    tokio::spawn(async move {
2689        let (model, version) = model_details(&request.model);
2690        let mut metadata = ExecutionMetadata {
2691            model,
2692            model_version: version,
2693            provider: selection.provider.clone(),
2694            execution_target: selection.provider.clone(),
2695            routing_policy: selection.policy.clone(),
2696            fallback: selection.fallback,
2697            fallback_from: selection.fallback_from.clone(),
2698            fallback_reason: Some(selection.reason.clone()),
2699            streaming_requested: true,
2700            streaming_supported,
2701            cache_hit,
2702            completion_state: Some("started".to_owned()),
2703            ..ExecutionMetadata::default()
2704        };
2705        if sender
2706            .send(Ok(InferenceStreamEvent::Started(metadata.clone())))
2707            .await
2708            .is_err()
2709        {
2710            return;
2711        }
2712
2713        let mut output_count = 0;
2714        while let Some(item) = tokio::select! {
2715            item = source.next() => item,
2716            _ = cancellation.cancelled() => {
2717                metrics.cancelled_streams.fetch_add(1, Ordering::Relaxed);
2718                let _ = sender.send(Err(RuntimeError::Cancelled)).await;
2719                return;
2720            }
2721        } {
2722            match item {
2723                Ok(chunk) => {
2724                    if let Some(chunk_metadata) = chunk.metadata {
2725                        metadata = chunk_metadata;
2726                    }
2727                    metadata.streaming_requested = true;
2728                    metadata.streaming_supported = streaming_supported;
2729                    metadata.cache_hit = cache_hit;
2730                    metadata.output_event_count = output_count + 1;
2731                    output_count += 1;
2732                    if sender
2733                        .send(Ok(InferenceStreamEvent::Output(chunk.output)))
2734                        .await
2735                        .is_err()
2736                    {
2737                        return;
2738                    }
2739                    metrics
2740                        .streamed_output_events
2741                        .fetch_add(1, Ordering::Relaxed);
2742                    if chunk.done {
2743                        metadata.output_event_count = output_count;
2744                        metadata.completion_state = Some("completed".to_owned());
2745                        let _ = sender
2746                            .send(Ok(InferenceStreamEvent::Completed(metadata)))
2747                            .await;
2748                        return;
2749                    }
2750                }
2751                Err(error) => {
2752                    let _ = sender.send(Err(error)).await;
2753                    return;
2754                }
2755            }
2756        }
2757        metadata.output_event_count = output_count;
2758        metadata.completion_state = Some("completed".to_owned());
2759        let _ = sender
2760            .send(Ok(InferenceStreamEvent::Completed(metadata)))
2761            .await;
2762    });
2763    Box::pin(ReceiverStream::new(receiver))
2764}
2765
2766#[cfg(test)]
2767mod tests {
2768    use super::*;
2769    use async_trait::async_trait;
2770    use futures_util::StreamExt;
2771    use ml_runtime_backend::BackendCapabilities;
2772    use ml_runtime_common::BoxStream;
2773    use ml_runtime_cpu_backend::CpuBackend;
2774    use ml_runtime_inference::InferenceOptions;
2775    use ml_runtime_provider::{Provider, ProviderCapabilities};
2776    use serde_json::json;
2777    use tokio_stream::iter;
2778
2779    #[derive(Clone)]
2780    struct BlockingBackend {
2781        started: Arc<Semaphore>,
2782    }
2783
2784    #[async_trait]
2785    impl Backend for BlockingBackend {
2786        fn name(&self) -> &str {
2787            "blocking"
2788        }
2789
2790        fn capabilities(&self) -> BackendCapabilities {
2791            BackendCapabilities {
2792                available: true,
2793                local: true,
2794                streaming: true,
2795                cancellation: true,
2796                supported_formats: vec![ModelFormat::Unknown],
2797                ..BackendCapabilities::default()
2798            }
2799        }
2800
2801        fn supports(&self, model: &ModelSpec) -> bool {
2802            model.format == ModelFormat::Unknown
2803        }
2804
2805        async fn load(&self, model: &ModelSpec) -> RuntimeResult<ModelHandle> {
2806            Ok(ModelHandle::new(
2807                model.id.clone(),
2808                self.name(),
2809                model.format.clone(),
2810                Arc::new(()),
2811            ))
2812        }
2813
2814        async fn infer(
2815            &self,
2816            _model: &ModelHandle,
2817            _request: &InferenceRequest,
2818            cancellation: Option<CancellationToken>,
2819        ) -> RuntimeResult<InferenceResult> {
2820            self.started.add_permits(1);
2821            cancellation
2822                .expect("runtime supplies cancellation")
2823                .cancelled()
2824                .await;
2825            Err(RuntimeError::Cancelled)
2826        }
2827
2828        async fn infer_stream(
2829            &self,
2830            _model: &ModelHandle,
2831            _request: &InferenceRequest,
2832            _cancellation: Option<CancellationToken>,
2833        ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
2834            Ok(Box::pin(futures_util::stream::pending()))
2835        }
2836    }
2837
2838    #[test]
2839    fn runtime_version_is_workspace_version() {
2840        assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
2841    }
2842
2843    #[derive(Clone)]
2844    struct MockRemoteProvider;
2845
2846    #[async_trait]
2847    impl Provider for MockRemoteProvider {
2848        fn name(&self) -> &str {
2849            "mock-remote"
2850        }
2851
2852        fn capabilities(&self) -> ProviderCapabilities {
2853            ProviderCapabilities {
2854                available: true,
2855                local: false,
2856                remote: true,
2857                streaming: true,
2858                cancellation: true,
2859                structured_output: true,
2860                batching: true,
2861                endpoint: Some("mock://remote".to_owned()),
2862                notes: vec!["test remote provider".to_owned()],
2863            }
2864        }
2865
2866        async fn infer(
2867            &self,
2868            request: InferenceRequest,
2869            cancellation: Option<CancellationToken>,
2870        ) -> RuntimeResult<InferenceResult> {
2871            if model_id(&request.model) == "blocking-remote" {
2872                cancellation
2873                    .expect("runtime supplies cancellation")
2874                    .cancelled()
2875                    .await;
2876                return Err(RuntimeError::Cancelled);
2877            }
2878            if cancellation
2879                .as_ref()
2880                .is_some_and(CancellationToken::is_cancelled)
2881            {
2882                return Err(RuntimeError::Cancelled);
2883            }
2884            Ok(InferenceResult {
2885                output: Output::Structured(json!({"remote": model_id(&request.model)})),
2886                metadata: ExecutionMetadata {
2887                    backend: "remote-api".to_owned(),
2888                    hardware: Some("network".to_owned()),
2889                    ..ExecutionMetadata::default()
2890                },
2891            })
2892        }
2893
2894        async fn infer_stream(
2895            &self,
2896            request: InferenceRequest,
2897            _cancellation: Option<CancellationToken>,
2898        ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>> {
2899            let stream = iter(vec![Ok(InferenceChunk {
2900                output: Output::Text(model_id(&request.model)),
2901                done: true,
2902                metadata: None,
2903            })]);
2904            Ok(Box::pin(stream))
2905        }
2906    }
2907
2908    #[tokio::test]
2909    async fn loads_and_runs_local_cpu_models() {
2910        let runtime = Runtime::builder()
2911            .register_backend(CpuBackend::default())
2912            .build();
2913        let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
2914        runtime.load(model.clone()).await.unwrap();
2915
2916        let result = runtime
2917            .infer(InferenceRequest {
2918                model: model.into(),
2919                input: Input::Text("hello runtime".to_owned()),
2920                options: InferenceOptions::default(),
2921            })
2922            .await
2923            .unwrap();
2924
2925        assert_eq!(result.output, Output::Text("hello runtime".to_owned()));
2926        assert_eq!(result.metadata.provider, "local");
2927        assert_eq!(result.metadata.backend, "cpu");
2928    }
2929
2930    #[tokio::test]
2931    async fn supports_remote_provider_selection() {
2932        let runtime = Runtime::builder()
2933            .register_backend(CpuBackend::default())
2934            .register_provider(MockRemoteProvider)
2935            .provider("mock-remote")
2936            .build();
2937
2938        let result = runtime
2939            .infer(InferenceRequest {
2940                model: ModelReference::id("remote-model", None),
2941                input: Input::Text("hello".to_owned()),
2942                options: InferenceOptions {
2943                    require_remote: true,
2944                    ..InferenceOptions::default()
2945                },
2946            })
2947            .await
2948            .unwrap();
2949
2950        assert_eq!(result.metadata.provider, "mock-remote");
2951        assert_eq!(result.metadata.backend, "remote-api");
2952    }
2953
2954    #[tokio::test]
2955    async fn only_falls_back_to_remote_when_enabled() {
2956        let request = InferenceRequest {
2957            model: ModelReference::id("missing", None),
2958            input: Input::Text("hello".to_owned()),
2959            options: InferenceOptions::default(),
2960        };
2961
2962        let local_only = Runtime::builder()
2963            .register_backend(CpuBackend::default())
2964            .register_provider(MockRemoteProvider)
2965            .build();
2966        assert!(matches!(
2967            local_only.infer(request.clone()).await,
2968            Err(RuntimeError::ModelNotFound { .. })
2969        ));
2970
2971        let hybrid = Runtime::builder()
2972            .register_backend(CpuBackend::default())
2973            .register_provider(MockRemoteProvider)
2974            .allow_remote_fallback(true)
2975            .build();
2976        let result = hybrid.infer(request).await.unwrap();
2977        assert_eq!(result.metadata.provider, "mock-remote");
2978        assert!(result.metadata.fallback);
2979        assert_eq!(result.metadata.fallback_from.as_deref(), Some("local"));
2980    }
2981
2982    #[tokio::test]
2983    async fn explicit_policies_choose_deterministic_targets() {
2984        let local = Runtime::builder()
2985            .register_backend(CpuBackend::default())
2986            .register_provider(MockRemoteProvider)
2987            .build();
2988        let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
2989        local.load(model.clone()).await.unwrap();
2990
2991        let local_result = local
2992            .infer(InferenceRequest {
2993                model: model.clone().into(),
2994                input: Input::Text("hello".to_owned()),
2995                options: InferenceOptions {
2996                    execution: ExecutionPolicy::PreferLocal,
2997                    ..InferenceOptions::default()
2998                },
2999            })
3000            .await
3001            .unwrap();
3002        assert_eq!(local_result.metadata.provider, "local");
3003
3004        let remote_result = local
3005            .infer(InferenceRequest {
3006                model: ModelReference::id("remote-model", None),
3007                input: Input::Text("hello".to_owned()),
3008                options: InferenceOptions {
3009                    execution: ExecutionPolicy::PreferRemote,
3010                    ..InferenceOptions::default()
3011                },
3012            })
3013            .await
3014            .unwrap();
3015        assert_eq!(remote_result.metadata.provider, "mock-remote");
3016        assert_eq!(
3017            remote_result.metadata.routing_policy,
3018            ExecutionPolicy::PreferRemote
3019        );
3020    }
3021
3022    #[tokio::test]
3023    async fn rejects_requests_above_batch_limit() {
3024        let runtime = Runtime::builder()
3025            .register_backend(CpuBackend::default())
3026            .max_batch_size(2)
3027            .build();
3028
3029        let err = runtime
3030            .infer(InferenceRequest {
3031                model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3032                input: Input::Tokens(vec![1, 2, 3]),
3033                options: InferenceOptions {
3034                    batch_size: Some(3),
3035                    ..InferenceOptions::default()
3036                },
3037            })
3038            .await
3039            .unwrap_err();
3040
3041        assert!(matches!(err, RuntimeError::ResourceLimit { .. }));
3042    }
3043
3044    #[tokio::test]
3045    async fn propagates_cancellation() {
3046        let runtime = Runtime::builder()
3047            .register_backend(CpuBackend::default())
3048            .build();
3049        let token = CancellationToken::new();
3050        token.cancel();
3051
3052        let err = runtime
3053            .infer_with_cancellation(
3054                InferenceRequest {
3055                    model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory)
3056                        .into(),
3057                    input: Input::Text("hello".to_owned()),
3058                    options: InferenceOptions::default(),
3059                },
3060                token,
3061            )
3062            .await
3063            .unwrap_err();
3064
3065        assert!(matches!(err, RuntimeError::Cancelled));
3066    }
3067
3068    #[tokio::test]
3069    async fn cancels_active_and_queued_inference() {
3070        let started = Arc::new(Semaphore::new(0));
3071        let runtime = Arc::new(
3072            Runtime::builder()
3073                .register_backend(BlockingBackend {
3074                    started: Arc::clone(&started),
3075                })
3076                .backend("blocking")
3077                .concurrency(1)
3078                .build(),
3079        );
3080        let make_request = || {
3081            InferenceRequest::new(
3082                ModelSpec::new("blocking", ModelFormat::Unknown, ModelLocation::Memory),
3083                "input",
3084            )
3085        };
3086
3087        let active_token = CancellationToken::new();
3088        let active = {
3089            let runtime = Arc::clone(&runtime);
3090            let token = active_token.clone();
3091            tokio::spawn(
3092                async move { runtime.infer_with_cancellation(make_request(), token).await },
3093            )
3094        };
3095        let permit = started.acquire().await.unwrap();
3096        permit.forget();
3097
3098        let queued_token = CancellationToken::new();
3099        let queued = {
3100            let runtime = Arc::clone(&runtime);
3101            let token = queued_token.clone();
3102            tokio::spawn(
3103                async move { runtime.infer_with_cancellation(make_request(), token).await },
3104            )
3105        };
3106        tokio::task::yield_now().await;
3107        queued_token.cancel();
3108        assert!(matches!(
3109            queued.await.unwrap(),
3110            Err(RuntimeError::Cancelled)
3111        ));
3112
3113        active_token.cancel();
3114        assert!(matches!(
3115            active.await.unwrap(),
3116            Err(RuntimeError::Cancelled)
3117        ));
3118    }
3119
3120    #[tokio::test]
3121    async fn cancels_an_active_stream() {
3122        let runtime = Runtime::builder()
3123            .register_backend(BlockingBackend {
3124                started: Arc::new(Semaphore::new(0)),
3125            })
3126            .backend("blocking")
3127            .build();
3128        let token = CancellationToken::new();
3129        let mut stream = runtime
3130            .infer_stream_with_cancellation(
3131                InferenceRequest::new(
3132                    ModelSpec::new("blocking", ModelFormat::Unknown, ModelLocation::Memory),
3133                    "input",
3134                )
3135                .with_options(InferenceOptions {
3136                    stream: true,
3137                    ..InferenceOptions::default()
3138                }),
3139                token.clone(),
3140            )
3141            .await
3142            .unwrap();
3143        assert!(matches!(
3144            stream.next().await.unwrap().unwrap(),
3145            InferenceStreamEvent::Started(_)
3146        ));
3147        token.cancel();
3148        assert!(matches!(
3149            stream.next().await.unwrap(),
3150            Err(RuntimeError::Cancelled)
3151        ));
3152    }
3153
3154    #[tokio::test]
3155    async fn propagates_remote_cancellation() {
3156        let runtime = Arc::new(
3157            Runtime::builder()
3158                .register_provider(MockRemoteProvider)
3159                .provider("mock-remote")
3160                .build(),
3161        );
3162        let token = CancellationToken::new();
3163        let task = {
3164            let runtime = Arc::clone(&runtime);
3165            let token = token.clone();
3166            tokio::spawn(async move {
3167                runtime
3168                    .infer_with_cancellation(
3169                        InferenceRequest::new(ModelReference::latest("blocking-remote"), "input")
3170                            .with_options(InferenceOptions {
3171                                execution: ExecutionPolicy::RemoteOnly,
3172                                ..InferenceOptions::default()
3173                            }),
3174                        token,
3175                    )
3176                    .await
3177            })
3178        };
3179        tokio::task::yield_now().await;
3180        token.cancel();
3181        assert!(matches!(task.await.unwrap(), Err(RuntimeError::Cancelled)));
3182    }
3183
3184    #[tokio::test]
3185    async fn streams_from_local_backend() {
3186        let runtime = Runtime::builder()
3187            .register_backend(CpuBackend::default())
3188            .build();
3189
3190        let mut stream = runtime
3191            .infer_stream(InferenceRequest {
3192                model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3193                input: Input::Text("hello runtime".to_owned()),
3194                options: InferenceOptions {
3195                    stream: true,
3196                    ..InferenceOptions::default()
3197                },
3198            })
3199            .await
3200            .unwrap();
3201
3202        assert!(matches!(
3203            stream.next().await.unwrap().unwrap(),
3204            InferenceStreamEvent::Started(_)
3205        ));
3206        assert!(matches!(
3207            stream.next().await.unwrap().unwrap(),
3208            InferenceStreamEvent::Output(Output::Text(text)) if text == "hello"
3209        ));
3210        assert!(matches!(
3211            stream.next().await.unwrap().unwrap(),
3212            InferenceStreamEvent::Output(Output::Text(text)) if text == "runtime"
3213        ));
3214        assert!(matches!(
3215            stream.next().await.unwrap().unwrap(),
3216            InferenceStreamEvent::Completed(metadata)
3217                if metadata.output_event_count == 2 && metadata.streaming_supported
3218        ));
3219    }
3220
3221    #[tokio::test]
3222    async fn rejects_stream_calls_without_stream_flag() {
3223        let runtime = Runtime::builder()
3224            .register_backend(CpuBackend::default())
3225            .build();
3226        let result = runtime
3227            .infer_stream(InferenceRequest {
3228                model: ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory).into(),
3229                input: Input::Text("hello".to_owned()),
3230                options: InferenceOptions::default(),
3231            })
3232            .await;
3233
3234        assert!(matches!(
3235            result,
3236            Err(RuntimeError::CapabilityMismatch { .. })
3237        ));
3238    }
3239
3240    #[tokio::test]
3241    async fn streams_from_remote_provider() {
3242        let runtime = Runtime::builder()
3243            .register_backend(CpuBackend::default())
3244            .register_provider(MockRemoteProvider)
3245            .provider("mock-remote")
3246            .build();
3247
3248        let mut stream = runtime
3249            .infer_stream(InferenceRequest {
3250                model: ModelReference::id("remote-model", None),
3251                input: Input::Text("hello".to_owned()),
3252                options: InferenceOptions {
3253                    stream: true,
3254                    require_remote: true,
3255                    ..InferenceOptions::default()
3256                },
3257            })
3258            .await
3259            .unwrap();
3260
3261        assert!(matches!(
3262            stream.next().await.unwrap().unwrap(),
3263            InferenceStreamEvent::Started(_)
3264        ));
3265        assert!(matches!(
3266            stream.next().await.unwrap().unwrap(),
3267            InferenceStreamEvent::Output(Output::Text(text)) if text == "remote-model"
3268        ));
3269        assert!(matches!(
3270            stream.next().await.unwrap().unwrap(),
3271            InferenceStreamEvent::Completed(metadata)
3272                if metadata.output_event_count == 1
3273        ));
3274    }
3275
3276    #[tokio::test]
3277    async fn explicit_handles_reuse_loaded_models_and_report_cache_hits() {
3278        let runtime = Runtime::builder()
3279            .register_backend(CpuBackend::default())
3280            .build();
3281        let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
3282        let handle = runtime.load(model).await.unwrap();
3283        assert_eq!(handle.id(), "echo");
3284        let result = runtime
3285            .infer_loaded(
3286                &handle,
3287                Input::Text("cached".to_owned()),
3288                Default::default(),
3289            )
3290            .await
3291            .unwrap();
3292        assert!(result.metadata.cache_hit);
3293        assert_eq!(runtime.models().list().len(), 1);
3294    }
3295
3296    #[tokio::test]
3297    async fn loaded_stream_reuses_the_cached_model() {
3298        let runtime = Runtime::builder()
3299            .register_backend(CpuBackend::default())
3300            .build();
3301        let handle = runtime
3302            .load(ModelSpec::new(
3303                "echo",
3304                ModelFormat::Unknown,
3305                ModelLocation::Memory,
3306            ))
3307            .await
3308            .unwrap();
3309        let mut stream = runtime
3310            .infer_loaded_stream(
3311                &handle,
3312                Input::Text("cached stream".to_owned()),
3313                InferenceOptions::default(),
3314            )
3315            .await
3316            .unwrap();
3317        let events = futures_util::StreamExt::collect::<Vec<_>>(&mut stream).await;
3318        assert!(matches!(
3319            events.first().and_then(|event| event.as_ref().ok()),
3320            Some(InferenceStreamEvent::Started(_))
3321        ));
3322        assert!(matches!(
3323            events.last().and_then(|event| event.as_ref().ok()),
3324            Some(InferenceStreamEvent::Completed(metadata)) if metadata.cache_hit
3325        ));
3326        assert_eq!(runtime.models().list().len(), 1);
3327    }
3328
3329    #[tokio::test]
3330    async fn cache_evicts_the_least_recently_used_target() {
3331        let runtime = Runtime::builder()
3332            .register_backend(CpuBackend::default())
3333            .max_models(1)
3334            .build();
3335        runtime
3336            .load(ModelSpec::new(
3337                "first",
3338                ModelFormat::Unknown,
3339                ModelLocation::Memory,
3340            ))
3341            .await
3342            .unwrap();
3343        runtime
3344            .load(ModelSpec::new(
3345                "second",
3346                ModelFormat::Unknown,
3347                ModelLocation::Memory,
3348            ))
3349            .await
3350            .unwrap();
3351        assert_eq!(runtime.models().list().len(), 1);
3352        assert_eq!(runtime.models().list()[0].id, "second");
3353    }
3354
3355    #[tokio::test]
3356    async fn batch_inference_preserves_result_order_and_identity() {
3357        let runtime = Runtime::builder()
3358            .register_backend(CpuBackend::default())
3359            .build();
3360        let model = ModelSpec::new("echo", ModelFormat::Unknown, ModelLocation::Memory);
3361        let results = runtime
3362            .infer_batch(vec![
3363                InferenceRequest {
3364                    model: model.clone().into(),
3365                    input: Input::Text("a".to_owned()),
3366                    options: Default::default(),
3367                },
3368                InferenceRequest {
3369                    model: model.into(),
3370                    input: Input::Text("b".to_owned()),
3371                    options: Default::default(),
3372                },
3373            ])
3374            .await
3375            .unwrap();
3376        assert_eq!(results.len(), 2);
3377        assert_eq!(results[0].output, Output::Text("a".to_owned()));
3378        assert_eq!(results[1].output, Output::Text("b".to_owned()));
3379        assert_eq!(results[0].metadata.batch_size, 2);
3380        assert_eq!(results[1].metadata.batch_size, 2);
3381    }
3382
3383    #[tokio::test]
3384    async fn records_execution_metrics_and_snapshot() {
3385        let runtime = Runtime::builder()
3386            .register_backend(CpuBackend::default())
3387            .build();
3388        let model = ModelSpec::new("observed", ModelFormat::Unknown, ModelLocation::Memory);
3389        let result = runtime
3390            .infer(InferenceRequest {
3391                model: model.into(),
3392                input: Input::Text("hello".to_owned()),
3393                options: Default::default(),
3394            })
3395            .await
3396            .unwrap();
3397
3398        let metrics = runtime.metrics();
3399        assert_eq!(metrics.total_requests, 1);
3400        assert_eq!(metrics.successful_requests, 1);
3401        assert_eq!(runtime.executions().len(), 1);
3402        assert_eq!(
3403            runtime.executions()[0].completion,
3404            ExecutionCompletion::Succeeded
3405        );
3406        assert_eq!(
3407            runtime.executions()[0].request_id,
3408            result.metadata.request_id.clone().unwrap()
3409        );
3410        assert_eq!(runtime.snapshot().loaded_models, 1);
3411    }
3412}