pub struct Runtime { /* private fields */ }Expand description
Coordinates catalog resolution, routing, model lifecycle, and inference.
A runtime is explicit application state; it does not create global state or
background services. Use Runtime::builder to obtain built-in CPU and
ONNX execution.
Implementations§
Source§impl Runtime
impl Runtime
pub fn new() -> Self
pub fn builder() -> RuntimeBuilder
pub fn config(&self) -> &RuntimeConfig
Sourcepub fn load_decision_model(
&self,
artifact: impl AsRef<Path>,
) -> RuntimeResult<Box<dyn DecisionModel>>
pub fn load_decision_model( &self, artifact: impl AsRef<Path>, ) -> RuntimeResult<Box<dyn DecisionModel>>
Loads an immutable local typed-decision artifact through the single matching backend. There is no fallback: missing, ambiguous, or invalid artifacts fail closed.
Sourcepub fn decision_capabilities(
&self,
model: &dyn DecisionModel,
) -> DecisionModelCapabilities
pub fn decision_capabilities( &self, model: &dyn DecisionModel, ) -> DecisionModelCapabilities
Returns model/backend execution facts without requiring applications to inspect artifact layouts or backend names.
Sourcepub fn explain_decision(
&self,
model: &dyn DecisionModel,
request: &DecisionGraphRequest,
) -> RuntimeResult<DecisionExecutionPlan>
pub fn explain_decision( &self, model: &dyn DecisionModel, request: &DecisionGraphRequest, ) -> RuntimeResult<DecisionExecutionPlan>
Builds the deterministic execution plan for a loaded model and graph.
Sourcepub async fn execute_decision_graph(
&self,
model: Arc<dyn DecisionModel>,
request: DecisionGraphRequest,
cancellation: Option<CancellationToken>,
) -> RuntimeResult<PlannedDecisionResult>
pub async fn execute_decision_graph( &self, model: Arc<dyn DecisionModel>, request: DecisionGraphRequest, cancellation: Option<CancellationToken>, ) -> RuntimeResult<PlannedDecisionResult>
Executes a graph asynchronously. Synchronous native model calls are moved off the caller’s async executor thread.
Sourcepub async fn decide_async(
&self,
model: Arc<dyn DecisionModel>,
request: DecisionRequest,
policy: DecisionExecutionPolicy,
cancellation: Option<CancellationToken>,
) -> RuntimeResult<PlannedDecisionResult>
pub async fn decide_async( &self, model: Arc<dyn DecisionModel>, request: DecisionRequest, policy: DecisionExecutionPolicy, cancellation: Option<CancellationToken>, ) -> RuntimeResult<PlannedDecisionResult>
Compatibility-friendly async execution for an ordinary independent
DecisionRequest.
pub fn installed_models(&self) -> RuntimeResult<Vec<InstalledModelRecord>>
pub async fn install_registered_model( &self, name: &str, source_override: Option<&str>, replace: bool, ) -> RuntimeResult<ModelLifecycleResult>
pub fn doctor_registered_model( &self, name: &str, ) -> RuntimeResult<ModelDoctorReport>
pub fn remove_registered_model( &self, name: &str, ) -> RuntimeResult<InstalledModelRecord>
Sourcepub fn load_model(&self, name: &str) -> RuntimeResult<Box<dyn DecisionModel>>
pub fn load_model(&self, name: &str) -> RuntimeResult<Box<dyn DecisionModel>>
Loads a validated installed model by its registry name. Source, artifact layout, tokenizer, and backend selection remain runtime concerns.
pub fn set_catalog<C>(&mut self, catalog: C)where
C: ModelCatalog + 'static,
pub async fn catalog_models(&self) -> RuntimeResult<Vec<ModelDescriptor>>
pub async fn resolve_model( &self, reference: &ModelReference, ) -> RuntimeResult<ModelDescriptor>
pub async fn refresh_catalog(&self) -> RuntimeResult<()>
pub fn models(&self) -> ModelRegistry
pub fn providers(&self) -> ProviderRegistry
pub fn backends(&self) -> BackendRegistry
pub fn capabilities(&self) -> RuntimeCapabilities
pub fn capability(&self, id: &str) -> Option<Capability>
pub fn capability_registry(&self) -> &CapabilityRegistry
Sourcepub fn load_capability_packages(
&mut self,
root: impl AsRef<Path>,
) -> RuntimeResult<Vec<String>>
pub fn load_capability_packages( &mut self, root: impl AsRef<Path>, ) -> RuntimeResult<Vec<String>>
Explicitly discovers and registers local capability package metadata.
Discovery never loads or executes a provider binary.
pub fn environment(&self) -> RuntimeEnvironment
pub fn resolve_capability( &self, request: &ExecutionRequest, ) -> RuntimeResult<CapabilityResolution>
pub fn resolve_capability_with_constraints( &self, request: &ExecutionRequest, constraints: &ResolutionConstraints, ) -> RuntimeResult<CapabilityResolution>
pub async fn execute_capability( &self, request: ExecutionRequest, authorizer: Option<&dyn CapabilityAuthorizer>, ) -> RuntimeResult<ExecutionResult>
pub fn status(&self) -> RuntimeStatus
pub fn metrics(&self) -> RuntimeMetricsSnapshot
pub fn executions(&self) -> Vec<ExecutionRecord>
pub fn resources(&self) -> Vec<ModelResource>
pub fn snapshot(&self) -> RuntimeSnapshot
pub fn selection_for( &self, model: &ModelSpec, ) -> RuntimeResult<RuntimeSelection>
Sourcepub async fn load(&self, model: ModelSpec) -> RuntimeResult<RuntimeModelHandle>
pub async fn load(&self, model: ModelSpec) -> RuntimeResult<RuntimeModelHandle>
Explicitly loads a model for repeated execution with Self::infer_loaded.
Normal one-off requests should use Self::infer, which loads lazily.
pub async fn load_reference( &self, reference: ModelReference, ) -> RuntimeResult<RuntimeModelHandle>
Sourcepub async fn infer(
&self,
request: InferenceRequest,
) -> RuntimeResult<InferenceResult>
pub async fn infer( &self, request: InferenceRequest, ) -> RuntimeResult<InferenceResult>
Resolves, loads, routes, and executes one request.
Sourcepub async fn infer_loaded(
&self,
model: &RuntimeModelHandle,
input: Input,
options: InferenceOptions,
) -> RuntimeResult<InferenceResult>
pub async fn infer_loaded( &self, model: &RuntimeModelHandle, input: Input, options: InferenceOptions, ) -> RuntimeResult<InferenceResult>
Executes against a previously loaded handle without exposing cache internals.
pub async fn infer_loaded_stream( &self, model: &RuntimeModelHandle, input: Input, options: InferenceOptions, ) -> RuntimeResult<InferenceStream>
Sourcepub async fn infer_batch(
&self,
requests: Vec<InferenceRequest>,
) -> RuntimeResult<Vec<InferenceResult>>
pub async fn infer_batch( &self, requests: Vec<InferenceRequest>, ) -> RuntimeResult<Vec<InferenceResult>>
Executes requests in stable input order.
The call fails as a whole on a request error. Backends without native batching are invoked individually behind the same API.
Sourcepub async fn infer_with_cancellation(
&self,
request: InferenceRequest,
cancellation: CancellationToken,
) -> RuntimeResult<InferenceResult>
pub async fn infer_with_cancellation( &self, request: InferenceRequest, cancellation: CancellationToken, ) -> RuntimeResult<InferenceResult>
Executes one request while propagating cancellation through scheduling, provider transport, and backend execution.
Sourcepub async fn infer_stream(
&self,
request: InferenceRequest,
) -> RuntimeResult<InferenceStream>
pub async fn infer_stream( &self, request: InferenceRequest, ) -> RuntimeResult<InferenceStream>
Starts normalized Started → Output* → Completed streaming.
Sourcepub async fn infer_stream_with_cancellation(
&self,
request: InferenceRequest,
cancellation: CancellationToken,
) -> RuntimeResult<InferenceStream>
pub async fn infer_stream_with_cancellation( &self, request: InferenceRequest, cancellation: CancellationToken, ) -> RuntimeResult<InferenceStream>
Starts a bounded stream with caller-controlled cancellation.
Trait Implementations§
Auto Trait Implementations§
impl !Freeze for Runtime
impl !RefUnwindSafe for Runtime
impl !UnwindSafe for Runtime
impl Send for Runtime
impl Sync for Runtime
impl Unpin for Runtime
impl UnsafeUnpin for Runtime
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self> ⓘ
fn instrument(self, span: Span) -> Instrumented<Self> ⓘ
Source§fn in_current_span(self) -> Instrumented<Self> ⓘ
fn in_current_span(self) -> Instrumented<Self> ⓘ
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more