Struct ModelServingManager 

pub struct ModelServingManager { /* private fields */ }

Model serving manager

Implementations

impl ModelServingManager

pub fn new(config: ServingConfig) -> Self

Create a new model serving manager

pub fn with_model<M: Model<Input = Tensor, Output = Tensor> + 'static>( config: ServingConfig, model: M, ) -> Self

Create a new model serving manager with a specific model

pub fn set_inference_fn(&mut self, inference_fn: ModelInferenceFn)

Set a custom inference function

pub fn add_instance(&self, instance: ModelInstance) -> Result<()>

Add a model instance

pub fn get_health_status(&self) -> Result<HashMap<String, CircuitBreakerState>>

Get health status for all instances

pub async fn perform_health_check(&self) -> Result<()>

Perform health check on all instances

pub async fn submit_request(&self, request: InferenceRequest) -> Result<()>

Submit a request for processing

pub async fn process_next_request(&self) -> Result<Option<InferenceResponse>>

Process the next request in the queue

pub async fn get_metrics(&self) -> ServingMetrics

Get current serving metrics

pub async fn cleanup_expired_requests(&self) -> Result<usize>

Cleanup expired requests

pub fn healthy_instances_count(&self) -> Result<usize>

Get healthy instances count

Trait Implementations

impl Debug for ModelServingManager

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.