Trait Scheduler 

pub trait Scheduler: Send + Sync {
    // Required methods
    fn submit<'life0, 'async_trait>(
        &'life0 self,
        request: InferenceRequest,
    ) -> Pin<Box<dyn Future<Output = Result<RequestId, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
    fn next_batch<'life0, 'async_trait>(
        &'life0 self,
        hint: BatchHint,
    ) -> Pin<Box<dyn Future<Output = Option<BatchPlan>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
    fn complete<'life0, 'life1, 'async_trait>(
        &'life0 self,
        request_id: RequestId,
        response: &'life1 InferenceResponse,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait;
    fn cancel<'life0, 'async_trait>(
        &'life0 self,
        request_id: RequestId,
    ) -> Pin<Box<dyn Future<Output = Result<bool, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
    fn update_priority<'life0, 'async_trait>(
        &'life0 self,
        request_id: RequestId,
        priority: Priority,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait;
    fn metrics(&self) -> SchedulerStats;
    fn config(&self) -> &SchedulerConfig;

    // Provided methods
    fn request_state(&self, request_id: &RequestId) -> Option<RequestState> { ... }
    fn preempt<'life0, 'async_trait>(
        &'life0 self,
        _request_id: RequestId,
    ) -> Pin<Box<dyn Future<Output = Result<PreemptionResult, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait { ... }
    fn resume<'life0, 'async_trait>(
        &'life0 self,
        _request_id: RequestId,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait { ... }
}

Main scheduler trait for request management and batching

Required Methods

fn submit<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<RequestId, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Submit new inference request

fn next_batch<'life0, 'async_trait>( &'life0 self, hint: BatchHint, ) -> Pin<Box<dyn Future<Output = Option<BatchPlan>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Get next batch of requests to execute

fn complete<'life0, 'life1, 'async_trait>( &'life0 self, request_id: RequestId, response: &'life1 InferenceResponse, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Mark request as completed

fn cancel<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<bool, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Cancel pending request

fn update_priority<'life0, 'async_trait>( &'life0 self, request_id: RequestId, priority: Priority, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Update request priority

fn metrics(&self) -> SchedulerStats

Get scheduler metrics

fn config(&self) -> &SchedulerConfig

Get scheduler configuration

Provided Methods

fn request_state(&self, request_id: &RequestId) -> Option<RequestState>

Get current request state if the request is tracked by scheduler.

fn preempt<'life0, 'async_trait>( &'life0 self, _request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<PreemptionResult, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Preempt running request (if supported)

fn resume<'life0, 'async_trait>( &'life0 self, _request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Resume preempted request

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementations on Foreign Types

impl Scheduler for ContinuousBatchScheduler

fn submit<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<RequestId, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, ContinuousBatchScheduler: 'async_trait,

fn next_batch<'life0, 'async_trait>( &'life0 self, hint: BatchHint, ) -> Pin<Box<dyn Future<Output = Option<BatchPlan>> + Send + 'async_trait>>

where 'life0: 'async_trait, ContinuousBatchScheduler: 'async_trait,

fn complete<'life0, 'life1, 'async_trait>( &'life0 self, request_id: RequestId, response: &'life1 InferenceResponse, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, ContinuousBatchScheduler: 'async_trait,

fn cancel<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<bool, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, ContinuousBatchScheduler: 'async_trait,

fn update_priority<'life0, 'async_trait>( &'life0 self, request_id: RequestId, priority: Priority, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, ContinuousBatchScheduler: 'async_trait,

fn metrics(&self) -> SchedulerStats

fn config(&self) -> &SchedulerConfig

fn request_state(&self, request_id: &RequestId) -> Option<RequestState>

fn preempt<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<PreemptionResult, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, ContinuousBatchScheduler: 'async_trait,

fn resume<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, ContinuousBatchScheduler: 'async_trait,

impl Scheduler for FifoScheduler

fn submit<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<RequestId, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, FifoScheduler: 'async_trait,

fn next_batch<'life0, 'async_trait>( &'life0 self, hint: BatchHint, ) -> Pin<Box<dyn Future<Output = Option<BatchPlan>> + Send + 'async_trait>>

where 'life0: 'async_trait, FifoScheduler: 'async_trait,

fn complete<'life0, 'life1, 'async_trait>( &'life0 self, request_id: RequestId, response: &'life1 InferenceResponse, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, FifoScheduler: 'async_trait,

fn cancel<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<bool, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, FifoScheduler: 'async_trait,

fn update_priority<'life0, 'async_trait>( &'life0 self, request_id: RequestId, _priority: Priority, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, FifoScheduler: 'async_trait,

fn metrics(&self) -> SchedulerStats

fn config(&self) -> &SchedulerConfig

fn request_state(&self, request_id: &RequestId) -> Option<RequestState>

fn preempt<'life0, 'async_trait>( &'life0 self, _request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<PreemptionResult, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, FifoScheduler: 'async_trait,

fn resume<'life0, 'async_trait>( &'life0 self, _request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, FifoScheduler: 'async_trait,

impl Scheduler for PriorityScheduler

fn submit<'life0, 'async_trait>( &'life0 self, request: InferenceRequest, ) -> Pin<Box<dyn Future<Output = Result<RequestId, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, PriorityScheduler: 'async_trait,

fn next_batch<'life0, 'async_trait>( &'life0 self, hint: BatchHint, ) -> Pin<Box<dyn Future<Output = Option<BatchPlan>> + Send + 'async_trait>>

where 'life0: 'async_trait, PriorityScheduler: 'async_trait,

fn complete<'life0, 'life1, 'async_trait>( &'life0 self, request_id: RequestId, response: &'life1 InferenceResponse, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, PriorityScheduler: 'async_trait,

fn cancel<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<bool, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, PriorityScheduler: 'async_trait,

fn update_priority<'life0, 'async_trait>( &'life0 self, request_id: RequestId, new_priority: Priority, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, PriorityScheduler: 'async_trait,

fn metrics(&self) -> SchedulerStats

fn config(&self) -> &SchedulerConfig

fn request_state(&self, request_id: &RequestId) -> Option<RequestState>

fn preempt<'life0, 'async_trait>( &'life0 self, request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<PreemptionResult, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, PriorityScheduler: 'async_trait,

fn resume<'life0, 'async_trait>( &'life0 self, _request_id: RequestId, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, PriorityScheduler: 'async_trait,

Implementors