Struct SchedulerQueue 

pub struct SchedulerQueue<P: SequencePublisher, C: WorkerConfigLike, Sel: WorkerSelector<C> = DefaultWorkerSelector, RF: OverlapScoresRefresh = NoopOverlapScoresRefresh> { /* private fields */ }

Queue that gates scheduling requests behind a capacity check. When all workers exceed threshold_frac utilisation the request is parked in pending. When capacity frees up (update()), pending requests are scheduled in priority order. If queueing is disabled (threshold_frac is None), requests are scheduled immediately.

Implementations

impl<P: SequencePublisher + 'static, C: WorkerConfigLike + Send + Sync + 'static, Sel: WorkerSelector<C> + Send + 'static, RF: OverlapScoresRefresh + Send + Sync + 'static> SchedulerQueue<P, C, Sel, RF>

pub fn new_with_overlap_refresh( slots: Arc<ActiveSequencesMultiWorker<P>>, workers_with_configs: Receiver<HashMap<WorkerId, C>>, threshold_frac: Option<f64>, block_size: u32, selector: Sel, queue_policy: RouterQueuePolicy, prefill_load_estimator: Option<Arc<dyn PrefillLoadEstimator>>, overlap_scores_refresh: Option<Arc<RF>>, overloaded_worker_provider: Option<OverloadedWorkerProvider>, ) -> Self

pub fn new_with_policy_profile( slots: Arc<ActiveSequencesMultiWorker<P>>, workers_with_configs: Receiver<HashMap<WorkerId, C>>, profile: PolicyProfile, block_size: u32, selector: Sel, prefill_load_estimator: Option<Arc<dyn PrefillLoadEstimator>>, overlap_scores_refresh: Option<Arc<RF>>, overloaded_worker_provider: Option<OverloadedWorkerProvider>, ) -> Self

impl<P: SequencePublisher + 'static, C: WorkerConfigLike + Send + Sync + 'static, Sel: WorkerSelector<C> + Send + 'static> SchedulerQueue<P, C, Sel, NoopOverlapScoresRefresh>

pub fn new( slots: Arc<ActiveSequencesMultiWorker<P>>, workers_with_configs: Receiver<HashMap<WorkerId, C>>, threshold_frac: Option<f64>, block_size: u32, selector: Sel, queue_policy: RouterQueuePolicy, prefill_load_estimator: Option<Arc<dyn PrefillLoadEstimator>>, ) -> Self

pub fn new_with_overload_provider( slots: Arc<ActiveSequencesMultiWorker<P>>, workers_with_configs: Receiver<HashMap<WorkerId, C>>, threshold_frac: Option<f64>, block_size: u32, selector: Sel, queue_policy: RouterQueuePolicy, prefill_load_estimator: Option<Arc<dyn PrefillLoadEstimator>>, overloaded_worker_provider: Option<OverloadedWorkerProvider>, ) -> Self

impl<P: SequencePublisher + 'static, C: WorkerConfigLike + Send + Sync + 'static, Sel: WorkerSelector<C> + Send + 'static, RF: OverlapScoresRefresh + Send + Sync + 'static> SchedulerQueue<P, C, Sel, RF>

pub fn register_workers(&self, worker_ids: &HashSet<u64>)

Register externally-provided workers in the slot tracker.

Looks up DP rank/size from the discovery watch channel; defaults to (0, 1) for workers not yet known to discovery.

pub async fn enqueue(&self, request: SchedulingRequest)

Enqueue a new request. If queueing is disabled or workers have capacity, schedule immediately. Otherwise park in the pending heap.

pub async fn enqueue_with_block_hashes( &self, request: SchedulingRequest, block_hashes: Option<Vec<LocalBlockHash>>, )

pub async fn update(&self)

Called on prefill_complete/free. Drains pending requests while workers have capacity. Each scheduled request updates active_tokens via add_request, so the prefill-busy check sees fresh state on the next iteration.

pub fn pending_count(&self) -> usize

Number of requests currently parked in the pending queue (lock-free).

pub fn pending_isl_tokens(&self) -> usize

Sum of isl_tokens for requests currently parked in the pending queue (lock-free).

pub fn class_queue_stats(&self, class_index: usize) -> Option<ClassQueueStats>

pub fn supports_overlap_refresh(&self) -> bool