skippy-scheduler 0.76.1

Iteration-level scheduler for concurrent staged serving with unified KV
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

#[derive(Debug, Clone, Default, PartialEq)]
pub struct SchedulerMetrics {
    pub iterations: u64,
    pub admitted: u64,
    pub preempted: u64,
    pub finished: u64,
    pub failed: u64,
    pub cancelled: u64,
    pub rejected_overload: u64,
    pub prefill_tokens: u64,
    pub recompute_tokens: u64,
    pub decode_tokens: u64,
    pub prefix_hits: u64,
    pub prefix_misses: u64,
    pub active_sequences: usize,
    pub waiting_sequences: usize,
}

#[derive(Debug, Clone, Default, PartialEq)]
pub struct IterationTelemetry {
    pub iteration: u64,
    pub active_sequences: usize,
    pub waiting_sequences: usize,
    pub admitted: usize,
    pub preempted: usize,
    pub prefill_tokens: usize,
    pub recompute_tokens: usize,
    pub decode_tokens: usize,
    pub component_used_bytes: Vec<(String, u64)>,
    pub component_available_bytes: Vec<(String, u64)>,
    pub prefix_hits: u64,
    pub prefix_misses: u64,
    pub finished: u64,
    pub failed: u64,
    pub cancelled: u64,
    pub rejected_overload: u64,
}