kapsl-scheduler 0.1.0

Request scheduling and batching for the Kapsl inference engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

use crate::priority::Priority;
use crate::request_metadata::{determine_priority, RequestMetadata};

#[test]
fn priority_override_wins() {
    let meta = RequestMetadata {
        priority: 0,
        sla_deadline: Some(10_000),
        batch_size: 32,
        input_size_bytes: Some(10_000_000),
        estimated_flops: Some(10_000_000_000),
    };

    assert_eq!(determine_priority(&meta), Priority::LatencyCritical);
}

#[test]
fn sla_deadline_triggers_latency() {
    let meta = RequestMetadata {
        priority: 1,
        sla_deadline: Some(200),
        batch_size: 16,
        input_size_bytes: Some(10_000_000),
        estimated_flops: Some(10_000_000_000),
    };

    assert_eq!(determine_priority(&meta), Priority::LatencyCritical);
}

#[test]
fn light_workload_by_input_size() {
    let meta = RequestMetadata {
        priority: 1,
        sla_deadline: None,
        batch_size: 16,
        input_size_bytes: Some(500_000),
        estimated_flops: Some(10_000_000_000),
    };

    assert_eq!(determine_priority(&meta), Priority::LatencyCritical);
}

#[test]
fn light_workload_by_flops() {
    let meta = RequestMetadata {
        priority: 1,
        sla_deadline: None,
        batch_size: 16,
        input_size_bytes: Some(10_000_000),
        estimated_flops: Some(1_000_000_000),
    };

    assert_eq!(determine_priority(&meta), Priority::LatencyCritical);
}

#[test]
fn light_workload_by_combined_size() {
    let meta = RequestMetadata {
        priority: 1,
        sla_deadline: None,
        batch_size: 4,
        input_size_bytes: Some(500_000),
        estimated_flops: Some(10_000_000_000),
    };

    assert_eq!(determine_priority(&meta), Priority::LatencyCritical);
}

#[test]
fn small_batch_triggers_latency() {
    let meta = RequestMetadata {
        priority: 1,
        sla_deadline: None,
        batch_size: 4,
        input_size_bytes: Some(10_000_000),
        estimated_flops: Some(10_000_000_000),
    };

    assert_eq!(determine_priority(&meta), Priority::LatencyCritical);
}

#[test]
fn heavy_workload_defaults_to_throughput() {
    let meta = RequestMetadata {
        priority: 1,
        sla_deadline: None,
        batch_size: 8,
        input_size_bytes: Some(10_000_000),
        estimated_flops: Some(10_000_000_000),
    };

    assert_eq!(determine_priority(&meta), Priority::Throughput);
}