batched-fn 0.2.5

Middleware for serving deep learning models with batched prediction
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

use batched_fn::batched_fn;

fn batch_double(batch: Vec<i32>) -> Vec<i32> {
    let mut out = Vec::with_capacity(batch.len());
    for x in batch {
        out.push(x * 2);
    }
    out
}

fn batch_multiply(batch: Vec<i32>, factor: i32) -> Vec<i32> {
    batch.into_iter().map(|x| x * factor).collect()
}

#[test]
fn test_basic_invocation() {
    let _ = batched_fn! {
        handler = |batch: Vec<i32>| -> Vec<i32> { batch_double(batch) };
        config = {
            max_batch_size: 4,
            max_delay: 50,
        };
        context = {};
    };
}

#[test]
fn test_with_context() {
    let _ = batched_fn! {
        handler = |batch: Vec<i32>, factor: &i32| -> Vec<i32> { batch_multiply(batch, *factor) };
        config = {
            max_batch_size: 4,
            max_delay: 50,
        };
        context = {
            factor: 3,
        };
    };
}