ai-batch-queue 0.2.0

Model-aware batch processing queue with ETA estimation for Tauri applications
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

use ai_batch_queue::*;

#[allow(dead_code)]
struct SimulatedProcessor;

impl BatchItemHandler<String> for SimulatedProcessor {
    async fn process(
        &self,
        _data: &String,
        _resource_key: &str,
        _operation: &str,
    ) -> anyhow::Result<ItemResult> {
        Ok(ItemResult::success())
    }
}

fn main() {
    let queue: BatchQueue<String> = BatchQueue::new();

    // Create a job with items of different sizes
    let items = vec![
        ("img-1".into(), "small.jpg".into(), SizeBucket::Small),
        ("img-2".into(), "medium.jpg".into(), SizeBucket::Medium),
        ("img-3".into(), "large.jpg".into(), SizeBucket::Large),
        ("img-4".into(), "small2.jpg".into(), SizeBucket::Small),
        ("img-5".into(), "medium2.jpg".into(), SizeBucket::Medium),
    ];

    let job_id = queue
        .enqueue(build_job("llava:13b", "tag", OverwritePolicy::Skip, items))
        .unwrap();

    queue.mark_running(&job_id).unwrap();

    // Before any completions: no ETA data available
    let eta = queue.estimate_remaining_ms(&job_id);
    println!("ETA before any data: {:?}", eta);

    // Simulate completing items with different durations per size
    println!("\nSimulating item completions...");

    // Small image: 500ms
    queue
        .update_item(
            &job_id,
            "img-1",
            BatchItemStatus::Completed,
            None,
            Some(500),
        )
        .unwrap();
    println!("  Completed small image in 500ms");

    let eta = queue.estimate_remaining_ms(&job_id);
    println!("  ETA for 4 remaining: {:?}ms", eta);

    // Medium image: 1200ms
    queue
        .update_item(
            &job_id,
            "img-2",
            BatchItemStatus::Completed,
            None,
            Some(1200),
        )
        .unwrap();
    println!("  Completed medium image in 1200ms");

    let eta = queue.estimate_remaining_ms(&job_id);
    println!("  ETA for 3 remaining: {:?}ms", eta);

    // Large image: 3000ms
    queue
        .update_item(
            &job_id,
            "img-3",
            BatchItemStatus::Completed,
            None,
            Some(3000),
        )
        .unwrap();
    println!("  Completed large image in 3000ms");

    let eta = queue.estimate_remaining_ms(&job_id);
    println!("  ETA for 2 remaining (small + medium): {:?}ms", eta);
    // Should be ~500 + 1200 = 1700ms

    println!("\nETA accuracy improves with more data points.");
    println!("Estimates are bucketed by (model, operation, image_size).");

    // Verify ETA tracking
    assert_eq!(
        queue.eta_sample_count("llava:13b", "tag", SizeBucket::Small),
        1
    );
    assert_eq!(
        queue.eta_sample_count("llava:13b", "tag", SizeBucket::Medium),
        1
    );
    assert_eq!(
        queue.eta_sample_count("llava:13b", "tag", SizeBucket::Large),
        1
    );
}