cortenforge-vision-runtime 0.6.0

Bevy plugins for capture and inference runtime built on vision_core for the CortenForge stack.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

use bevy::ecs::system::RunSystemOnce;
use bevy::prelude::*;
use bevy::tasks::AsyncComputeTaskPool;
use inference::InferenceThresholds;
use vision_core::interfaces::{self, DetectionResult, Detector, Frame};
use vision_runtime::prelude::{
    AsyncInferenceState, CapturePlugin, DetectionOverlayState, DetectorHandle, DetectorKind,
    InferenceRuntimePlugin, InferenceThresholdsResource,
};

struct DummyDetector;
impl Detector for DummyDetector {
    fn detect(&mut self, frame: &Frame) -> DetectionResult {
        DetectionResult {
            frame_id: frame.id,
            positive: true,
            confidence: 0.9,
            boxes: vec![[0.1, 0.1, 0.2, 0.2]],
            scores: vec![0.9],
        }
    }
}

// Smoke test: ensure the plugins add resources/systems without panicking and that the overlay
// updates when the detector kind is heuristic.
#[test]
fn inference_plugin_smoke_updates_overlay() {
    let mut app = App::new();
    app.add_plugins(MinimalPlugins)
        .insert_resource(DetectorHandle {
            detector: Box::new(DummyDetector),
            kind: DetectorKind::Heuristic,
        })
        .insert_resource(InferenceThresholdsResource(InferenceThresholds {
            objectness_threshold: 0.3,
            iou_threshold: 0.5,
        }))
        .add_plugins(CapturePlugin)
        .add_plugins(InferenceRuntimePlugin)
        .insert_resource(Assets::<Image>::default())
        .insert_resource(sim_core::SimRunMode::Inference)
        .insert_resource(ButtonInput::<KeyCode>::default());

    app.update();
    // Simulate a completed task by enqueuing a ready task and calling poll system.
    {
        let mut jobs = app
            .world_mut()
            .get_resource_mut::<AsyncInferenceState>()
            .unwrap();
        let task = AsyncComputeTaskPool::get().spawn(async {
            (
                Box::new(DummyDetector) as Box<dyn interfaces::Detector + Send + Sync>,
                DetectorKind::Heuristic,
                DetectionResult {
                    frame_id: 1,
                    positive: true,
                    confidence: 0.8,
                    boxes: vec![[0.1, 0.1, 0.2, 0.2]],
                    scores: vec![0.8],
                },
                1.0f32,
                (64, 64),
            )
        });
        jobs.pending = Some(task);
    }
    // Poll a few times to allow the async task to complete.
    for _ in 0..3 {
        app.world_mut()
            .run_system_once(vision_runtime::poll_inference_task)
            .unwrap();
        app.update();
    }
    let overlay = app.world().get_resource::<DetectionOverlayState>().unwrap();
    assert_eq!(overlay.boxes.len(), 1);
}