visual-cortex 0.13.0

Watch screen regions and receive typed events over async streams when patterns match: OCR, template matching, and pixel conditions on live captures.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

//! Watcher-level debug sinks: on every tick the detector runs, sinks get a
//! `DebugStage::Spans` frame — the evaluated frame annotated with markers
//! where OCR read text.

use std::sync::{Arc, Mutex};
use std::time::Duration;

use visual_cortex::{
    DebugSink, DebugStage, DetectorError, FakeSource, Frame, FrameView, OcrEngine, PxRect, Rate,
    Region, Session, TextSpan,
};

struct FixedOcr;
impl OcrEngine for FixedOcr {
    fn recognize(&mut self, _view: &FrameView<'_>) -> Result<Vec<TextSpan>, DetectorError> {
        Ok(vec![TextSpan {
            text: "HP 1234".into(),
            confidence: 1.0,
            bbox: PxRect {
                x: 4,
                y: 4,
                w: 12,
                h: 6,
            },
        }])
    }
}

#[derive(Clone, Default)]
struct Recording {
    frames: Arc<Mutex<Vec<(u64, DebugStage, Frame)>>>,
    label: Arc<Mutex<String>>,
}
impl DebugSink for Recording {
    fn write(&mut self, tick: u64, stage: DebugStage, frame: &Frame) {
        let copy = Frame::new(frame.width(), frame.height(), frame.data().to_vec()).unwrap();
        self.frames.lock().unwrap().push((tick, stage, copy));
    }
    fn set_label(&mut self, label: &str) {
        *self.label.lock().unwrap() = label.to_string();
    }
}

#[tokio::test(start_paused = true)]
async fn ocr_watcher_debug_sink_receives_annotated_spans_frames() {
    let session = Session::builder()
        .source(FakeSource::new([Frame::solid(24, 24, [50, 50, 50, 255])]))
        .build()
        .await
        .unwrap();

    let sink = Recording::default();
    let frames = sink.frames.clone();
    let label = sink.label.clone();
    let mut stream = session
        .watch("markers")
        .region(Region::Full)
        .rate(Rate::hz(10.0))
        .debug_sink(sink)
        .ocr_text(FixedOcr)
        .subscribe()
        .unwrap();

    // First evaluation produces the initial event; the sink fires alongside.
    tokio::time::timeout(Duration::from_secs(5), stream.next())
        .await
        .expect("event")
        .expect("stream open");

    assert_eq!(*label.lock().unwrap(), "markers", "label plumbed");
    let got = frames.lock().unwrap();
    assert!(!got.is_empty(), "sink received frames");
    let (tick, stage, frame) = &got[0];
    assert_eq!(*tick, 0);
    assert_eq!(*stage, DebugStage::Spans);
    // Marker border at the span bbox (confidence 1.0 -> green in BGRA),
    // interior untouched.
    let px = |x: usize, y: usize| {
        let i = (y * 24 + x) * 4;
        [frame.data()[i], frame.data()[i + 1], frame.data()[i + 2]]
    };
    assert_eq!(px(4, 4), [0, 220, 0], "span border drawn");
    assert_eq!(px(10, 7), [50, 50, 50], "span interior untouched");
}