visual-cortex 0.2.0

Watch screen regions and receive typed events over async streams when patterns match: OCR, template matching, and pixel conditions on live captures.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155

use std::collections::VecDeque;
use std::time::Duration;

use visual_cortex::{
    patterns, DetectorError, DetectorOutput, EventKind, FakeSource, Frame, FrameView, OcrEngine,
    PxRect, Rate, Region, Session, TextSpan, ThresholdDirection,
};

/// Scripted OCR engine: pops one span-list per recognize call, then repeats
/// the last one forever.
struct ScriptedOcr {
    script: VecDeque<Vec<TextSpan>>,
    last: Vec<TextSpan>,
}

impl ScriptedOcr {
    fn new(script: impl IntoIterator<Item = Vec<TextSpan>>) -> Self {
        Self {
            script: script.into_iter().collect(),
            last: Vec::new(),
        }
    }
}

impl OcrEngine for ScriptedOcr {
    fn recognize(&mut self, _view: &FrameView<'_>) -> Result<Vec<TextSpan>, DetectorError> {
        if let Some(spans) = self.script.pop_front() {
            self.last = spans;
        }
        Ok(self.last.clone())
    }
}

fn span(text: &str, y: u32) -> TextSpan {
    TextSpan {
        text: text.to_string(),
        confidence: 0.95,
        bbox: PxRect {
            x: 2,
            y,
            w: 60,
            h: 10,
        },
    }
}

async fn next_event(stream: &mut visual_cortex::EventStream) -> visual_cortex::Event {
    tokio::time::timeout(Duration::from_secs(30), stream.next())
        .await
        .expect("timed out waiting for event")
        .expect("stream closed unexpectedly")
}

#[tokio::test(start_paused = true)]
async fn item_observed_via_text_transitions() {
    // Two empty ticks, then a tooltip appears for three ticks, then vanishes.
    let script = vec![
        vec![],
        vec![],
        vec![span("Ancestral Unique Sword", 20), span("Doombringer", 4)],
        vec![span("Ancestral Unique Sword", 20), span("Doombringer", 4)],
        vec![span("Ancestral Unique Sword", 20), span("Doombringer", 4)],
        vec![],
    ];

    let session = Session::builder()
        .source(FakeSource::new([Frame::solid(64, 64, [20, 20, 20, 255])]))
        .build()
        .await
        .unwrap();

    let mut stream = session
        .watch("tooltip")
        .region(Region::Full)
        .rate(Rate::hz(10.0))
        .frame_diff(false) // frame is static; the OCR script is what varies
        .ocr_text(ScriptedOcr::new(script))
        .subscribe()
        .unwrap();

    // First evaluation emits the initial observation (old: None -> None output).
    let first = next_event(&mut stream).await;
    assert!(
        matches!(
            &first.kind,
            EventKind::Changed {
                old: None,
                new: DetectorOutput::None
            }
        ),
        "expected initial None observation, got {:?}",
        first.kind
    );

    // "Item observed": None -> Text, spans joined in bbox order.
    let observed = next_event(&mut stream).await;
    let EventKind::Changed {
        old: Some(DetectorOutput::None),
        new: DetectorOutput::Text(text),
    } = &observed.kind
    else {
        panic!("expected None->Text transition, got {:?}", observed.kind);
    };
    assert_eq!(text, "Doombringer\nAncestral Unique Sword");

    // Tooltip dismissed: Text -> None.
    let gone = next_event(&mut stream).await;
    assert!(
        matches!(
            &gone.kind,
            EventKind::Changed {
                old: Some(DetectorOutput::Text(_)),
                new: DetectorOutput::None
            }
        ),
        "expected Text->None transition, got {:?}",
        gone.kind
    );
}

#[tokio::test(start_paused = true)]
async fn hp_threshold_via_ocr_number() {
    let script = vec![
        vec![span("HP 500", 2)],
        vec![span("HP 500", 2)],
        vec![span("HP 250", 2)],
    ];

    let session = Session::builder()
        .source(FakeSource::new([Frame::solid(64, 16, [0, 0, 0, 255])]))
        .build()
        .await
        .unwrap();

    let mut stream = session
        .watch("hp")
        .region(Region::Full)
        .rate(Rate::hz(10.0))
        .frame_diff(false)
        .ocr(ScriptedOcr::new(script), patterns::number())
        .on_threshold(|hp| hp < 300.0)
        .subscribe()
        .unwrap();

    let ev = next_event(&mut stream).await;
    assert!(
        matches!(
            ev.kind,
            EventKind::ThresholdCrossed { value, direction: ThresholdDirection::Entered }
                if value == 250.0
        ),
        "expected Entered at 250, got {:?}",
        ev.kind
    );
}