visual-cortex 0.3.0

Watch screen regions and receive typed events over async streams when patterns match: OCR, template matching, and pixel conditions on live captures.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

//! OCR pipeline demo against scripted frames and a scripted OCR engine — no
//! real screen or models needed.
//! Run: cargo run -p visual_cortex --example ocr_demo

use std::collections::VecDeque;

use visual_cortex::{
    patterns, DetectorError, EventKind, FakeSource, Frame, FrameView, OcrEngine, PxRect, Rate,
    Region, Session, TextSpan,
};

struct ScriptedOcr(VecDeque<Vec<TextSpan>>);

impl OcrEngine for ScriptedOcr {
    fn recognize(&mut self, _view: &FrameView<'_>) -> Result<Vec<TextSpan>, DetectorError> {
        Ok(self.0.pop_front().unwrap_or_default())
    }
}

fn span(text: &str, y: u32) -> TextSpan {
    TextSpan {
        text: text.to_string(),
        confidence: 0.97,
        bbox: PxRect {
            x: 4,
            y,
            w: 80,
            h: 12,
        },
    }
}

#[tokio::main]
async fn main() -> visual_cortex::Result<()> {
    // A tooltip appears on tick 3 and vanishes on tick 6.
    let tooltip_script: VecDeque<Vec<TextSpan>> = vec![
        vec![],
        vec![],
        vec![span("Doombringer", 4), span("Ancestral Unique Sword", 20)],
        vec![span("Doombringer", 4), span("Ancestral Unique Sword", 20)],
        vec![span("Doombringer", 4), span("Ancestral Unique Sword", 20)],
        vec![],
    ]
    .into();
    // HP drops below 300 on tick 3.
    let hp_script: VecDeque<Vec<TextSpan>> = vec![
        vec![span("HP 500", 2)],
        vec![span("HP 450", 2)],
        vec![span("HP 250", 2)],
    ]
    .into();

    let session = Session::builder()
        .source(FakeSource::new([Frame::solid(160, 90, [20, 20, 20, 255])]))
        .build()
        .await?;

    let tooltip = session
        .watch("tooltip")
        .region(Region::percent(0.5, 0.05, 0.48, 0.9))
        .rate(Rate::hz(10.0))
        .frame_diff(false)
        .ocr_text(ScriptedOcr(tooltip_script))
        .subscribe()?;

    let hp = session
        .watch("hp")
        .region(Region::percent(0.05, 0.03, 0.25, 0.06))
        .rate(Rate::hz(10.0))
        .frame_diff(false)
        .ocr(ScriptedOcr(hp_script), patterns::number())
        .on_threshold(|hp| hp < 300.0)
        .subscribe()?;

    let mut events = tooltip.merge(hp);
    println!("watching for tooltips and low HP...");
    for _ in 0..4 {
        if let Some(ev) = events.next().await {
            match &ev.kind {
                EventKind::Changed { new, .. } => {
                    println!("[{}] now: {:?}", &*ev.watcher, new)
                }
                other => println!("[{}] {:?}", &*ev.watcher, other),
            }
        }
    }
    println!("done");
    Ok(())
}