use crate::error::ComputerError;
use crate::image_util::ScreenshotMap;
use image::RgbaImage;
#[derive(Debug, Clone, PartialEq)]
pub struct OcrBlock {
pub text: String,
pub x: i32,
pub y: i32,
pub width: u32,
pub height: u32,
pub confidence: f32,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct OcrResult {
pub blocks: Vec<OcrBlock>,
pub note: Option<String>,
}
pub fn recognize(image: &RgbaImage, _map: &ScreenshotMap) -> Result<OcrResult, ComputerError> {
#[cfg(target_os = "macos")]
{
macos::recognize(image)
}
#[cfg(target_os = "windows")]
{
windows::recognize(image)
}
#[cfg(not(any(target_os = "macos", target_os = "windows")))]
{
let _ = image;
Err(unsupported_on_this_os())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ImageRect {
pub x: i32,
pub y: i32,
pub width: u32,
pub height: u32,
}
impl ImageRect {
pub fn contains(&self, other: &ImageRect) -> bool {
let other_right = other.x.saturating_add(other.width as i32);
let other_bottom = other.y.saturating_add(other.height as i32);
let self_right = self.x.saturating_add(self.width as i32);
let self_bottom = self.y.saturating_add(self.height as i32);
other.x >= self.x
&& other.y >= self.y
&& other_right <= self_right
&& other_bottom <= self_bottom
}
pub fn to_screen(&self, map: &ScreenshotMap) -> Option<(f64, f64)> {
let center_x = self.x as f64 + (self.width as f64) / 2.0;
let center_y = self.y as f64 + (self.height as f64) / 2.0;
let content_w = f64::from(map.content_width.max(1));
let content_h = f64::from(map.content_height.max(1));
let pad_x = map.pad.x as f64;
let pad_y = map.pad.y as f64;
if center_x < pad_x || center_y < pad_y {
return None;
}
if center_x > pad_x + content_w || center_y > pad_y + content_h {
return None;
}
let logical_w = f64::from(map.logical_width.max(1));
let logical_h = f64::from(map.logical_height.max(1));
let screen_x = map.origin.x as f64 + (center_x - pad_x) * logical_w / content_w;
let screen_y = map.origin.y as f64 + (center_y - pad_y) * logical_h / content_h;
Some((screen_x, screen_y))
}
}
pub fn redact_blocks<I, R>(blocks: I, secure_frames: R) -> Vec<OcrBlock>
where
I: IntoIterator<Item = OcrBlock>,
R: IntoIterator<Item = ImageRect>,
{
let frames: Vec<ImageRect> = secure_frames.into_iter().collect();
if frames.is_empty() {
return blocks.into_iter().collect();
}
blocks
.into_iter()
.filter(|block| {
let image_rect = ImageRect {
x: block.x,
y: block.y,
width: block.width,
height: block.height,
};
!frames.iter().any(|frame| frame.contains(&image_rect))
})
.collect()
}
#[cfg_attr(any(target_os = "macos", target_os = "windows"), allow(dead_code))]
fn unsupported_on_this_os() -> ComputerError {
#[cfg(target_os = "linux")]
{
ComputerError::Unsupported(
"no OCR backend on Linux in 0.2.0 — OCR returns unsupported".into(),
)
}
#[cfg(not(target_os = "linux"))]
{
ComputerError::Unsupported("ocr backend missing on this platform".into())
}
}
#[cfg_attr(all(not(target_os = "windows"), not(test)), allow(dead_code))]
pub(crate) fn ocr_box_to_image(
image_width: u32,
image_height: u32,
x: f64,
y: f64,
width: f64,
height: f64,
) -> ImageRect {
let max_x = image_width as f64;
let max_y = image_height as f64;
let left = x.round().clamp(0.0, max_x) as i32;
let top = y.round().clamp(0.0, max_y) as i32;
let right = (x + width).round().clamp(f64::from(left), max_x) as i32;
let bottom = (y + height).round().clamp(f64::from(top), max_y) as i32;
ImageRect {
x: left,
y: top,
width: (right - left).max(1) as u32,
height: (bottom - top).max(1) as u32,
}
}
#[cfg(target_os = "macos")]
pub(crate) fn vision_box_to_image(image: &RgbaImage, rect: VisionRect) -> ImageRect {
let img_w = image.width() as f64;
let img_h = image.height() as f64;
let left = (rect.left.clamp(0.0, 1.0) * img_w).round() as i32;
let right = (rect.right.clamp(0.0, 1.0) * img_w).round() as i32;
let top_v = rect.top.clamp(0.0, 1.0);
let bot_v = rect.bottom.clamp(0.0, 1.0);
let flipped_top = (1.0 - bot_v) * img_h;
let flipped_bottom = (1.0 - top_v) * img_h;
let top = flipped_top.round().min(flipped_bottom.round()) as i32;
let bottom = flipped_top.round().max(flipped_bottom.round()) as i32;
let img_h_i = image.height() as i32;
ImageRect {
x: left.max(0),
y: top.max(0).min(img_h_i),
width: (right - left).max(1) as u32,
height: (bottom - top).max(1) as u32,
}
}
#[cfg(target_os = "macos")]
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) struct VisionRect {
pub left: f64,
pub top: f64,
pub right: f64,
pub bottom: f64,
}
#[cfg(target_os = "macos")]
mod macos {
use super::{vision_box_to_image, OcrBlock, OcrResult, VisionRect};
use crate::error::ComputerError;
use image::{ExtendedColorType, ImageEncoder, RgbaImage};
use objc2::rc::Retained;
use objc2::runtime::AnyObject;
use objc2::AnyThread;
use objc2_foundation::{NSArray, NSData, NSDictionary, NSString};
use objc2_vision::{
VNDetectedObjectObservation, VNImageRequestHandler, VNRecognizeTextRequest,
VNRecognizedText, VNRecognizedTextObservation, VNRequest, VNRequestTextRecognitionLevel,
};
#[allow(unsafe_code)]
pub fn recognize(image: &RgbaImage) -> Result<OcrResult, ComputerError> {
let (img_w, img_h) = (image.width(), image.height());
let mut bytes = Vec::new();
let encoder = image::codecs::png::PngEncoder::new(&mut bytes);
encoder
.write_image(image.as_raw(), img_w, img_h, ExtendedColorType::Rgba8)
.map_err(|error| {
ComputerError::ExecutionFailed(format!("encode capture for OCR: {error}"))
})?;
let png = NSData::with_bytes(&bytes);
let options: Retained<NSDictionary<NSString, AnyObject>> = NSDictionary::new();
let handler = VNImageRequestHandler::initWithData_options(
VNImageRequestHandler::alloc(),
&png,
&options,
);
let request = VNRecognizeTextRequest::new();
request.setRecognitionLevel(VNRequestTextRecognitionLevel::Accurate);
request.setUsesLanguageCorrection(true);
let upcast: Retained<VNRequest> = request.clone().into_super().into_super();
let requests: Retained<NSArray<VNRequest>> = NSArray::from_retained_slice(&[upcast]);
if let Err(error) = handler.performRequests_error(&requests) {
return Err(ComputerError::ExecutionFailed(format!(
"VNImageRequestHandler performRequests failed: {error}"
)));
}
let observations = match request.results() {
Some(obs) => obs,
None => {
return Ok(OcrResult {
blocks: Vec::new(),
note: None,
})
}
};
let observations_ref: &NSArray<VNRecognizedTextObservation> = &observations;
let count = observations_ref.len();
let mut blocks = Vec::with_capacity(count);
for i in 0..count {
let observation_retained: Retained<VNRecognizedTextObservation> =
observations_ref.objectAtIndex(i);
let rect = unsafe {
let obs_super: Retained<VNDetectedObjectObservation> =
Retained::cast_unchecked(observation_retained.clone());
(*obs_super).boundingBox()
};
let vision_rect = VisionRect {
left: rect.origin.x,
bottom: rect.origin.y,
right: rect.origin.x + rect.size.width,
top: rect.origin.y + rect.size.height,
};
let candidates = observation_retained.topCandidates(1);
let candidate: Retained<VNRecognizedText> = match candidates.into_iter().next() {
Some(c) => c,
None => continue,
};
let text_ns = candidate.string();
let confidence: f32 = candidate.confidence();
let image_rect = vision_box_to_image(image, vision_rect);
blocks.push(OcrBlock {
text: text_ns.to_string(),
x: image_rect.x,
y: image_rect.y,
width: image_rect.width,
height: image_rect.height,
confidence,
});
}
Ok(OcrResult { blocks, note: None })
}
}
#[cfg(target_os = "windows")]
mod windows {
#![allow(unsafe_code)]
use std::sync::mpsc::{self, RecvTimeoutError};
use std::thread;
use std::time::Duration;
use super::{ocr_box_to_image, OcrBlock, OcrResult};
use crate::error::ComputerError;
use image::RgbaImage;
use windows::Graphics::Imaging::{BitmapAlphaMode, BitmapPixelFormat, SoftwareBitmap};
use windows::Media::Ocr::{OcrEngine, OcrLine};
use windows::Security::Cryptography::CryptographicBuffer;
use windows::Win32::Foundation::S_FALSE;
use windows::Win32::System::Com::{
CoDecrementMTAUsage, CoIncrementMTAUsage, CO_MTA_USAGE_COOKIE,
};
use windows::Win32::System::WinRT::{RoInitialize, RoUninitialize, RO_INIT_MULTITHREADED};
const OCR_TIMEOUT: Duration = Duration::from_secs(10);
pub fn recognize(image: &RgbaImage) -> Result<OcrResult, ComputerError> {
let image = image.clone();
let (sender, receiver) = mpsc::channel::<Result<OcrResult, ComputerError>>();
let worker = thread::Builder::new()
.name("computer-use-ocr".into())
.spawn(move || {
let result = recognize_on_worker(&image);
let _ = sender.send(result);
});
if worker.is_err() {
return Err(ComputerError::ExecutionFailed(
"Windows.Media.Ocr worker thread could not start".into(),
));
}
match receiver.recv_timeout(OCR_TIMEOUT) {
Ok(result) => result,
Err(RecvTimeoutError::Timeout) => Err(ComputerError::ExecutionFailed(format!(
"Windows.Media.Ocr did not finish within {}s",
OCR_TIMEOUT.as_secs()
))),
Err(RecvTimeoutError::Disconnected) => Err(ComputerError::ExecutionFailed(
"Windows.Media.Ocr worker stopped without a result".into(),
)),
}
}
fn recognize_on_worker(image: &RgbaImage) -> Result<OcrResult, ComputerError> {
let _mta = MtaUsage::new()?;
let _apartment = Apartment::new()?;
let (width, height) = (image.width(), image.height());
let bitmap = software_bitmap(image)?;
let engine = engine()?;
let operation = engine.RecognizeAsync(&bitmap).map_err(|error| {
ComputerError::ExecutionFailed(format!("Windows.Media.Ocr RecognizeAsync: {error}"))
})?;
let result = operation.join().map_err(|error| {
ComputerError::ExecutionFailed(format!("Windows.Media.Ocr RecognizeAsync: {error}"))
})?;
let mut blocks = Vec::new();
let lines = result.Lines().map_err(|error| {
ComputerError::ExecutionFailed(format!("Windows.Media.Ocr OcrResult.Lines: {error}"))
})?;
for line in &lines {
let Some((text, box_rect)) = line_block(&line) else {
continue;
};
let rect = ocr_box_to_image(
width, height, box_rect.0, box_rect.1, box_rect.2, box_rect.3,
);
blocks.push(OcrBlock {
text,
x: rect.x,
y: rect.y,
width: rect.width,
height: rect.height,
confidence: 0.0,
});
}
Ok(OcrResult {
blocks,
note: language_note(&engine),
})
}
fn line_block(line: &OcrLine) -> Option<(String, (f64, f64, f64, f64))> {
let words = line.Words().ok()?;
let mut text_parts: Vec<String> = Vec::new();
let mut left = f64::MAX;
let mut top = f64::MAX;
let mut right = f64::MIN;
let mut bottom = f64::MIN;
for word in &words {
let word_text = word.Text().ok()?.to_string();
if word_text.trim().is_empty() {
continue;
}
let rect = word.BoundingRect().ok()?;
text_parts.push(word_text);
let (x, y) = (f64::from(rect.X), f64::from(rect.Y));
left = left.min(x);
top = top.min(y);
right = right.max(x + f64::from(rect.Width));
bottom = bottom.max(y + f64::from(rect.Height));
}
if text_parts.is_empty() {
return None;
}
Some((
text_parts.join(" "),
(left, top, (right - left).max(1.0), (bottom - top).max(1.0)),
))
}
fn engine() -> Result<OcrEngine, ComputerError> {
OcrEngine::TryCreateFromUserProfileLanguages().map_err(|error| {
ComputerError::ExecutionFailed(format!(
"Windows.Media.Ocr has no engine for the user-profile languages. Install a \
language pack (Settings > Time & Language > Language > Language options, or \
the OCR Recognizer in Settings > Apps > Optional features): {error}"
))
})
}
fn language_note(engine: &OcrEngine) -> Option<String> {
let tag = engine
.RecognizerLanguage()
.ok()?
.LanguageTag()
.ok()?
.to_string();
if tag.is_empty() {
None
} else {
Some(format!("Media OCR recognizer language: {tag}"))
}
}
fn software_bitmap(image: &RgbaImage) -> Result<SoftwareBitmap, ComputerError> {
let (width, height) = (image.width(), image.height());
let width = i32::try_from(width).map_err(|_| {
ComputerError::ExecutionFailed(format!("capture width {width} is too large"))
})?;
let height = i32::try_from(height).map_err(|_| {
ComputerError::ExecutionFailed(format!("capture height {height} is too large"))
})?;
let max_edge = OcrEngine::MaxImageDimension().unwrap_or(0);
if max_edge > 0
&& (u32::try_from(width).unwrap_or(u32::MAX) > max_edge
|| u32::try_from(height).unwrap_or(u32::MAX) > max_edge)
{
return Err(ComputerError::ExecutionFailed(format!(
"Windows.Media.Ocr accepts at most {max_edge} px per edge; the capture is \
{width}x{height}"
)));
}
let source = image.as_raw();
let mut bgra = Vec::with_capacity(source.len());
for [r, g, b, a] in source.as_chunks::<4>().0 {
bgra.extend_from_slice(&[*b, *g, *r, *a]);
}
let buffer = CryptographicBuffer::CreateFromByteArray(&bgra).map_err(|error| {
ComputerError::ExecutionFailed(format!(
"CryptographicBuffer::CreateFromByteArray: {error}"
))
})?;
SoftwareBitmap::CreateCopyWithAlphaFromBuffer(
&buffer,
BitmapPixelFormat::Bgra8,
width,
height,
BitmapAlphaMode::Premultiplied,
)
.map_err(|error| {
ComputerError::ExecutionFailed(format!(
"SoftwareBitmap::CreateCopyWithAlphaFromBuffer: {error}"
))
})
}
struct MtaUsage {
cookie: CO_MTA_USAGE_COOKIE,
}
impl MtaUsage {
fn new() -> Result<Self, ComputerError> {
let cookie = unsafe { CoIncrementMTAUsage() }.map_err(|error| {
ComputerError::ExecutionFailed(format!("CoIncrementMTAUsage: {error}"))
})?;
Ok(Self { cookie })
}
}
impl Drop for MtaUsage {
fn drop(&mut self) {
let _ = unsafe { CoDecrementMTAUsage(self.cookie) };
}
}
struct Apartment {
initialized: bool,
}
impl Apartment {
fn new() -> Result<Self, ComputerError> {
match unsafe { RoInitialize(RO_INIT_MULTITHREADED) } {
Ok(()) => Ok(Self { initialized: true }),
Err(error) if error.code() == S_FALSE => Ok(Self { initialized: false }),
Err(error) => Err(ComputerError::ExecutionFailed(format!(
"RoInitialize(RO_INIT_MULTITHREADED) failed: {error}"
))),
}
}
}
impl Drop for Apartment {
fn drop(&mut self) {
if self.initialized {
unsafe { RoUninitialize() };
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ocr_box_passes_through_image_pixels() {
let rect = ocr_box_to_image(800, 600, 100.0, 50.0, 200.0, 30.0);
assert_eq!(
rect,
ImageRect {
x: 100,
y: 50,
width: 200,
height: 30
}
);
}
#[test]
fn ocr_box_keeps_the_top_left_origin() {
let rect = ocr_box_to_image(400, 400, 10.0, 20.0, 50.0, 15.0);
assert_eq!(rect.y, 20);
assert!(rect.y < 200, "{rect:?}");
}
#[test]
fn ocr_box_rounds_to_whole_pixels() {
let rect = ocr_box_to_image(800, 600, 10.4, 20.6, 30.5, 40.4);
assert_eq!(rect.x, 10);
assert_eq!(rect.y, 21);
assert_eq!(rect.width, 31);
assert_eq!(rect.height, 40);
}
#[test]
fn ocr_box_is_clamped_to_the_image() {
let rect = ocr_box_to_image(200, 100, 190.0, 90.0, 80.0, 80.0);
assert_eq!(rect.x, 190);
assert_eq!(rect.y, 90);
assert!(rect.x + rect.width as i32 <= 200, "{rect:?}");
assert!(rect.y + rect.height as i32 <= 100, "{rect:?}");
}
#[test]
fn ocr_box_clamps_a_negative_origin_to_zero() {
let rect = ocr_box_to_image(200, 100, -40.0, -10.0, 60.0, 20.0);
assert_eq!(rect.x, 0);
assert_eq!(rect.y, 0);
assert_eq!(rect.width, 20);
assert_eq!(rect.height, 10);
}
#[test]
fn ocr_box_is_never_empty() {
let rect = ocr_box_to_image(200, 100, 50.0, 50.0, 0.0, 0.0);
assert_eq!(rect.width, 1);
assert_eq!(rect.height, 1);
}
#[test]
fn ocr_box_feeds_the_screenshot_map_back_to_a_screen_point() {
let map = crate::image_util::ScreenshotMap {
width: 800,
height: 600,
scale: 1.0,
logical_width: 800,
logical_height: 600,
origin: crate::image_util::MapPoint { x: 120, y: 40 },
fit: "none".into(),
pad: crate::image_util::MapPoint { x: 0, y: 0 },
content_width: 800,
content_height: 600,
map: crate::image_util::MapFormulas {
image_to_screen: String::new(),
screen_to_image: String::new(),
},
};
let rect = ocr_box_to_image(800, 600, 100.0, 50.0, 200.0, 30.0);
let (screen_x, screen_y) = rect.to_screen(&map).expect("on screen");
assert!((screen_x - 320.0).abs() < 0.001, "{screen_x}");
assert!((screen_y - 105.0).abs() < 0.001, "{screen_y}");
}
fn block(x: i32, y: i32, w: u32, h: u32) -> OcrBlock {
OcrBlock {
text: "secret".into(),
x,
y,
width: w,
height: h,
confidence: 0.95,
}
}
fn rect(x: i32, y: i32, w: u32, h: u32) -> ImageRect {
ImageRect {
x,
y,
width: w,
height: h,
}
}
#[test]
fn redaction_drops_blocks_inside_secure_frame() {
let blocks = vec![block(10, 10, 50, 20), block(200, 200, 5, 5)];
let frames = vec![rect(0, 0, 100, 100)];
let kept = redact_blocks(blocks, frames);
assert_eq!(kept.len(), 1, "secure frame must drop the inner block");
assert_eq!((kept[0].x, kept[0].y), (200, 200));
}
#[test]
fn redaction_keeps_blocks_overlapping_an_edge() {
let blocks = vec![block(90, 90, 30, 30)];
let frames = vec![rect(0, 0, 100, 100)];
let kept = redact_blocks(blocks, frames);
assert_eq!(kept.len(), 1, "blocks that overflow the secure frame stay");
}
#[test]
fn redaction_with_no_frames_returns_all_blocks() {
let blocks = vec![block(0, 0, 10, 10), block(50, 50, 5, 5)];
let kept = redact_blocks(blocks, Vec::<ImageRect>::new());
assert_eq!(kept.len(), 2);
}
#[test]
#[cfg(target_os = "linux")]
fn linux_ocr_returns_unsupported() {
use image::RgbaImage;
let image = RgbaImage::from_pixel(8, 8, image::Rgba([255, 255, 255, 255]));
let map = make_stub_map();
let err = recognize(&image, &map).unwrap_err();
match err {
ComputerError::Unsupported(reason) => {
assert!(
reason.to_lowercase().contains("linux"),
"Linux unsupported message must mention linux: {reason}"
);
}
other => panic!("expected Unsupported, got {other:?}"),
}
}
#[test]
#[cfg(target_os = "windows")]
fn windows_media_ocr_runs_and_never_returns_unsupported() {
use image::RgbaImage;
let image = RgbaImage::from_pixel(64, 32, image::Rgba([255, 255, 255, 255]));
let map = make_stub_map();
match recognize(&image, &map) {
Ok(result) => {
assert!(
result.blocks.iter().all(|block| {
block.width >= 1 && block.height >= 1 && block.x >= 0 && block.y >= 0
}),
"blocks must be non-empty image rects: {:?}",
result.blocks
);
}
Err(ComputerError::Unsupported(_)) => {
panic!("Windows OCR must not be unsupported after P12b")
}
Err(ComputerError::ExecutionFailed(reason)) => assert!(
reason.contains("Windows.Media.Ocr"),
"a Windows OCR failure must name the engine: {reason}"
),
Err(other) => panic!("unexpected error from Windows OCR: {other:?}"),
}
}
#[cfg(target_os = "macos")]
#[test]
fn vision_rect_flips_origin() {
use image::{Rgba, RgbaImage};
let image = RgbaImage::from_pixel(100, 80, Rgba([255, 255, 255, 255]));
let rect = super::VisionRect {
left: 0.0,
top: 1.0,
right: 1.0,
bottom: 0.0,
};
let image_rect = super::vision_box_to_image(&image, rect);
assert_eq!(image_rect.x, 0);
assert_eq!(image_rect.y, 0);
assert_eq!(image_rect.width, 100);
assert_eq!(image_rect.height, 80);
}
#[cfg(target_os = "macos")]
#[test]
#[ignore = "requires a live Vision framework; run locally with `cargo test --release vision_recognizes_drawn_text -- --ignored --nocapture`"]
fn vision_recognizes_drawn_text() {
use image::{Rgba, RgbaImage};
const FONT_W: usize = 5;
const FONT_H: usize = 7;
const FONT: &[(char, [u8; FONT_H])] = &[
(
'H',
[
0b10001, 0b10001, 0b10001, 0b11111, 0b10001, 0b10001, 0b10001,
],
),
(
'E',
[
0b11111, 0b10000, 0b10000, 0b11110, 0b10000, 0b10000, 0b11111,
],
),
(
'L',
[
0b10000, 0b10000, 0b10000, 0b10000, 0b10000, 0b10000, 0b11111,
],
),
(
'O',
[
0b01110, 0b10001, 0b10001, 0b10001, 0b10001, 0b10001, 0b01110,
],
),
(' ', [0, 0, 0, 0, 0, 0, 0]),
];
fn put_pixel(image: &mut RgbaImage, x: i32, y: i32, color: Rgba<u8>) {
if x < 0 || y < 0 {
return;
}
let w = image.width() as i32;
let h = image.height() as i32;
if x >= w || y >= h {
return;
}
image.put_pixel(x as u32, y as u32, color);
}
let mut image = RgbaImage::from_pixel(200, 60, Rgba([255, 255, 255, 255]));
let ink = Rgba([0, 0, 0, 255]);
let scale = 4_u32;
let text = "HELLO";
let origin_x: i32 = 10;
let origin_y: i32 = 10;
let cell_w = (FONT_W as u32) * scale;
let cell_h = (FONT_H as u32) * scale;
let mut pen_x = origin_x;
for ch in text.chars() {
let glyph = match FONT.iter().find(|(key, _)| *key == ch) {
Some((_, g)) => *g,
None => {
pen_x += cell_w as i32;
continue;
}
};
for (row, byte) in glyph.iter().enumerate() {
for col in 0..FONT_W {
if (byte >> (FONT_W - 1 - col)) & 1 == 1 {
let px0 = pen_x + (col as i32) * scale as i32;
let py0 = origin_y + (row as i32) * scale as i32;
for dy in 0..scale as i32 {
for dx in 0..scale as i32 {
put_pixel(&mut image, px0 + dx, py0 + dy, ink);
}
}
}
}
}
pen_x += cell_w as i32;
if pen_x + cell_w as i32 > image.width() as i32 {
break;
}
if origin_y + cell_h as i32 > image.height() as i32 {
break;
}
}
let map = make_stub_map();
let result =
recognize(&image, &map).expect("recognize should not error on a synthetic image");
assert!(
!result.blocks.is_empty(),
"Vision must return at least one block for a drawn text buffer; got {:?}",
result
);
let joined: String = result
.blocks
.iter()
.map(|block| block.text.to_uppercase())
.collect::<Vec<_>>()
.join(" ");
assert!(
joined.contains("HELLO"),
"Vision output should contain HELLO; got: {joined:?}"
);
}
#[cfg(any(test, target_os = "linux", target_os = "windows"))]
#[allow(dead_code)]
fn make_stub_map() -> crate::image_util::ScreenshotMap {
crate::image_util::ScreenshotMap {
width: 8,
height: 8,
scale: 1.0,
logical_width: 8,
logical_height: 8,
origin: crate::image_util::MapPoint { x: 0, y: 0 },
fit: "none".into(),
pad: crate::image_util::MapPoint { x: 0, y: 0 },
content_width: 8,
content_height: 8,
map: crate::image_util::MapFormulas {
image_to_screen: String::new(),
screen_to_image: String::new(),
},
}
}
}