auv-driver-linux 0.0.22

Linux driver for AUV
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

//! Capture-driven text recognition that backs the Linux session `VisionApi`.
//!
//! Linux OCR mirrors the Windows driver shape: `ocr` owns the Tesseract call,
//! while this module owns crop math and projection from cropped-image pixels
//! back into capture coordinates.

use auv_driver_common::capture::Capture;
use auv_driver_common::error::DriverResult;
use auv_driver_common::geometry::{RatioRect, Rect};
pub use auv_driver_common::vision::{OcrMatch, OcrMatches};
use auv_driver_common::vision::{RecognizedText, TextRecognition, TextRecognitionOptions};

use crate::error::backend;
use crate::ocr::{find_text_in_rgba, recognize_text_in_rgba};

pub fn recognize_text_in_capture(capture: &Capture, region: RatioRect, options: &TextRecognitionOptions) -> DriverResult<TextRecognition> {
  let crop = crop_pixels(capture, region);
  if crop.width == 0 || crop.height == 0 {
    return Ok(TextRecognition {
      origin: capture.recognition_origin(),
      ..Default::default()
    });
  }
  let cropped = image::imageops::crop_imm(&capture.image, crop.x, crop.y, crop.width, crop.height).to_image();
  let recognition =
    recognize_text_in_rgba(cropped.as_raw(), crop.width, crop.height, options).map_err(|error| backend(error.to_string()))?;
  Ok(map_recognition_to_capture(&recognition, capture, crop))
}

pub fn find_text_in_capture(
  capture: &Capture,
  query: &str,
  region: RatioRect,
  options: &TextRecognitionOptions,
) -> DriverResult<OcrMatches> {
  let crop = crop_pixels(capture, region);
  if crop.width == 0 || crop.height == 0 {
    return Ok(OcrMatches::default());
  }
  let cropped = image::imageops::crop_imm(&capture.image, crop.x, crop.y, crop.width, crop.height).to_image();
  let matches = find_text_in_rgba(cropped.as_raw(), crop.width, crop.height, query, options).map_err(|error| backend(error.to_string()))?;
  Ok(map_matches_to_capture(&matches, capture, crop))
}

#[derive(Clone, Copy, Debug, PartialEq)]
struct CropPixels {
  x: u32,
  y: u32,
  width: u32,
  height: u32,
}

fn crop_pixels(capture: &Capture, region: RatioRect) -> CropPixels {
  let image_width = capture.image.width();
  let image_height = capture.image.height();
  let x = ratio_to_pixel(region.x, image_width);
  let y = ratio_to_pixel(region.y, image_height);
  let width = ratio_to_pixel(region.width, image_width).min(image_width - x);
  let height = ratio_to_pixel(region.height, image_height).min(image_height - y);
  CropPixels {
    x,
    y,
    width,
    height,
  }
}

fn ratio_to_pixel(ratio: f64, extent: u32) -> u32 {
  if !ratio.is_finite() || ratio <= 0.0 {
    return 0;
  }
  (f64::from(extent) * ratio).round().clamp(0.0, f64::from(extent)) as u32
}

fn map_recognition_to_capture(recognition: &TextRecognition, capture: &Capture, crop: CropPixels) -> TextRecognition {
  let regions = recognition
    .regions
    .iter()
    .map(|region| RecognizedText {
      text: region.text.clone(),
      confidence: region.confidence,
      bounds: map_rect_to_capture(region.bounds, capture, crop),
    })
    .collect::<Vec<_>>();
  TextRecognition {
    origin: capture.recognition_origin(),
    text: regions.iter().map(|region| region.text.as_str()).collect::<Vec<_>>().join("\n"),
    regions,
  }
}

fn map_matches_to_capture(matches: &OcrMatches, capture: &Capture, crop: CropPixels) -> OcrMatches {
  OcrMatches {
    matches: matches
      .matches
      .iter()
      .map(|matched| OcrMatch {
        text: matched.text.clone(),
        confidence: matched.confidence,
        bounds: map_rect_to_capture(matched.bounds, capture, crop),
      })
      .collect(),
  }
}

fn map_rect_to_capture(bounds: Rect, capture: &Capture, crop: CropPixels) -> Rect {
  let x_scale = if capture.bounds.size.width > 0.0 {
    f64::from(capture.image.width()) / capture.bounds.size.width
  } else {
    1.0
  };
  let y_scale = if capture.bounds.size.height > 0.0 {
    f64::from(capture.image.height()) / capture.bounds.size.height
  } else {
    1.0
  };
  let full_x = bounds.origin.x + f64::from(crop.x);
  let full_y = bounds.origin.y + f64::from(crop.y);
  Rect::new(
    capture.bounds.origin.x + full_x / x_scale,
    capture.bounds.origin.y + full_y / y_scale,
    bounds.size.width / x_scale,
    bounds.size.height / y_scale,
  )
}

#[cfg(test)]
#[path = "vision_test.rs"]
mod tests;