rustautogui 2.5.0

Highly optimized GUI automation library for controlling the mouse and keyboard, with template matching support.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189

use core_graphics::display;
use core_graphics::display::CGDisplay;

use crate::{errors::AutoGuiError, imgtools};

#[cfg(not(feature = "lite"))]
use image::{
    imageops::{resize, FilterType::Nearest},
    GrayImage, ImageBuffer, Luma, Rgba,
};

#[derive(Debug, Clone)]
pub struct Screen {
    pub screen_width: i32,
    pub screen_height: i32,
    #[cfg(not(feature = "lite"))]
    pub screen_data: ScreenImgData,
}
#[cfg(not(feature = "lite"))]
#[derive(Debug, Clone)]
pub struct ScreenImgData {
    pub display: CGDisplay,
    pub pixel_data: Vec<u8>,
    pub scaling_factor_x: f32, // difference between logical and phyisical resolution
    pub scaling_factor_y: f32,
    pub screen_region_width: u32,
    pub screen_region_height: u32,
}

impl Screen {
    pub fn new() -> Result<Self, AutoGuiError> {
        unsafe {
            let main_display_id = display::CGMainDisplayID();
            let main_display = CGDisplay::new(main_display_id);
            // because of retina display,  and scaling factors, image captured can be double the size
            // for that detection of retina is needed to divide all the pixel positions
            // by the factor. As far as i understood it should actually always be 2 but leaving it like this
            // shouldnt produce errors and covers any different case
            #[allow(unused_variables)]
            let image = main_display.image().ok_or(AutoGuiError::OSFailure(
                "Failed to create CGImage from display".to_string(),
            ))?;

            let screen_width = main_display.pixels_wide() as i32;
            let screen_height = main_display.pixels_high() as i32;

            #[cfg(not(feature = "lite"))]
            let screen_data = ScreenImgData {
                display: main_display,
                pixel_data: vec![0u8; (screen_width * screen_height * 4) as usize],
                scaling_factor_x: image.width() as f32 / screen_width as f32,
                scaling_factor_y: image.height() as f32 / screen_height as f32,
                screen_region_width: 0,
                screen_region_height: 0,
            };
            Ok(Self {
                screen_height,
                screen_width,

                #[cfg(not(feature = "lite"))]
                screen_data,
            })
        }
    }

    /// returns screen dimensions. All monitors included
    pub fn dimension(&self) -> (i32, i32) {
        let dimensions = (self.screen_width, self.screen_height);
        dimensions
    }
    #[cfg(not(feature = "lite"))]
    #[allow(dead_code)]
    /// return region dimension which is set up when template is precalculated
    pub fn region_dimension(&self) -> (u32, u32) {
        let dimensions = (
            self.screen_data.screen_region_width,
            self.screen_data.screen_region_height,
        );
        dimensions
    }
    #[cfg(not(feature = "lite"))]
    #[allow(dead_code)]
    /// executes convert_bitmap_to_rgba, meaning it converts Vector of values to RGBA and crops the image
    /// as inputted region area. Not used anywhere at the moment
    pub fn grab_screen_image(
        &mut self,
        region: (u32, u32, u32, u32),
    ) -> Result<ImageBuffer<Rgba<u8>, Vec<u8>>, AutoGuiError> {
        let (x, y, width, height) = region;
        self.screen_data.screen_region_width = width;
        self.screen_data.screen_region_height = height;
        self.capture_screen()?;
        let image = self.convert_bitmap_to_rgba()?;
        let cropped_image: ImageBuffer<Rgba<u8>, Vec<u8>> =
            imgtools::cut_screen_region(x, y, width, height, &image);
        Ok(cropped_image)
    }
    #[cfg(not(feature = "lite"))]
    /// executes convert_bitmap_to_grayscale, meaning it converts Vector of values to grayscale and crops the image
    /// as inputted region area
    pub fn grab_screen_image_grayscale(
        &mut self,
        region: &(u32, u32, u32, u32),
    ) -> Result<ImageBuffer<Luma<u8>, Vec<u8>>, AutoGuiError> {
        let (x, y, width, height) = region;
        self.screen_data.screen_region_width = *width;
        self.screen_data.screen_region_height = *height;
        self.capture_screen()?;
        let image: ImageBuffer<Luma<u8>, Vec<u8>> = self.convert_bitmap_to_grayscale()?;
        let cropped_image: ImageBuffer<Luma<u8>, Vec<u8>> =
            imgtools::cut_screen_region(*x, *y, *width, *height, &image);
        Ok(cropped_image)
    }
    #[cfg(not(feature = "lite"))]
    /// captures and saves screenshot of monitors
    pub fn grab_screenshot(&mut self, image_path: &str) -> Result<(), AutoGuiError> {
        self.capture_screen()?;
        let image = self.convert_bitmap_to_rgba()?;
        Ok(image.save(image_path)?)
    }
    #[cfg(not(feature = "lite"))]
    /// first order capture screen function. it captures screen image and stores it as vector in self.pixel_data
    fn capture_screen(&mut self) -> Result<(), AutoGuiError> {
        let image = self
            .screen_data
            .display
            .image()
            .ok_or(AutoGuiError::OSFailure(
                "Failed to capture screen image".to_string(),
            ))?;

        let pixel_data: Vec<u8> = image
            .data()
            .bytes()
            .chunks(4)
            .flat_map(|chunk| {
                // reorder color components
                if let &[b, g, r, a] = chunk {
                    vec![r, g, b, a]
                } else {
                    unreachable!()
                }
            })
            .collect();
        self.screen_data.pixel_data = pixel_data;
        Ok(())
    }
    #[cfg(not(feature = "lite"))]
    /// convert vector to Luma Imagebuffer
    fn convert_bitmap_to_grayscale(&self) -> Result<ImageBuffer<Luma<u8>, Vec<u8>>, AutoGuiError> {
        let mut grayscale_data =
            Vec::with_capacity((self.screen_width * self.screen_height) as usize);
        for chunk in self.screen_data.pixel_data.chunks_exact(4) {
            let r = chunk[2] as u32;
            let g = chunk[1] as u32;
            let b = chunk[0] as u32;
            // calculate the grayscale value using the luminance formula
            let gray_value = ((r * 30 + g * 59 + b * 11) / 100) as u8;
            grayscale_data.push(gray_value);
        }
        let mut image = GrayImage::from_raw(
            (self.screen_data.scaling_factor_x * self.screen_width as f32) as u32,
            (self.screen_data.scaling_factor_y * self.screen_height as f32) as u32,
            grayscale_data,
        )
        .ok_or(AutoGuiError::ImgError(
            "Could not convert image to grayscale".to_string(),
        ))?;
        let image = resize(
            &mut image,
            self.screen_width as u32,
            self.screen_height as u32,
            Nearest,
        );
        Ok(image)
    }
    #[cfg(not(feature = "lite"))]
    /// convert vector to RGBA ImageBuffer
    fn convert_bitmap_to_rgba(&self) -> Result<ImageBuffer<Rgba<u8>, Vec<u8>>, AutoGuiError> {
        ImageBuffer::from_raw(
            (self.screen_data.scaling_factor_x * self.screen_width as f32) as u32,
            (self.screen_data.scaling_factor_y * self.screen_height as f32) as u32,
            self.screen_data.pixel_data.clone(),
        )
        .ok_or(AutoGuiError::ImgError(
            "Could not convert image to rgba".to_string(),
        ))
    }
}