Skip to main content

AutomationBackend

punch_runtime::automation

Trait AutomationBackend 

Source 
pub trait AutomationBackend: Send + Sync {
    // Required methods
    fn screenshot<'life0, 'life1, 'async_trait>(
        &'life0 self,
        window: Option<&'life1 str>,
    ) -> Pin<Box<dyn Future<Output = PunchResult<ScreenshotResult>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn ui_screenshot<'life0, 'life1, 'async_trait>(
        &'life0 self,
        element_id: Option<&'life1 str>,
        bounds: Option<(i32, i32, u32, u32)>,
    ) -> Pin<Box<dyn Future<Output = PunchResult<ScreenshotResult>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn app_ocr<'life0, 'life1, 'async_trait>(
        &'life0 self,
        app: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = PunchResult<OcrResult>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn list_windows<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = PunchResult<Vec<WindowInfo>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn find_ui_elements<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        app: &'life1 str,
        selector: &'life2 UiSelector,
    ) -> Pin<Box<dyn Future<Output = PunchResult<Vec<UiElement>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
    fn click_element<'life0, 'life1, 'async_trait>(
        &'life0 self,
        element_id: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = PunchResult<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn type_text<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        element_id: &'life1 str,
        text: &'life2 str,
    ) -> Pin<Box<dyn Future<Output = PunchResult<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
    fn read_element_attribute<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        element_id: &'life1 str,
        attribute: &'life2 str,
    ) -> Pin<Box<dyn Future<Output = PunchResult<String>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
}
Expand description

Platform-agnostic desktop automation backend.

Required Methods§

Source

fn screenshot<'life0, 'life1, 'async_trait>( &'life0 self, window: Option<&'life1 str>, ) -> Pin<Box<dyn Future<Output = PunchResult<ScreenshotResult>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Capture a screenshot of the full screen or a specific window.

If window is Some, captures only that window (matched by title). Returns base64-encoded PNG data.

Source

fn ui_screenshot<'life0, 'life1, 'async_trait>( &'life0 self, element_id: Option<&'life1 str>, bounds: Option<(i32, i32, u32, u32)>, ) -> Pin<Box<dyn Future<Output = PunchResult<ScreenshotResult>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Capture a screenshot of a specific UI region by bounds.

If element_id is provided, captures the region of that element. If bounds is provided, captures that exact rectangle (x, y, w, h).

Source

fn app_ocr<'life0, 'life1, 'async_trait>( &'life0 self, app: &'life1 str, ) -> Pin<Box<dyn Future<Output = PunchResult<OcrResult>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Extract text from an app window using OCR.

This is cheaper than a screenshot + vision model for text-heavy content.

Source

fn list_windows<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = PunchResult<Vec<WindowInfo>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

List all visible windows with their titles and owning apps.

Source

fn find_ui_elements<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, app: &'life1 str, selector: &'life2 UiSelector, ) -> Pin<Box<dyn Future<Output = PunchResult<Vec<UiElement>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Query the accessibility tree for UI elements matching a selector.

Source

fn click_element<'life0, 'life1, 'async_trait>( &'life0 self, element_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = PunchResult<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Click a UI element by its element ID (from find_ui_elements).

Source

fn type_text<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, element_id: &'life1 str, text: &'life2 str, ) -> Pin<Box<dyn Future<Output = PunchResult<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Type text into a UI element by its element ID.

Source

fn read_element_attribute<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, element_id: &'life1 str, attribute: &'life2 str, ) -> Pin<Box<dyn Future<Output = PunchResult<String>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Read an accessibility attribute from a UI element.

Implementors§