eoka-agent 0.1.4

AI agent interaction layer for browser automation — MCP server, observe/act loop
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

//! Screenshot annotation — injects numbered labels over interactive elements.

use eoka::{Page, Result};

use crate::InteractiveElement;

const ANNOTATE_JS: &str = include_str!("js/annotate.js");

/// Inject numbered overlay labels, take screenshot, remove overlays.
pub async fn annotated_screenshot(page: &Page, elements: &[InteractiveElement]) -> Result<Vec<u8>> {
    if elements.is_empty() {
        return page.screenshot().await;
    }

    // Build element data as JSON — avoids all escaping issues
    let elem_data: Vec<serde_json::Value> = elements
        .iter()
        .map(|el| {
            serde_json::json!({
                "i": el.index,
                "x": el.bbox.x as i32,
                "y": el.bbox.y as i32,
                "w": el.bbox.width as i32,
                "h": el.bbox.height as i32,
            })
        })
        .collect();

    let json = serde_json::to_string(&elem_data).unwrap_or_default();
    let inject_js = ANNOTATE_JS.replace("/*ELEM_DATA*/", &json);

    page.execute(&inject_js).await?;
    tokio::time::sleep(std::time::Duration::from_millis(50)).await;
    let png = page.screenshot().await?;
    page.execute("document.getElementById('__eoka_overlay')?.remove()")
        .await?;

    Ok(png)
}