ghost_core/cdp/
mod.rs

1// CDP bridge — Chrome DevTools Protocol over HTTP + WebSocket.
2//
3// Chrome must be launched with --remote-debugging-port=9222.
4// Provides element finding in pages where the AX tree is empty or incomplete
5// (iframes, canvas-heavy SPAs, Gmail, Figma, etc.).
6//
7// Port from ghost-os/Sources/GhostOS/Vision/CDPBridge.swift.
8
9use anyhow::Result;
10use futures_util::{SinkExt, StreamExt};
11use serde_json::{json, Value};
12
13const CDP_HTTP: &str = "http://127.0.0.1:9222/json";
14const HTTP_TIMEOUT_MS: u64 = 1500;
15const WS_TIMEOUT_MS: u64 = 3000;
16
17/// One element found via CDP Runtime.evaluate.
18/// Coordinates are viewport-relative (not screen-absolute).
19#[derive(Debug, Clone)]
20pub struct CdpElement {
21    /// Viewport-relative horizontal center.
22    pub center_x: f64,
23    /// Viewport-relative vertical center.
24    pub center_y: f64,
25    pub text: String,
26    pub tag: String,
27    pub match_type: String,
28}
29
30/// Returns `true` if Chrome is listening on port 9222.
31pub async fn is_available() -> bool {
32    let Ok(client) = reqwest::Client::builder()
33        .timeout(std::time::Duration::from_millis(HTTP_TIMEOUT_MS))
34        .build()
35    else {
36        return false;
37    };
38    client.get(CDP_HTTP).send().await.is_ok()
39}
40
41/// Find elements matching `query` in the frontmost Chrome tab.
42///
43/// Uses five strategies in order: aria-label, placeholder, text content,
44/// label-for, title/alt — same as CDPBridge.swift.
45pub async fn find_elements(query: &str) -> Result<Vec<CdpElement>> {
46    // 1. List open tabs via HTTP
47    let client = reqwest::Client::builder()
48        .timeout(std::time::Duration::from_millis(HTTP_TIMEOUT_MS))
49        .build()?;
50    let tabs: Value = client.get(CDP_HTTP).send().await?.json().await?;
51
52    let ws_url = tabs
53        .as_array()
54        .and_then(|a| a.iter().find(|t| t["type"].as_str() == Some("page")))
55        .and_then(|t| t["webSocketDebuggerUrl"].as_str())
56        .ok_or_else(|| anyhow::anyhow!("No Chrome page tab found on port 9222"))?
57        .to_string();
58
59    // 2. Open WebSocket connection to the tab
60    let (mut ws, _) = tokio::time::timeout(
61        std::time::Duration::from_millis(WS_TIMEOUT_MS),
62        tokio_tungstenite::connect_async(&ws_url),
63    )
64    .await
65    .map_err(|_| anyhow::anyhow!("WebSocket connect timeout"))??;
66
67    // 3. Build the JS element-matching expression (5 strategies, de-duplicated by position)
68    //    Sanitise query for template literal embedding.
69    let q = query.replace('`', "\\`").replace('\\', "\\\\");
70    let js = format!(
71        r#"
72(() => {{
73  const q = `{q}`;
74  const ql = q.toLowerCase();
75  const dedup = new Set();
76  const out = [];
77  const push = (e, mt) => {{
78    const r = e.getBoundingClientRect();
79    if (r.width === 0 || r.height === 0) return;
80    const key = `${{Math.round(r.x)}}_${{Math.round(r.y)}}`;
81    if (dedup.has(key)) return;
82    dedup.add(key);
83    out.push({{
84      text: e.textContent.trim().slice(0, 80),
85      tag:  e.tagName.toLowerCase(),
86      role: e.getAttribute('role') || '',
87      centerX: r.x + r.width  / 2,
88      centerY: r.y + r.height / 2,
89      matchType: mt,
90    }});
91  }};
92  // 1. aria-label
93  Array.from(document.querySelectorAll('[aria-label]'))
94    .filter(e => (e.getAttribute('aria-label') || '').toLowerCase().includes(ql))
95    .forEach(e => push(e, 'aria-label'));
96  // 2. placeholder
97  Array.from(document.querySelectorAll('input[placeholder],textarea[placeholder]'))
98    .filter(e => (e.getAttribute('placeholder') || '').toLowerCase().includes(ql))
99    .forEach(e => push(e, 'placeholder'));
100  // 3. text content (buttons, links, tabs, menu items)
101  Array.from(document.querySelectorAll('button,a,[role="tab"],[role="menuitem"],[role="option"]'))
102    .filter(e => e.textContent.trim().toLowerCase().includes(ql))
103    .forEach(e => push(e, 'text'));
104  // 4. label element text → associated input
105  Array.from(document.querySelectorAll('label'))
106    .filter(e => e.textContent.trim().toLowerCase().includes(ql))
107    .map(l => document.getElementById(l.getAttribute('for') || '') || l)
108    .forEach(e => push(e, 'label'));
109  // 5. title / alt attributes
110  Array.from(document.querySelectorAll('[title],[alt]'))
111    .filter(e => ((e.getAttribute('title') || '') + (e.getAttribute('alt') || '')).toLowerCase().includes(ql))
112    .forEach(e => push(e, 'title'));
113  // 6. CSS selector (for dom_class queries like ".myClass" or id-selectors)
114  if (q.startsWith('.') || q.startsWith('\x23')) {{
115    try {{
116      Array.from(document.querySelectorAll(q)).forEach(e => push(e, 'css'));
117    }} catch(e) {{}}
118  }}
119  return out.slice(0, 20);
120}})()
121"#
122    );
123
124    // 4. Send Runtime.evaluate command
125    let cmd = json!({
126        "id": 1,
127        "method": "Runtime.evaluate",
128        "params": { "expression": js, "returnByValue": true }
129    });
130    ws.send(tokio_tungstenite::tungstenite::Message::Text(
131        cmd.to_string(),
132    ))
133    .await?;
134
135    // 5. Read response — skip Ping/Pong/Close; match on id=1
136    let resp_text = tokio::time::timeout(std::time::Duration::from_millis(WS_TIMEOUT_MS), async {
137        loop {
138            match ws.next().await {
139                Some(Ok(tokio_tungstenite::tungstenite::Message::Text(t))) => {
140                    let v: Value = serde_json::from_str(&t)?;
141                    if v["id"] == 1 {
142                        return Ok::<_, anyhow::Error>(t);
143                    }
144                }
145                Some(Ok(_)) => continue, // Ping, Pong, Binary — skip
146                Some(Err(e)) => return Err(anyhow::anyhow!("WebSocket error: {e}")),
147                None => return Err(anyhow::anyhow!("WebSocket closed unexpectedly")),
148            }
149        }
150    })
151    .await
152    .map_err(|_| anyhow::anyhow!("WebSocket response timeout"))??;
153
154    // 6. Parse results from Runtime.evaluate response
155    let resp: Value = serde_json::from_str(&resp_text)?;
156    let items = resp["result"]["result"]["value"]
157        .as_array()
158        .cloned()
159        .unwrap_or_default();
160
161    Ok(items
162        .iter()
163        .filter_map(|item| {
164            Some(CdpElement {
165                center_x: item["centerX"].as_f64()?,
166                center_y: item["centerY"].as_f64()?,
167                text: item["text"].as_str().unwrap_or("").to_string(),
168                tag: item["tag"].as_str().unwrap_or("").to_string(),
169                match_type: item["matchType"].as_str().unwrap_or("").to_string(),
170            })
171        })
172        .collect())
173}
174
175/// Convert viewport-relative coordinates to screen-absolute coordinates.
176///
177/// Chrome reports element positions relative to the viewport (top-left of page
178/// content area). To click them we need screen-absolute coords, which requires
179/// adding the Chrome window origin and the browser chrome height (toolbar).
180///
181/// `win_x`, `win_y` — screen position of the Chrome window's top-left corner.
182/// Chrome toolbar height (title bar 36px + toolbar 52px) is hardcoded at 88px,
183/// matching CDPBridge.swift's value.
184pub fn viewport_to_screen(vp_x: f64, vp_y: f64, win_x: i32, win_y: i32) -> (i32, i32) {
185    const CHROME_TOOLBAR_HEIGHT: i32 = 88;
186    (
187        win_x + vp_x as i32,
188        win_y + CHROME_TOOLBAR_HEIGHT + vp_y as i32,
189    )
190}
ghost_core/cdp/mod.rs

ghost_core/cdp/
mod.rs
