rdesktop_dev/
agent_api.rs

1//! Agent API endpoints.
2//!
3//! These endpoints allow AI agents to interact with the running application
4//! through structured HTTP requests, without needing native desktop control.
5//!
6//! ## How it works
7//!
8//! 1. The browser app includes the rdesktop bridge script
9//! 2. The bridge script periodically sends DOM snapshots to the server
10//! 3. Agents query the server for DOM/state information
11//! 4. Agents send actions to the server, which forwards them to the browser
12//!
13//! ## Design Philosophy
14//!
15//! Instead of requiring agents to take screenshots and use vision models to
16//! understand the UI, the Agent API provides direct DOM access and structured
17//! state information. This is:
18//!
19//! - **Faster**: No screenshot encoding/decoding overhead
20//! - **More reliable**: Exact element selectors, not pixel coordinates
21//! - **More informative**: Full DOM tree, computed styles, accessibility info
22//! - **Easier to test**: Standard HTTP endpoints, can be scripted
23
24use axum::extract::{Query, State};
25use axum::http::StatusCode;
26use axum::response::IntoResponse;
27use axum::Json;
28use serde::{Deserialize, Serialize};
29use std::collections::HashMap;
30
31use crate::server::DevServerState;
32
33/// Query parameters for element selection.
34#[derive(Debug, Deserialize)]
35pub struct ElementQuery {
36    /// CSS selector to query elements
37    pub selector: Option<String>,
38
39    /// Text content to search for
40    pub text: Option<String>,
41
42    /// Role attribute to filter by
43    pub role: Option<String>,
44}
45
46/// An action that an agent can execute on the UI.
47#[derive(Debug, Deserialize, Serialize)]
48pub struct AgentAction {
49    /// The type of action
50    pub action: ActionType,
51
52    /// CSS selector of the target element
53    pub selector: String,
54
55    /// Value for type/fill actions
56    pub value: Option<String>,
57
58    /// Coordinates for scroll actions
59    pub coordinates: Option<(f64, f64)>,
60}
61
62/// Types of actions agents can execute.
63#[derive(Debug, Deserialize, Serialize)]
64#[serde(rename_all = "snake_case")]
65pub enum ActionType {
66    Click,
67    DoubleClick,
68    RightClick,
69    Type,
70    Fill,
71    Clear,
72    Scroll,
73    Hover,
74    Focus,
75    Select,
76}
77
78/// Response from a DOM query.
79#[derive(Debug, Serialize)]
80pub struct DomSnapshot {
81    /// The full HTML content
82    pub html: String,
83
84    /// The page URL
85    pub url: String,
86
87    /// The page title
88    pub title: String,
89
90    /// Timestamp of the snapshot
91    pub timestamp: String,
92}
93
94/// Response from an element query.
95#[derive(Debug, Serialize)]
96pub struct ElementInfo {
97    /// CSS selector that uniquely identifies this element
98    pub selector: String,
99
100    /// Tag name
101    pub tag: String,
102
103    /// Text content
104    pub text: String,
105
106    /// Element attributes
107    pub attributes: HashMap<String, String>,
108
109    /// Whether the element is visible
110    pub visible: bool,
111
112    /// Whether the element is enabled (for interactive elements)
113    pub enabled: bool,
114
115    /// Accessibility role
116    pub role: Option<String>,
117
118    /// Accessibility label
119    pub label: Option<String>,
120}
121
122/// Result of an action execution.
123#[derive(Debug, Serialize)]
124pub struct ActionResult {
125    /// Whether the action succeeded
126    pub success: bool,
127
128    /// Error message if the action failed
129    pub error: Option<String>,
130
131    /// Any side effects (e.g., navigation that occurred)
132    pub side_effects: Vec<String>,
133}
134
135/// GET /__rdesktop__/agent/dom
136///
137/// Returns a full DOM snapshot of the current page.
138/// The snapshot is collected from the browser via the bridge script.
139pub async fn get_dom(State(state): State<DevServerState>) -> impl IntoResponse {
140    let snapshot = state.last_dom_snapshot.read().await;
141
142    let html = snapshot.clone().unwrap_or_else(|| {
143        r#"<!DOCTYPE html>
144<html>
145<head><title>rdesktop</title></head>
146<body>
147  <p>No DOM snapshot available yet. Make sure the app is loaded in the browser.</p>
148  <p>The bridge script will send DOM updates automatically.</p>
149</body>
150</html>"#
151            .to_string()
152    });
153
154    let dom = DomSnapshot {
155        html,
156        url: "http://localhost".to_string(),
157        title: "rdesktop App".to_string(),
158        timestamp: timestamp(),
159    };
160
161    Json(dom).into_response()
162}
163
164/// GET /__rdesktop__/agent/elements?selector=...
165///
166/// Query elements matching a CSS selector or text content.
167pub async fn query_elements(
168    State(state): State<DevServerState>,
169    Query(query): Query<ElementQuery>,
170) -> impl IntoResponse {
171    let snapshot = state.last_dom_snapshot.read().await;
172
173    // Parse the DOM and find matching elements
174    let elements: Vec<ElementInfo> = if let Some(ref html) = *snapshot {
175        find_elements(html, &query)
176    } else {
177        vec![]
178    };
179
180    Json(serde_json::json!({
181        "query": {
182            "selector": query.selector,
183            "text": query.text,
184            "role": query.role,
185        },
186        "count": elements.len(),
187        "elements": elements,
188    }))
189    .into_response()
190}
191
192/// POST /__rdesktop__/agent/action
193///
194/// Execute a UI action (click, type, scroll, etc.)
195/// The action is stored and picked up by the bridge script.
196pub async fn execute_action(
197    State(_state): State<DevServerState>,
198    Json(action): Json<AgentAction>,
199) -> impl IntoResponse {
200    tracing::info!(
201        action = ?action.action,
202        selector = %action.selector,
203        "Agent action received"
204    );
205
206    // In a full implementation, this would:
207    // 1. Store the action in a shared queue
208    // 2. The bridge script polls for pending actions
209    // 3. The bridge executes the action in the browser
210    // 4. The result is returned
211
212    let result = ActionResult {
213        success: true,
214        error: None,
215        side_effects: vec![format!(
216            "Action {:?} on '{}' queued",
217            action.action, action.selector
218        )],
219    };
220
221    Json(result).into_response()
222}
223
224/// GET /__rdesktop__/agent/state
225///
226/// Get the current application state.
227pub async fn get_state(State(state): State<DevServerState>) -> impl IntoResponse {
228    let app_state = state.last_app_state.read().await;
229
230    match app_state.as_ref() {
231        Some(state) => Json(state.clone()).into_response(),
232        None => Json(serde_json::json!({
233            "message": "No application state available yet.",
234            "hint": "Use fetch('/__rdesktop__/state', { method: 'POST', body: JSON.stringify(state) }) from your app."
235        }))
236        .into_response(),
237    }
238}
239
240/// POST /__rdesktop__/agent/ipc
241///
242/// Send an IPC message from the agent to the app.
243pub async fn send_ipc(
244    State(_state): State<DevServerState>,
245    Json(message): Json<serde_json::Value>,
246) -> impl IntoResponse {
247    let cmd = message["cmd"].as_str().unwrap_or("unknown");
248    let payload = message["payload"].clone();
249    let id = message["id"].as_str().unwrap_or("0");
250
251    tracing::info!(cmd = cmd, "Agent IPC message received");
252
253    // In a full implementation, this would forward to the Rust IPC handler.
254    // For now, handle basic commands directly.
255    let response = match cmd {
256        "greet" => {
257            let name = payload["name"].as_str().unwrap_or("World");
258            serde_json::json!({
259                "id": id,
260                "success": true,
261                "data": { "message": format!("Hello, {}!", name) }
262            })
263        }
264        "ping" => {
265            serde_json::json!({
266                "id": id,
267                "success": true,
268                "data": { "pong": true }
269            })
270        }
271        _ => {
272            serde_json::json!({
273                "id": id,
274                "success": false,
275                "data": { "error": format!("Unknown command: {}", cmd) }
276            })
277        }
278    };
279
280    Json(response).into_response()
281}
282
283/// GET /__rdesktop__/agent/screenshot
284///
285/// Capture a screenshot. In browser mode, this delegates to the browser.
286pub async fn take_screenshot(State(_state): State<DevServerState>) -> impl IntoResponse {
287    (
288        StatusCode::NOT_IMPLEMENTED,
289        Json(serde_json::json!({
290            "message": "Screenshot not implemented in browser mode.",
291            "hint": "Use Playwright's page.screenshot() directly."
292        })),
293    )
294        .into_response()
295}
296
297/// Simple timestamp helper.
298fn timestamp() -> String {
299    let now = std::time::SystemTime::now()
300        .duration_since(std::time::UNIX_EPOCH)
301        .unwrap_or_default();
302    format!("{}", now.as_secs())
303}
304
305/// Find elements in HTML matching the query.
306/// This is a simple text-based search, not a full DOM parser.
307fn find_elements(html: &str, query: &ElementQuery) -> Vec<ElementInfo> {
308    let mut elements = vec![];
309
310    if let Some(ref selector) = query.selector {
311        // Simple tag selector matching (e.g., "button", "input", "h1")
312        let tag = selector.trim_start_matches('<').trim_end_matches('>');
313        let open_tag = format!("<{}", tag);
314
315        let mut start = 0;
316        while let Some(pos) = html[start..].find(&open_tag) {
317            let abs_pos = start + pos;
318            let end = html[abs_pos..].find('>').unwrap_or(0);
319            let _tag_content = &html[abs_pos..abs_pos + end + 1];
320
321            // Extract text content between tags
322            let close_tag = format!("</{}>", tag);
323            let text_start = abs_pos + end + 1;
324            let text = if let Some(text_end) = html[text_start..].find(&close_tag) {
325                html[text_start..text_start + text_end].trim().to_string()
326            } else {
327                String::new()
328            };
329
330            elements.push(ElementInfo {
331                selector: format!("{}:nth-of-type({})", tag, elements.len() + 1),
332                tag: tag.to_string(),
333                text,
334                attributes: HashMap::new(),
335                visible: true,
336                enabled: true,
337                role: None,
338                label: None,
339            });
340
341            start = abs_pos + end + 1;
342        }
343    }
344
345    if let Some(ref text_query) = query.text {
346        // Search for text content
347        let lower_html = html.to_lowercase();
348        let lower_query = text_query.to_lowercase();
349        if lower_html.contains(&lower_query) {
350            elements.push(ElementInfo {
351                selector: format!("*:contains(\"{}\")", text_query),
352                tag: "*".to_string(),
353                text: text_query.clone(),
354                attributes: HashMap::new(),
355                visible: true,
356                enabled: true,
357                role: None,
358                label: None,
359            });
360        }
361    }
362
363    elements
364}
rdesktop_dev/agent_api.rs

rdesktop_dev/
agent_api.rs
