rdesktop_dev/
agent_api.rs

1//! Agent API endpoints.
2//!
3//! These endpoints allow AI agents to interact with the running application
4//! through structured HTTP requests, without needing native desktop control.
5//!
6//! ## How it works
7//!
8//! 1. The browser app includes the rdesktop bridge script
9//! 2. The bridge script periodically sends DOM snapshots to the server
10//! 3. Agents query the server for DOM/state information
11//! 4. Agents send actions to the server, which forwards them to the browser
12//!
13//! ## Design Philosophy
14//!
15//! Instead of requiring agents to take screenshots and use vision models to
16//! understand the UI, the Agent API provides direct DOM access and structured
17//! state information. This is:
18//!
19//! - **Faster**: No screenshot encoding/decoding overhead
20//! - **More reliable**: Exact element selectors, not pixel coordinates
21//! - **More informative**: Full DOM tree, computed styles, accessibility info
22//! - **Easier to test**: Standard HTTP endpoints, can be scripted
23
24use axum::extract::{Query, State};
25use axum::http::StatusCode;
26use axum::response::IntoResponse;
27use axum::Json;
28use serde::{Deserialize, Serialize};
29use std::collections::HashMap;
30
31use crate::server::DevServerState;
32
33/// Query parameters for element selection.
34#[derive(Debug, Deserialize)]
35pub struct ElementQuery {
36    /// CSS selector to query elements
37    pub selector: Option<String>,
38
39    /// Text content to search for
40    pub text: Option<String>,
41
42    /// Role attribute to filter by
43    pub role: Option<String>,
44}
45
46/// An action that an agent can execute on the UI.
47#[derive(Debug, Deserialize, Serialize)]
48pub struct AgentAction {
49    /// The type of action
50    pub action: ActionType,
51
52    /// CSS selector of the target element
53    pub selector: String,
54
55    /// Value for type/fill actions
56    pub value: Option<String>,
57
58    /// Coordinates for scroll actions
59    pub coordinates: Option<(f64, f64)>,
60}
61
62/// Types of actions agents can execute.
63#[derive(Debug, Deserialize, Serialize)]
64#[serde(rename_all = "snake_case")]
65pub enum ActionType {
66    Click,
67    DoubleClick,
68    RightClick,
69    Type,
70    Fill,
71    Clear,
72    Scroll,
73    Hover,
74    Focus,
75    Select,
76}
77
78/// Response from a DOM query.
79#[derive(Debug, Serialize)]
80pub struct DomSnapshot {
81    /// The full HTML content
82    pub html: String,
83
84    /// The page URL
85    pub url: String,
86
87    /// The page title
88    pub title: String,
89
90    /// Timestamp of the snapshot
91    pub timestamp: String,
92}
93
94/// Response from an element query.
95#[derive(Debug, Serialize)]
96pub struct ElementInfo {
97    /// CSS selector that uniquely identifies this element
98    pub selector: String,
99
100    /// Tag name
101    pub tag: String,
102
103    /// Text content
104    pub text: String,
105
106    /// Element attributes
107    pub attributes: HashMap<String, String>,
108
109    /// Whether the element is visible
110    pub visible: bool,
111
112    /// Whether the element is enabled (for interactive elements)
113    pub enabled: bool,
114
115    /// Accessibility role
116    pub role: Option<String>,
117
118    /// Accessibility label
119    pub label: Option<String>,
120}
121
122/// Result of an action execution.
123#[derive(Debug, Serialize)]
124pub struct ActionResult {
125    /// Whether the action succeeded
126    pub success: bool,
127
128    /// Error message if the action failed
129    pub error: Option<String>,
130
131    /// Any side effects (e.g., navigation that occurred)
132    pub side_effects: Vec<String>,
133}
134
135/// GET /__rdesktop__/agent/dom
136///
137/// Returns a full DOM snapshot of the current page.
138/// The snapshot is collected from the browser via the bridge script.
139pub async fn get_dom(
140    State(state): State<DevServerState>,
141) -> impl IntoResponse {
142    let snapshot = state.last_dom_snapshot.read().await;
143
144    let html = snapshot.clone().unwrap_or_else(|| {
145        r#"<!DOCTYPE html>
146<html>
147<head><title>rdesktop</title></head>
148<body>
149  <p>No DOM snapshot available yet. Make sure the app is loaded in the browser.</p>
150  <p>The bridge script will send DOM updates automatically.</p>
151</body>
152</html>"#
153            .to_string()
154    });
155
156    let dom = DomSnapshot {
157        html,
158        url: "http://localhost".to_string(),
159        title: "rdesktop App".to_string(),
160        timestamp: timestamp(),
161    };
162
163    Json(dom).into_response()
164}
165
166/// GET /__rdesktop__/agent/elements?selector=...
167///
168/// Query elements matching a CSS selector or text content.
169pub async fn query_elements(
170    State(state): State<DevServerState>,
171    Query(query): Query<ElementQuery>,
172) -> impl IntoResponse {
173    let snapshot = state.last_dom_snapshot.read().await;
174
175    // Parse the DOM and find matching elements
176    let elements: Vec<ElementInfo> = if let Some(ref html) = *snapshot {
177        find_elements(html, &query)
178    } else {
179        vec![]
180    };
181
182    Json(serde_json::json!({
183        "query": {
184            "selector": query.selector,
185            "text": query.text,
186            "role": query.role,
187        },
188        "count": elements.len(),
189        "elements": elements,
190    }))
191    .into_response()
192}
193
194/// POST /__rdesktop__/agent/action
195///
196/// Execute a UI action (click, type, scroll, etc.)
197/// The action is stored and picked up by the bridge script.
198pub async fn execute_action(
199    State(_state): State<DevServerState>,
200    Json(action): Json<AgentAction>,
201) -> impl IntoResponse {
202    tracing::info!(
203        action = ?action.action,
204        selector = %action.selector,
205        "Agent action received"
206    );
207
208    // In a full implementation, this would:
209    // 1. Store the action in a shared queue
210    // 2. The bridge script polls for pending actions
211    // 3. The bridge executes the action in the browser
212    // 4. The result is returned
213
214    let result = ActionResult {
215        success: true,
216        error: None,
217        side_effects: vec![format!("Action {:?} on '{}' queued", action.action, action.selector)],
218    };
219
220    Json(result).into_response()
221}
222
223/// GET /__rdesktop__/agent/state
224///
225/// Get the current application state.
226pub async fn get_state(
227    State(state): State<DevServerState>,
228) -> impl IntoResponse {
229    let app_state = state.last_app_state.read().await;
230
231    match app_state.as_ref() {
232        Some(state) => Json(state.clone()).into_response(),
233        None => Json(serde_json::json!({
234            "message": "No application state available yet.",
235            "hint": "Use fetch('/__rdesktop__/state', { method: 'POST', body: JSON.stringify(state) }) from your app."
236        }))
237        .into_response(),
238    }
239}
240
241/// POST /__rdesktop__/agent/ipc
242///
243/// Send an IPC message from the agent to the app.
244pub async fn send_ipc(
245    State(_state): State<DevServerState>,
246    Json(message): Json<serde_json::Value>,
247) -> impl IntoResponse {
248    let cmd = message["cmd"].as_str().unwrap_or("unknown");
249    let payload = message["payload"].clone();
250    let id = message["id"].as_str().unwrap_or("0");
251
252    tracing::info!(cmd = cmd, "Agent IPC message received");
253
254    // In a full implementation, this would forward to the Rust IPC handler.
255    // For now, handle basic commands directly.
256    let response = match cmd {
257        "greet" => {
258            let name = payload["name"].as_str().unwrap_or("World");
259            serde_json::json!({
260                "id": id,
261                "success": true,
262                "data": { "message": format!("Hello, {}!", name) }
263            })
264        }
265        "ping" => {
266            serde_json::json!({
267                "id": id,
268                "success": true,
269                "data": { "pong": true }
270            })
271        }
272        _ => {
273            serde_json::json!({
274                "id": id,
275                "success": false,
276                "data": { "error": format!("Unknown command: {}", cmd) }
277            })
278        }
279    };
280
281    Json(response).into_response()
282}
283
284/// GET /__rdesktop__/agent/screenshot
285///
286/// Capture a screenshot. In browser mode, this delegates to the browser.
287pub async fn take_screenshot(
288    State(_state): State<DevServerState>,
289) -> impl IntoResponse {
290    (
291        StatusCode::NOT_IMPLEMENTED,
292        Json(serde_json::json!({
293            "message": "Screenshot not implemented in browser mode.",
294            "hint": "Use Playwright's page.screenshot() directly."
295        })),
296    )
297        .into_response()
298}
299
300/// Simple timestamp helper.
301fn timestamp() -> String {
302    let now = std::time::SystemTime::now()
303        .duration_since(std::time::UNIX_EPOCH)
304        .unwrap_or_default();
305    format!("{}", now.as_secs())
306}
307
308/// Find elements in HTML matching the query.
309/// This is a simple text-based search, not a full DOM parser.
310fn find_elements(html: &str, query: &ElementQuery) -> Vec<ElementInfo> {
311    let mut elements = vec![];
312
313    if let Some(ref selector) = query.selector {
314        // Simple tag selector matching (e.g., "button", "input", "h1")
315        let tag = selector.trim_start_matches('<').trim_end_matches('>');
316        let open_tag = format!("<{}", tag);
317
318        let mut start = 0;
319        while let Some(pos) = html[start..].find(&open_tag) {
320            let abs_pos = start + pos;
321            let end = html[abs_pos..].find('>').unwrap_or(0);
322            let _tag_content = &html[abs_pos..abs_pos + end + 1];
323
324            // Extract text content between tags
325            let close_tag = format!("</{}>", tag);
326            let text_start = abs_pos + end + 1;
327            let text = if let Some(text_end) = html[text_start..].find(&close_tag) {
328                html[text_start..text_start + text_end].trim().to_string()
329            } else {
330                String::new()
331            };
332
333            elements.push(ElementInfo {
334                selector: format!("{}:nth-of-type({})", tag, elements.len() + 1),
335                tag: tag.to_string(),
336                text,
337                attributes: HashMap::new(),
338                visible: true,
339                enabled: true,
340                role: None,
341                label: None,
342            });
343
344            start = abs_pos + end + 1;
345        }
346    }
347
348    if let Some(ref text_query) = query.text {
349        // Search for text content
350        let lower_html = html.to_lowercase();
351        let lower_query = text_query.to_lowercase();
352        if lower_html.contains(&lower_query) {
353            elements.push(ElementInfo {
354                selector: format!("*:contains(\"{}\")", text_query),
355                tag: "*".to_string(),
356                text: text_query.clone(),
357                attributes: HashMap::new(),
358                visible: true,
359                enabled: true,
360                role: None,
361                label: None,
362            });
363        }
364    }
365
366    elements
367}
rdesktop_dev/agent_api.rs

rdesktop_dev/
agent_api.rs
