Skip to main content

gpui_mcp/
lib.rs

1#![doc = include_str!("../README.md")]
2
3//! GPUI semantic inspection state and authenticated loopback transport.
4//!
5//! The listener is only constructed by opt-in inspection builds. It never
6//! executes application behavior itself: actions cross a bounded channel and
7//! are handled by the GPUI foreground executor.
8
9mod selector;
10
11pub use selector::{SemanticSelector, SemanticSelectorError};
12
13use std::{
14    io::{BufRead, BufReader, Read, Write},
15    net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, TcpStream},
16    sync::{Arc, Condvar, Mutex, mpsc},
17    thread,
18    time::{Duration, Instant},
19};
20
21use serde::{Deserialize, Serialize};
22
23const MAX_REQUEST_BYTES: u64 = 64 * 1024;
24const ACTION_TIMEOUT: Duration = Duration::from_secs(3);
25const SCREENSHOT_TIMEOUT: Duration = Duration::from_secs(15);
26const RESIZE_SETTLE_QUIET: Duration = Duration::from_millis(75);
27const RESIZE_NO_CHANGE_GRACE: Duration = Duration::from_millis(250);
28const MAX_LOGICAL_WINDOW_SIZE: u32 = 8_192;
29const MAX_KEY_NAME_BYTES: usize = 64;
30const MAX_TYPED_TEXT_BYTES: usize = 4_096;
31const MAX_SCREENSHOT_NAME_BYTES: usize = 64;
32const ROOT_NODE_ROLE: &str = "application";
33
34/// One normalized rectangle in logical pixels.
35#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
36pub struct SemanticBounds {
37    pub x: f32,
38    pub y: f32,
39    pub width: f32,
40    pub height: f32,
41}
42
43/// One framework-neutral node exposed to inspection clients.
44#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
45pub struct SemanticNode {
46    pub id: String,
47    pub role: String,
48    pub label: String,
49    pub value: Option<String>,
50    pub bounds: Option<SemanticBounds>,
51    pub focused: bool,
52    pub actions: Vec<String>,
53}
54
55/// Immutable semantic snapshot published by the UI.
56///
57/// Exactly one node should have the `application` role and logical bounds.
58/// Resize and screenshot operations treat that node as the semantic root.
59#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
60pub struct SemanticTree {
61    pub revision: u64,
62    pub nodes: Vec<SemanticNode>,
63}
64
65#[derive(Default)]
66struct RegistryState {
67    tree: Mutex<Arc<SemanticTree>>,
68    changed: Condvar,
69}
70
71/// Latest-value semantic registry. Reads never wait for GPUI.
72#[derive(Clone, Default)]
73pub struct InspectionRegistry(Arc<RegistryState>);
74
75impl InspectionRegistry {
76    pub fn publish(&self, tree: SemanticTree) {
77        let mut current = self
78            .0
79            .tree
80            .lock()
81            .unwrap_or_else(|error| error.into_inner());
82        if current.as_ref() == &tree {
83            return;
84        }
85        *current = Arc::new(tree);
86        drop(current);
87        self.0.changed.notify_all();
88    }
89
90    #[must_use]
91    pub fn snapshot(&self) -> Arc<SemanticTree> {
92        self.0
93            .tree
94            .lock()
95            .unwrap_or_else(|error| error.into_inner())
96            .clone()
97    }
98
99    #[must_use]
100    pub fn node(&self, id: &str) -> Option<SemanticNode> {
101        self.snapshot()
102            .nodes
103            .iter()
104            .find(|node| node.id == id)
105            .cloned()
106    }
107
108    #[must_use]
109    pub fn wait_for_value(
110        &self,
111        id: &str,
112        expected: &str,
113        timeout: Duration,
114    ) -> Option<SemanticNode> {
115        let deadline = Instant::now() + timeout;
116        let mut guard = self
117            .0
118            .tree
119            .lock()
120            .unwrap_or_else(|error| error.into_inner());
121        loop {
122            if let Some(node) = guard
123                .nodes
124                .iter()
125                .find(|node| node.id == id)
126                .filter(|node| node.value.as_deref() == Some(expected))
127                .cloned()
128            {
129                return Some(node);
130            }
131            let remaining = deadline.saturating_duration_since(Instant::now());
132            if remaining.is_zero() {
133                return None;
134            }
135            let waited = self
136                .0
137                .changed
138                .wait_timeout(guard, remaining)
139                .unwrap_or_else(|error| error.into_inner());
140            guard = waited.0;
141            if waited.1.timed_out() {
142                return guard
143                    .nodes
144                    .iter()
145                    .find(|node| node.id == id)
146                    .filter(|node| node.value.as_deref() == Some(expected))
147                    .cloned();
148            }
149        }
150    }
151
152    #[must_use]
153    pub fn wait_for_focus(
154        &self,
155        id: &str,
156        expected: bool,
157        timeout: Duration,
158    ) -> Option<SemanticNode> {
159        let deadline = Instant::now() + timeout;
160        let mut guard = self
161            .0
162            .tree
163            .lock()
164            .unwrap_or_else(|error| error.into_inner());
165        loop {
166            if let Some(node) = guard
167                .nodes
168                .iter()
169                .find(|node| node.id == id && node.focused == expected)
170                .cloned()
171            {
172                return Some(node);
173            }
174            let remaining = deadline.saturating_duration_since(Instant::now());
175            if remaining.is_zero() {
176                return None;
177            }
178            let waited = self
179                .0
180                .changed
181                .wait_timeout(guard, remaining)
182                .unwrap_or_else(|error| error.into_inner());
183            guard = waited.0;
184            if waited.1.timed_out() {
185                return guard
186                    .nodes
187                    .iter()
188                    .find(|node| node.id == id && node.focused == expected)
189                    .cloned();
190            }
191        }
192    }
193
194    #[must_use]
195    pub fn wait_for_root_bounds_settled(
196        &self,
197        requested_width: u32,
198        requested_height: u32,
199        initial: SemanticBounds,
200        timeout: Duration,
201    ) -> Option<SemanticBounds> {
202        let started = Instant::now();
203        let deadline = Instant::now() + timeout;
204        let mut guard = self
205            .0
206            .tree
207            .lock()
208            .unwrap_or_else(|error| error.into_inner());
209        let mut latest = root_bounds(&guard)?;
210        let mut last_change = Instant::now();
211        let mut observed_response = latest != initial
212            || (latest.width == requested_width as f32 && latest.height == requested_height as f32);
213        loop {
214            let now = Instant::now();
215            let quiet_remaining =
216                RESIZE_SETTLE_QUIET.saturating_sub(now.saturating_duration_since(last_change));
217            if quiet_remaining.is_zero()
218                && (observed_response
219                    || now.saturating_duration_since(started) >= RESIZE_NO_CHANGE_GRACE)
220            {
221                return Some(latest);
222            }
223            let deadline_remaining = deadline.saturating_duration_since(now);
224            if deadline_remaining.is_zero() {
225                return None;
226            }
227            let wait_duration = if observed_response {
228                quiet_remaining
229            } else {
230                quiet_remaining.max(
231                    RESIZE_NO_CHANGE_GRACE.saturating_sub(now.saturating_duration_since(started)),
232                )
233            };
234            let waited = self
235                .0
236                .changed
237                .wait_timeout(guard, wait_duration.min(deadline_remaining))
238                .unwrap_or_else(|error| error.into_inner());
239            guard = waited.0;
240            let current = root_bounds(&guard)?;
241            if current != latest {
242                latest = current;
243                last_change = Instant::now();
244                observed_response = true;
245            }
246        }
247    }
248}
249
250fn root_bounds(tree: &SemanticTree) -> Option<SemanticBounds> {
251    tree.nodes
252        .iter()
253        .find(|node| node.role.eq_ignore_ascii_case(ROOT_NODE_ROLE))
254        .and_then(|node| node.bounds)
255}
256
257fn pointer_position(
258    registry: &InspectionRegistry,
259    id: Option<&str>,
260    x: Option<f32>,
261    y: Option<f32>,
262) -> Result<SemanticPoint, String> {
263    match (id, x, y) {
264        (Some(id), None, None) => {
265            let node = registry
266                .node(id)
267                .ok_or_else(|| format!("unknown semantic node '{id}'"))?;
268            let bounds = node
269                .bounds
270                .ok_or_else(|| format!("semantic node '{id}' has no published bounds"))?;
271            Ok(SemanticPoint {
272                x: bounds.x + bounds.width / 2.0,
273                y: bounds.y + bounds.height / 2.0,
274            })
275        }
276        (None, Some(x), Some(y)) if x.is_finite() && y.is_finite() && x >= 0.0 && y >= 0.0 => {
277            Ok(SemanticPoint { x, y })
278        }
279        (Some(_), Some(_), Some(_)) => {
280            Err("pointer target must use either a semantic id or logical coordinates".to_owned())
281        }
282        _ => Err("pointer target requires a semantic id or both logical coordinates".to_owned()),
283    }
284}
285
286#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
287pub struct SemanticPoint {
288    pub x: f32,
289    pub y: f32,
290}
291
292/// An action that must be executed on the UI thread.
293pub struct ActionRequest {
294    pub action: InspectionAction,
295    pub response: mpsc::SyncSender<Result<InspectionActionResult, String>>,
296}
297
298#[derive(Clone, Debug, PartialEq)]
299pub enum InspectionActionResult {
300    Complete,
301    Screenshot {
302        path: String,
303        width: u32,
304        height: u32,
305    },
306}
307
308#[derive(Clone, Debug, PartialEq)]
309pub enum InspectionAction {
310    Click {
311        id: String,
312    },
313    Resize {
314        width: u32,
315        height: u32,
316    },
317    Focus {
318        id: String,
319    },
320    Key {
321        key: String,
322        modifiers: NormalizedModifiers,
323    },
324    TypeText {
325        text: String,
326    },
327    Scroll {
328        position: SemanticPoint,
329        delta_x: f32,
330        delta_y: f32,
331        modifiers: NormalizedModifiers,
332    },
333    Drag {
334        from: SemanticPoint,
335        to: SemanticPoint,
336    },
337    Screenshot {
338        path: String,
339        width: u32,
340        height: u32,
341    },
342}
343
344#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
345pub struct NormalizedModifiers {
346    pub platform: bool,
347    pub control: bool,
348    pub alt: bool,
349    pub shift: bool,
350    pub function: bool,
351}
352
353#[derive(Clone, Debug, Serialize, Deserialize)]
354#[serde(tag = "kind", rename_all = "snake_case")]
355pub enum InspectionRequest {
356    Tree,
357    Query {
358        id: String,
359    },
360    Click {
361        id: String,
362    },
363    Wait {
364        id: String,
365        value: String,
366        timeout_ms: u64,
367    },
368    Resize {
369        width: u32,
370        height: u32,
371        timeout_ms: u64,
372    },
373    Focus {
374        id: String,
375        timeout_ms: u64,
376    },
377    Key {
378        key: String,
379        modifiers: NormalizedModifiers,
380    },
381    TypeText {
382        text: String,
383    },
384    Scroll {
385        id: Option<String>,
386        x: Option<f32>,
387        y: Option<f32>,
388        delta_x: f32,
389        delta_y: f32,
390        #[serde(default)]
391        modifiers: NormalizedModifiers,
392    },
393    Drag {
394        from_id: Option<String>,
395        from_x: Option<f32>,
396        from_y: Option<f32>,
397        to_id: Option<String>,
398        to_x: Option<f32>,
399        to_y: Option<f32>,
400    },
401    Screenshot {
402        name: String,
403    },
404}
405
406#[derive(Clone, Debug, Serialize, Deserialize)]
407pub struct AuthenticatedRequest {
408    pub token: String,
409    pub request: InspectionRequest,
410}
411
412#[derive(Clone, Debug, Serialize, Deserialize)]
413pub struct InspectionResponse {
414    pub ok: bool,
415    pub result: Option<serde_json::Value>,
416    pub error: Option<String>,
417}
418
419impl InspectionResponse {
420    fn success(value: impl Serialize) -> Self {
421        match serde_json::to_value(value) {
422            Ok(result) => Self {
423                ok: true,
424                result: Some(result),
425                error: None,
426            },
427            Err(error) => Self::failure(format!("serialize response: {error}")),
428        }
429    }
430
431    fn failure(error: impl Into<String>) -> Self {
432        Self {
433            ok: false,
434            result: None,
435            error: Some(error.into()),
436        }
437    }
438}
439
440/// Running loopback inspection endpoint.
441pub struct InspectionServer {
442    address: SocketAddr,
443    token: String,
444}
445
446impl InspectionServer {
447    pub fn start(
448        port: u16,
449        registry: InspectionRegistry,
450        actions: async_channel::Sender<ActionRequest>,
451    ) -> Result<Self, String> {
452        let listener = TcpListener::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port))
453            .map_err(|error| format!("bind GPUI inspection listener: {error}"))?;
454        let address = listener
455            .local_addr()
456            .map_err(|error| format!("read GPUI inspection address: {error}"))?;
457        let token = uuid::Uuid::new_v4().simple().to_string();
458        let server_token = token.clone();
459        thread::Builder::new()
460            .name("gpui-mcp-inspection".to_owned())
461            .spawn(move || serve(listener, &server_token, registry, actions))
462            .map_err(|error| format!("start GPUI inspection listener: {error}"))?;
463        Ok(Self { address, token })
464    }
465
466    #[must_use]
467    pub fn address(&self) -> SocketAddr {
468        self.address
469    }
470
471    #[must_use]
472    pub fn token(&self) -> &str {
473        &self.token
474    }
475}
476
477fn serve(
478    listener: TcpListener,
479    token: &str,
480    registry: InspectionRegistry,
481    actions: async_channel::Sender<ActionRequest>,
482) {
483    for stream in listener.incoming() {
484        let Ok(stream) = stream else { continue };
485        let _ = stream.set_read_timeout(Some(Duration::from_secs(2)));
486        let _ = stream.set_write_timeout(Some(Duration::from_secs(2)));
487        let response = handle_stream(stream.try_clone().ok(), token, &registry, &actions);
488        if let Ok(mut stream) = stream.try_clone() {
489            let _ = write_response(&mut stream, &response);
490        }
491    }
492}
493
494fn handle_stream(
495    stream: Option<TcpStream>,
496    token: &str,
497    registry: &InspectionRegistry,
498    actions: &async_channel::Sender<ActionRequest>,
499) -> InspectionResponse {
500    let Some(stream) = stream else {
501        return InspectionResponse::failure("clone inspection stream");
502    };
503    let mut line = String::new();
504    if let Err(error) = BufReader::new(stream)
505        .take(MAX_REQUEST_BYTES)
506        .read_line(&mut line)
507    {
508        return InspectionResponse::failure(format!("read request: {error}"));
509    }
510    let request: AuthenticatedRequest = match serde_json::from_str(&line) {
511        Ok(request) => request,
512        Err(error) => return InspectionResponse::failure(format!("invalid request: {error}")),
513    };
514    if request.token != token {
515        return InspectionResponse::failure("authentication failed");
516    }
517    dispatch(request.request, registry, actions)
518}
519
520fn dispatch(
521    request: InspectionRequest,
522    registry: &InspectionRegistry,
523    actions: &async_channel::Sender<ActionRequest>,
524) -> InspectionResponse {
525    match request {
526        InspectionRequest::Tree => InspectionResponse::success(registry.snapshot().as_ref()),
527        InspectionRequest::Query { id } => InspectionResponse::success(registry.node(&id)),
528        InspectionRequest::Wait {
529            id,
530            value,
531            timeout_ms,
532        } => {
533            let timeout = Duration::from_millis(timeout_ms.min(30_000));
534            match registry.wait_for_value(&id, &value, timeout) {
535                Some(node) => InspectionResponse::success(node),
536                None => InspectionResponse::failure(format!(
537                    "timed out waiting for '{id}' to have value '{value}'"
538                )),
539            }
540        }
541        InspectionRequest::Click { id } => {
542            match dispatch_action(actions, InspectionAction::Click { id }) {
543                Ok(Ok(InspectionActionResult::Complete)) => {
544                    InspectionResponse::success(serde_json::json!({ "clicked": true }))
545                }
546                Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
547                    InspectionResponse::failure("GPUI click action returned a screenshot artifact")
548                }
549                Ok(Err(error)) => InspectionResponse::failure(error),
550                Err(_) => InspectionResponse::failure("GPUI action timed out"),
551            }
552        }
553        InspectionRequest::Resize {
554            width,
555            height,
556            timeout_ms,
557        } => {
558            if width == 0
559                || height == 0
560                || width > MAX_LOGICAL_WINDOW_SIZE
561                || height > MAX_LOGICAL_WINDOW_SIZE
562            {
563                return InspectionResponse::failure(format!(
564                    "resize dimensions must be between 1 and {MAX_LOGICAL_WINDOW_SIZE} logical pixels"
565                ));
566            }
567            let Some(initial) = root_bounds(&registry.snapshot()) else {
568                return InspectionResponse::failure(
569                    "GPUI root bounds are unavailable before resize",
570                );
571            };
572            match dispatch_action(actions, InspectionAction::Resize { width, height }) {
573                Ok(Ok(InspectionActionResult::Complete)) => {}
574                Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
575                    return InspectionResponse::failure(
576                        "GPUI resize action returned a screenshot artifact",
577                    );
578                }
579                Ok(Err(error)) => return InspectionResponse::failure(error),
580                Err(_) => return InspectionResponse::failure("GPUI resize action timed out"),
581            }
582            let timeout = Duration::from_millis(timeout_ms.clamp(1, 30_000));
583            match registry.wait_for_root_bounds_settled(width, height, initial, timeout) {
584                Some(actual) => InspectionResponse::success(serde_json::json!({
585                    "requested": { "width": width, "height": height },
586                    "actual": { "width": actual.width, "height": actual.height },
587                    "constrained": actual.width != width as f32 || actual.height != height as f32,
588                    "settled": true
589                })),
590                None => InspectionResponse::failure("timed out waiting for GPUI resize to settle"),
591            }
592        }
593        InspectionRequest::Focus { id, timeout_ms } => {
594            match dispatch_action(actions, InspectionAction::Focus { id: id.clone() }) {
595                Ok(Ok(InspectionActionResult::Complete)) => {}
596                Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
597                    return InspectionResponse::failure(
598                        "GPUI focus action returned a screenshot artifact",
599                    );
600                }
601                Ok(Err(error)) => return InspectionResponse::failure(error),
602                Err(_) => return InspectionResponse::failure("GPUI focus action timed out"),
603            }
604            let timeout = Duration::from_millis(timeout_ms.clamp(1, 30_000));
605            match registry.wait_for_focus(&id, true, timeout) {
606                Some(node) => InspectionResponse::success(node),
607                None => InspectionResponse::failure(format!(
608                    "timed out waiting for '{id}' to receive focus"
609                )),
610            }
611        }
612        InspectionRequest::Key { key, modifiers } => {
613            if key.is_empty() || key.len() > MAX_KEY_NAME_BYTES {
614                return InspectionResponse::failure(format!(
615                    "key must contain between 1 and {MAX_KEY_NAME_BYTES} bytes"
616                ));
617            }
618            match dispatch_action(actions, InspectionAction::Key { key, modifiers }) {
619                Ok(Ok(InspectionActionResult::Complete)) => {
620                    InspectionResponse::success(serde_json::json!({ "pressed": true }))
621                }
622                Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
623                    InspectionResponse::failure("GPUI key action returned a screenshot artifact")
624                }
625                Ok(Err(error)) => InspectionResponse::failure(error),
626                Err(_) => InspectionResponse::failure("GPUI key action timed out"),
627            }
628        }
629        InspectionRequest::TypeText { text } => {
630            if text.len() > MAX_TYPED_TEXT_BYTES {
631                return InspectionResponse::failure(format!(
632                    "text must not exceed {MAX_TYPED_TEXT_BYTES} UTF-8 bytes"
633                ));
634            }
635            let characters = text.chars().count();
636            match dispatch_action(actions, InspectionAction::TypeText { text }) {
637                Ok(Ok(InspectionActionResult::Complete)) => {
638                    InspectionResponse::success(serde_json::json!({
639                        "typed": true,
640                        "characters": characters
641                    }))
642                }
643                Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
644                    InspectionResponse::failure("GPUI typing action returned a screenshot artifact")
645                }
646                Ok(Err(error)) => InspectionResponse::failure(error),
647                Err(_) => InspectionResponse::failure("GPUI typing action timed out"),
648            }
649        }
650        InspectionRequest::Scroll {
651            id,
652            x,
653            y,
654            delta_x,
655            delta_y,
656            modifiers,
657        } => {
658            if !delta_x.is_finite() || !delta_y.is_finite() {
659                return InspectionResponse::failure("scroll deltas must be finite");
660            }
661            let position = match pointer_position(registry, id.as_deref(), x, y) {
662                Ok(position) => position,
663                Err(error) => return InspectionResponse::failure(error),
664            };
665            match dispatch_action(
666                actions,
667                InspectionAction::Scroll {
668                    position,
669                    delta_x,
670                    delta_y,
671                    modifiers,
672                },
673            ) {
674                Ok(Ok(InspectionActionResult::Complete)) => {
675                    InspectionResponse::success(serde_json::json!({
676                        "scrolled": true,
677                        "position": position,
678                        "delta": { "x": delta_x, "y": delta_y },
679                        "modifiers": modifiers
680                    }))
681                }
682                Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
683                    InspectionResponse::failure("GPUI scroll action returned a screenshot artifact")
684                }
685                Ok(Err(error)) => InspectionResponse::failure(error),
686                Err(_) => InspectionResponse::failure("GPUI scroll action timed out"),
687            }
688        }
689        InspectionRequest::Drag {
690            from_id,
691            from_x,
692            from_y,
693            to_id,
694            to_x,
695            to_y,
696        } => {
697            let from = match pointer_position(registry, from_id.as_deref(), from_x, from_y) {
698                Ok(position) => position,
699                Err(error) => {
700                    return InspectionResponse::failure(format!("invalid drag start: {error}"));
701                }
702            };
703            let to = match pointer_position(registry, to_id.as_deref(), to_x, to_y) {
704                Ok(position) => position,
705                Err(error) => {
706                    return InspectionResponse::failure(format!("invalid drag end: {error}"));
707                }
708            };
709            match dispatch_action(actions, InspectionAction::Drag { from, to }) {
710                Ok(Ok(InspectionActionResult::Complete)) => {
711                    InspectionResponse::success(serde_json::json!({
712                        "dragged": true,
713                        "from": from,
714                        "to": to
715                    }))
716                }
717                Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
718                    InspectionResponse::failure("GPUI drag action returned a screenshot artifact")
719                }
720                Ok(Err(error)) => InspectionResponse::failure(error),
721                Err(_) => InspectionResponse::failure("GPUI drag action timed out"),
722            }
723        }
724        InspectionRequest::Screenshot { name } => {
725            if name.is_empty()
726                || name.len() > MAX_SCREENSHOT_NAME_BYTES
727                || !name
728                    .chars()
729                    .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
730                || name.contains("..")
731            {
732                return InspectionResponse::failure(format!(
733                    "screenshot name must contain 1-{MAX_SCREENSHOT_NAME_BYTES} ASCII letters, digits, '-', '_', or '.'"
734                ));
735            }
736            let Some(bounds) = root_bounds(&registry.snapshot()) else {
737                return InspectionResponse::failure(
738                    "GPUI root bounds are unavailable before screenshot capture",
739                );
740            };
741            let width = bounds.width.round() as u32;
742            let height = bounds.height.round() as u32;
743            let directory = std::env::temp_dir().join("gpui-mcp");
744            if let Err(error) = std::fs::create_dir_all(&directory) {
745                return InspectionResponse::failure(format!(
746                    "create screenshot artifact directory: {error}"
747                ));
748            }
749            let path = directory.join(format!("{name}-{}.png", uuid::Uuid::new_v4().simple()));
750            let path = path.to_string_lossy().into_owned();
751            match dispatch_action_with_timeout(
752                actions,
753                InspectionAction::Screenshot {
754                    path: path.clone(),
755                    width,
756                    height,
757                },
758                SCREENSHOT_TIMEOUT,
759            ) {
760                Ok(Ok(InspectionActionResult::Screenshot {
761                    path,
762                    width,
763                    height,
764                })) => InspectionResponse::success(serde_json::json!({
765                    "captured": true,
766                    "path": path,
767                    "width": width,
768                    "height": height,
769                    "coordinate_scale": "logical"
770                })),
771                Ok(Ok(InspectionActionResult::Complete)) => {
772                    InspectionResponse::failure("GPUI screenshot action returned no artifact")
773                }
774                Ok(Err(error)) => InspectionResponse::failure(error),
775                Err(_) => InspectionResponse::failure("GPUI screenshot action timed out"),
776            }
777        }
778    }
779}
780
781fn dispatch_action(
782    actions: &async_channel::Sender<ActionRequest>,
783    action: InspectionAction,
784) -> Result<Result<InspectionActionResult, String>, mpsc::RecvTimeoutError> {
785    dispatch_action_with_timeout(actions, action, ACTION_TIMEOUT)
786}
787
788fn dispatch_action_with_timeout(
789    actions: &async_channel::Sender<ActionRequest>,
790    action: InspectionAction,
791    timeout: Duration,
792) -> Result<Result<InspectionActionResult, String>, mpsc::RecvTimeoutError> {
793    let (response_tx, response_rx) = mpsc::sync_channel(1);
794    if let Err(error) = actions.try_send(ActionRequest {
795        action,
796        response: response_tx,
797    }) {
798        return Ok(Err(format!("GPUI action queue is unavailable: {error}")));
799    }
800    response_rx.recv_timeout(timeout)
801}
802
803fn write_response(stream: &mut TcpStream, response: &InspectionResponse) -> std::io::Result<()> {
804    serde_json::to_writer(&mut *stream, response)?;
805    stream.write_all(b"\n")?;
806    stream.flush()
807}
808
809/// Send one request to a running inspection endpoint.
810pub fn request(
811    address: SocketAddr,
812    token: &str,
813    request: InspectionRequest,
814) -> Result<InspectionResponse, String> {
815    let mut stream = TcpStream::connect_timeout(&address, Duration::from_secs(2))
816        .map_err(|error| format!("connect to {address}: {error}"))?;
817    stream
818        .set_read_timeout(Some(Duration::from_secs(32)))
819        .map_err(|error| format!("configure inspection socket: {error}"))?;
820    serde_json::to_writer(
821        &mut stream,
822        &AuthenticatedRequest {
823            token: token.to_owned(),
824            request,
825        },
826    )
827    .map_err(|error| format!("encode inspection request: {error}"))?;
828    stream
829        .write_all(b"\n")
830        .map_err(|error| format!("write inspection request: {error}"))?;
831    let mut line = String::new();
832    BufReader::new(stream)
833        .read_line(&mut line)
834        .map_err(|error| format!("read inspection response: {error}"))?;
835    serde_json::from_str(&line).map_err(|error| format!("decode inspection response: {error}"))
836}
837
838#[cfg(test)]
839mod tests {
840    use super::*;
841
842    fn node(value: &str) -> SemanticNode {
843        focused_node(value, false)
844    }
845
846    fn focused_node(value: &str, focused: bool) -> SemanticNode {
847        SemanticNode {
848            id: "preview.plugin.editor".to_owned(),
849            role: "button".to_owned(),
850            label: "Plugin editor".to_owned(),
851            value: Some(value.to_owned()),
852            bounds: None,
853            focused,
854            actions: vec!["click".to_owned()],
855        }
856    }
857
858    fn root(width: f32, height: f32) -> SemanticNode {
859        SemanticNode {
860            id: "test.root".to_owned(),
861            role: "application".to_owned(),
862            label: "Preview".to_owned(),
863            value: None,
864            bounds: Some(SemanticBounds {
865                x: 0.0,
866                y: 0.0,
867                width,
868                height,
869            }),
870            focused: false,
871            actions: Vec::new(),
872        }
873    }
874
875    #[test]
876    fn publish_should_replace_latest_snapshot() {
877        let registry = InspectionRegistry::default();
878        registry.publish(SemanticTree {
879            revision: 7,
880            nodes: vec![node("closed")],
881        });
882        assert_eq!(registry.snapshot().revision, 7);
883    }
884
885    #[test]
886    fn wait_for_value_should_observe_a_later_publish() {
887        let registry = InspectionRegistry::default();
888        registry.publish(SemanticTree {
889            revision: 1,
890            nodes: vec![node("opening")],
891        });
892        let publisher = registry.clone();
893        thread::spawn(move || {
894            thread::sleep(Duration::from_millis(10));
895            publisher.publish(SemanticTree {
896                revision: 2,
897                nodes: vec![node("open")],
898            });
899        });
900        assert!(
901            registry
902                .wait_for_value("preview.plugin.editor", "open", Duration::from_secs(1))
903                .is_some()
904        );
905    }
906
907    #[test]
908    fn listener_should_reject_an_invalid_token() {
909        let registry = InspectionRegistry::default();
910        let (action_tx, _action_rx) = async_channel::bounded(1);
911        let server = InspectionServer::start(0, registry, action_tx).expect("start listener");
912        let response = request(server.address(), "wrong", InspectionRequest::Tree)
913            .expect("receive authentication response");
914        assert!(!response.ok);
915    }
916
917    #[test]
918    fn click_and_wait_should_follow_a_published_lifecycle() {
919        let registry = InspectionRegistry::default();
920        registry.publish(SemanticTree {
921            revision: 1,
922            nodes: vec![node("closed")],
923        });
924        let (action_tx, action_rx) = async_channel::bounded(1);
925        let server = InspectionServer::start(0, registry.clone(), action_tx)
926            .expect("start inspection listener");
927        thread::spawn(move || {
928            let action = action_rx.recv_blocking().expect("receive click action");
929            assert_eq!(
930                action.action,
931                InspectionAction::Click {
932                    id: "preview.plugin.editor".to_owned()
933                }
934            );
935            registry.publish(SemanticTree {
936                revision: 2,
937                nodes: vec![node("open")],
938            });
939            action
940                .response
941                .send(Ok(InspectionActionResult::Complete))
942                .expect("acknowledge click");
943        });
944
945        let click = request(
946            server.address(),
947            server.token(),
948            InspectionRequest::Click {
949                id: "preview.plugin.editor".to_owned(),
950            },
951        )
952        .expect("click response");
953        assert!(click.ok);
954        let wait = request(
955            server.address(),
956            server.token(),
957            InspectionRequest::Wait {
958                id: "preview.plugin.editor".to_owned(),
959                value: "open".to_owned(),
960                timeout_ms: 1_000,
961            },
962        )
963        .expect("wait response");
964        assert!(wait.ok);
965    }
966
967    #[test]
968    fn resize_should_wait_for_settled_actual_bounds() {
969        let registry = InspectionRegistry::default();
970        registry.publish(SemanticTree {
971            revision: 1,
972            nodes: vec![root(1024.0, 640.0)],
973        });
974        let (action_tx, action_rx) = async_channel::bounded(1);
975        let server = InspectionServer::start(0, registry.clone(), action_tx)
976            .expect("start inspection listener");
977        thread::spawn(move || {
978            let action = action_rx.recv_blocking().expect("receive resize action");
979            assert_eq!(
980                action.action,
981                InspectionAction::Resize {
982                    width: 1200,
983                    height: 720
984                }
985            );
986            action
987                .response
988                .send(Ok(InspectionActionResult::Complete))
989                .expect("acknowledge resize");
990            registry.publish(SemanticTree {
991                revision: 1,
992                nodes: vec![root(1200.0, 720.0)],
993            });
994        });
995
996        let response = request(
997            server.address(),
998            server.token(),
999            InspectionRequest::Resize {
1000                width: 1200,
1001                height: 720,
1002                timeout_ms: 1_000,
1003            },
1004        )
1005        .expect("resize response");
1006        assert!(response.ok);
1007        let result = response.result.expect("resize result");
1008        assert_eq!(result["actual"]["width"], 1200.0);
1009        assert_eq!(result["actual"]["height"], 720.0);
1010        assert_eq!(result["constrained"], false);
1011        assert_eq!(result["settled"], true);
1012    }
1013
1014    #[test]
1015    fn resize_should_report_a_constrained_no_op() {
1016        let registry = InspectionRegistry::default();
1017        registry.publish(SemanticTree {
1018            revision: 1,
1019            nodes: vec![root(1024.0, 640.0)],
1020        });
1021        let (action_tx, action_rx) = async_channel::bounded(1);
1022        let server =
1023            InspectionServer::start(0, registry, action_tx).expect("start inspection listener");
1024        thread::spawn(move || {
1025            let action = action_rx.recv_blocking().expect("receive resize action");
1026            action
1027                .response
1028                .send(Ok(InspectionActionResult::Complete))
1029                .expect("acknowledge resize");
1030        });
1031
1032        let response = request(
1033            server.address(),
1034            server.token(),
1035            InspectionRequest::Resize {
1036                width: 800,
1037                height: 500,
1038                timeout_ms: 1_000,
1039            },
1040        )
1041        .expect("resize response");
1042        assert!(response.ok);
1043        assert_eq!(response.result.expect("resize result")["constrained"], true);
1044    }
1045
1046    #[test]
1047    fn resize_should_reject_invalid_dimensions_before_enqueuing() {
1048        let registry = InspectionRegistry::default();
1049        let (action_tx, action_rx) = async_channel::bounded(1);
1050        let response = dispatch(
1051            InspectionRequest::Resize {
1052                width: 0,
1053                height: 640,
1054                timeout_ms: 1_000,
1055            },
1056            &registry,
1057            &action_tx,
1058        );
1059
1060        assert!(!response.ok);
1061        assert!(action_rx.is_empty());
1062    }
1063
1064    #[test]
1065    fn focus_should_wait_for_semantic_publication() {
1066        let registry = InspectionRegistry::default();
1067        registry.publish(SemanticTree {
1068            revision: 1,
1069            nodes: vec![focused_node("closed", false)],
1070        });
1071        let (action_tx, action_rx) = async_channel::bounded(1);
1072        let server = InspectionServer::start(0, registry.clone(), action_tx)
1073            .expect("start inspection listener");
1074        thread::spawn(move || {
1075            let action = action_rx.recv_blocking().expect("receive focus action");
1076            assert_eq!(
1077                action.action,
1078                InspectionAction::Focus {
1079                    id: "preview.plugin.editor".to_owned()
1080                }
1081            );
1082            registry.publish(SemanticTree {
1083                revision: 1,
1084                nodes: vec![focused_node("closed", true)],
1085            });
1086            action
1087                .response
1088                .send(Ok(InspectionActionResult::Complete))
1089                .expect("acknowledge focus");
1090        });
1091
1092        let response = request(
1093            server.address(),
1094            server.token(),
1095            InspectionRequest::Focus {
1096                id: "preview.plugin.editor".to_owned(),
1097                timeout_ms: 1_000,
1098            },
1099        )
1100        .expect("focus response");
1101        assert!(response.ok);
1102        assert_eq!(response.result.expect("focused node")["focused"], true);
1103    }
1104
1105    #[test]
1106    fn keyboard_inputs_should_be_bounded_before_enqueuing() {
1107        let registry = InspectionRegistry::default();
1108        let (action_tx, action_rx) = async_channel::bounded(1);
1109        let empty_key = dispatch(
1110            InspectionRequest::Key {
1111                key: String::new(),
1112                modifiers: NormalizedModifiers::default(),
1113            },
1114            &registry,
1115            &action_tx,
1116        );
1117        let oversized_text = dispatch(
1118            InspectionRequest::TypeText {
1119                text: "x".repeat(MAX_TYPED_TEXT_BYTES + 1),
1120            },
1121            &registry,
1122            &action_tx,
1123        );
1124
1125        assert!(!empty_key.ok);
1126        assert!(!oversized_text.ok);
1127        assert!(action_rx.is_empty());
1128    }
1129
1130    #[test]
1131    fn pointer_targets_should_resolve_semantic_centers_or_coordinates() {
1132        let registry = InspectionRegistry::default();
1133        registry.publish(SemanticTree {
1134            revision: 1,
1135            nodes: vec![root(1024.0, 640.0)],
1136        });
1137
1138        assert_eq!(
1139            pointer_position(&registry, Some("test.root"), None, None),
1140            Ok(SemanticPoint { x: 512.0, y: 320.0 })
1141        );
1142        assert_eq!(
1143            pointer_position(&registry, None, Some(12.0), Some(34.0)),
1144            Ok(SemanticPoint { x: 12.0, y: 34.0 })
1145        );
1146        assert!(pointer_position(&registry, None, Some(12.0), None).is_err());
1147        assert!(pointer_position(&registry, Some("test.root"), Some(1.0), Some(2.0)).is_err());
1148    }
1149
1150    #[test]
1151    fn screenshot_should_use_root_logical_dimensions_and_generated_path() {
1152        let registry = InspectionRegistry::default();
1153        registry.publish(SemanticTree {
1154            revision: 1,
1155            nodes: vec![root(1024.0, 640.0)],
1156        });
1157        let (action_tx, action_rx) = async_channel::bounded(1);
1158        let server =
1159            InspectionServer::start(0, registry, action_tx).expect("start inspection listener");
1160        thread::spawn(move || {
1161            let action = action_rx
1162                .recv_blocking()
1163                .expect("receive screenshot action");
1164            let InspectionAction::Screenshot {
1165                path,
1166                width,
1167                height,
1168            } = action.action
1169            else {
1170                panic!("expected screenshot action")
1171            };
1172            assert_eq!((width, height), (1024, 640));
1173            assert!(path.ends_with(".png"));
1174            action
1175                .response
1176                .send(Ok(InspectionActionResult::Screenshot {
1177                    path,
1178                    width,
1179                    height,
1180                }))
1181                .expect("acknowledge screenshot");
1182        });
1183
1184        let response = request(
1185            server.address(),
1186            server.token(),
1187            InspectionRequest::Screenshot {
1188                name: "logical-home".to_owned(),
1189            },
1190        )
1191        .expect("screenshot response");
1192        assert!(response.ok);
1193        let result = response.result.expect("screenshot result");
1194        assert_eq!(result["width"], 1024);
1195        assert_eq!(result["height"], 640);
1196        assert_eq!(result["coordinate_scale"], "logical");
1197    }
1198}