1#![doc = include_str!("../README.md")]
2
3mod selector;
10
11pub use selector::{SemanticSelector, SemanticSelectorError};
12
13use std::{
14 io::{BufRead, BufReader, Read, Write},
15 net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, TcpStream},
16 sync::{Arc, Condvar, Mutex, mpsc},
17 thread,
18 time::{Duration, Instant},
19};
20
21use serde::{Deserialize, Serialize};
22
23const MAX_REQUEST_BYTES: u64 = 64 * 1024;
24const ACTION_TIMEOUT: Duration = Duration::from_secs(3);
25const SCREENSHOT_TIMEOUT: Duration = Duration::from_secs(15);
26const RESIZE_SETTLE_QUIET: Duration = Duration::from_millis(75);
27const RESIZE_NO_CHANGE_GRACE: Duration = Duration::from_millis(250);
28const MAX_LOGICAL_WINDOW_SIZE: u32 = 8_192;
29const MAX_KEY_NAME_BYTES: usize = 64;
30const MAX_TYPED_TEXT_BYTES: usize = 4_096;
31const MAX_SCREENSHOT_NAME_BYTES: usize = 64;
32const ROOT_NODE_ROLE: &str = "application";
33
34#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
36pub struct SemanticBounds {
37 pub x: f32,
38 pub y: f32,
39 pub width: f32,
40 pub height: f32,
41}
42
43#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
45pub struct SemanticNode {
46 pub id: String,
47 pub role: String,
48 pub label: String,
49 pub value: Option<String>,
50 pub bounds: Option<SemanticBounds>,
51 pub focused: bool,
52 pub actions: Vec<String>,
53}
54
55#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
60pub struct SemanticTree {
61 pub revision: u64,
62 pub nodes: Vec<SemanticNode>,
63}
64
65#[derive(Default)]
66struct RegistryState {
67 tree: Mutex<Arc<SemanticTree>>,
68 changed: Condvar,
69}
70
71#[derive(Clone, Default)]
73pub struct InspectionRegistry(Arc<RegistryState>);
74
75impl InspectionRegistry {
76 pub fn publish(&self, tree: SemanticTree) {
77 let mut current = self
78 .0
79 .tree
80 .lock()
81 .unwrap_or_else(|error| error.into_inner());
82 if current.as_ref() == &tree {
83 return;
84 }
85 *current = Arc::new(tree);
86 drop(current);
87 self.0.changed.notify_all();
88 }
89
90 #[must_use]
91 pub fn snapshot(&self) -> Arc<SemanticTree> {
92 self.0
93 .tree
94 .lock()
95 .unwrap_or_else(|error| error.into_inner())
96 .clone()
97 }
98
99 #[must_use]
100 pub fn node(&self, id: &str) -> Option<SemanticNode> {
101 self.snapshot()
102 .nodes
103 .iter()
104 .find(|node| node.id == id)
105 .cloned()
106 }
107
108 #[must_use]
109 pub fn wait_for_value(
110 &self,
111 id: &str,
112 expected: &str,
113 timeout: Duration,
114 ) -> Option<SemanticNode> {
115 let deadline = Instant::now() + timeout;
116 let mut guard = self
117 .0
118 .tree
119 .lock()
120 .unwrap_or_else(|error| error.into_inner());
121 loop {
122 if let Some(node) = guard
123 .nodes
124 .iter()
125 .find(|node| node.id == id)
126 .filter(|node| node.value.as_deref() == Some(expected))
127 .cloned()
128 {
129 return Some(node);
130 }
131 let remaining = deadline.saturating_duration_since(Instant::now());
132 if remaining.is_zero() {
133 return None;
134 }
135 let waited = self
136 .0
137 .changed
138 .wait_timeout(guard, remaining)
139 .unwrap_or_else(|error| error.into_inner());
140 guard = waited.0;
141 if waited.1.timed_out() {
142 return guard
143 .nodes
144 .iter()
145 .find(|node| node.id == id)
146 .filter(|node| node.value.as_deref() == Some(expected))
147 .cloned();
148 }
149 }
150 }
151
152 #[must_use]
153 pub fn wait_for_focus(
154 &self,
155 id: &str,
156 expected: bool,
157 timeout: Duration,
158 ) -> Option<SemanticNode> {
159 let deadline = Instant::now() + timeout;
160 let mut guard = self
161 .0
162 .tree
163 .lock()
164 .unwrap_or_else(|error| error.into_inner());
165 loop {
166 if let Some(node) = guard
167 .nodes
168 .iter()
169 .find(|node| node.id == id && node.focused == expected)
170 .cloned()
171 {
172 return Some(node);
173 }
174 let remaining = deadline.saturating_duration_since(Instant::now());
175 if remaining.is_zero() {
176 return None;
177 }
178 let waited = self
179 .0
180 .changed
181 .wait_timeout(guard, remaining)
182 .unwrap_or_else(|error| error.into_inner());
183 guard = waited.0;
184 if waited.1.timed_out() {
185 return guard
186 .nodes
187 .iter()
188 .find(|node| node.id == id && node.focused == expected)
189 .cloned();
190 }
191 }
192 }
193
194 #[must_use]
195 pub fn wait_for_root_bounds_settled(
196 &self,
197 requested_width: u32,
198 requested_height: u32,
199 initial: SemanticBounds,
200 timeout: Duration,
201 ) -> Option<SemanticBounds> {
202 let started = Instant::now();
203 let deadline = Instant::now() + timeout;
204 let mut guard = self
205 .0
206 .tree
207 .lock()
208 .unwrap_or_else(|error| error.into_inner());
209 let mut latest = root_bounds(&guard)?;
210 let mut last_change = Instant::now();
211 let mut observed_response = latest != initial
212 || (latest.width == requested_width as f32 && latest.height == requested_height as f32);
213 loop {
214 let now = Instant::now();
215 let quiet_remaining =
216 RESIZE_SETTLE_QUIET.saturating_sub(now.saturating_duration_since(last_change));
217 if quiet_remaining.is_zero()
218 && (observed_response
219 || now.saturating_duration_since(started) >= RESIZE_NO_CHANGE_GRACE)
220 {
221 return Some(latest);
222 }
223 let deadline_remaining = deadline.saturating_duration_since(now);
224 if deadline_remaining.is_zero() {
225 return None;
226 }
227 let wait_duration = if observed_response {
228 quiet_remaining
229 } else {
230 quiet_remaining.max(
231 RESIZE_NO_CHANGE_GRACE.saturating_sub(now.saturating_duration_since(started)),
232 )
233 };
234 let waited = self
235 .0
236 .changed
237 .wait_timeout(guard, wait_duration.min(deadline_remaining))
238 .unwrap_or_else(|error| error.into_inner());
239 guard = waited.0;
240 let current = root_bounds(&guard)?;
241 if current != latest {
242 latest = current;
243 last_change = Instant::now();
244 observed_response = true;
245 }
246 }
247 }
248}
249
250fn root_bounds(tree: &SemanticTree) -> Option<SemanticBounds> {
251 tree.nodes
252 .iter()
253 .find(|node| node.role.eq_ignore_ascii_case(ROOT_NODE_ROLE))
254 .and_then(|node| node.bounds)
255}
256
257fn pointer_position(
258 registry: &InspectionRegistry,
259 id: Option<&str>,
260 x: Option<f32>,
261 y: Option<f32>,
262) -> Result<SemanticPoint, String> {
263 match (id, x, y) {
264 (Some(id), None, None) => {
265 let node = registry
266 .node(id)
267 .ok_or_else(|| format!("unknown semantic node '{id}'"))?;
268 let bounds = node
269 .bounds
270 .ok_or_else(|| format!("semantic node '{id}' has no published bounds"))?;
271 Ok(SemanticPoint {
272 x: bounds.x + bounds.width / 2.0,
273 y: bounds.y + bounds.height / 2.0,
274 })
275 }
276 (None, Some(x), Some(y)) if x.is_finite() && y.is_finite() && x >= 0.0 && y >= 0.0 => {
277 Ok(SemanticPoint { x, y })
278 }
279 (Some(_), Some(_), Some(_)) => {
280 Err("pointer target must use either a semantic id or logical coordinates".to_owned())
281 }
282 _ => Err("pointer target requires a semantic id or both logical coordinates".to_owned()),
283 }
284}
285
286#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
287pub struct SemanticPoint {
288 pub x: f32,
289 pub y: f32,
290}
291
292pub struct ActionRequest {
294 pub action: InspectionAction,
295 pub response: mpsc::SyncSender<Result<InspectionActionResult, String>>,
296}
297
298#[derive(Clone, Debug, PartialEq)]
299pub enum InspectionActionResult {
300 Complete,
301 Screenshot {
302 path: String,
303 width: u32,
304 height: u32,
305 },
306}
307
308#[derive(Clone, Debug, PartialEq)]
309pub enum InspectionAction {
310 Click {
311 id: String,
312 },
313 Resize {
314 width: u32,
315 height: u32,
316 },
317 Focus {
318 id: String,
319 },
320 Key {
321 key: String,
322 modifiers: NormalizedModifiers,
323 },
324 TypeText {
325 text: String,
326 },
327 Scroll {
328 position: SemanticPoint,
329 delta_x: f32,
330 delta_y: f32,
331 modifiers: NormalizedModifiers,
332 },
333 Drag {
334 from: SemanticPoint,
335 to: SemanticPoint,
336 },
337 Screenshot {
338 path: String,
339 width: u32,
340 height: u32,
341 },
342}
343
344#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
345pub struct NormalizedModifiers {
346 pub platform: bool,
347 pub control: bool,
348 pub alt: bool,
349 pub shift: bool,
350 pub function: bool,
351}
352
353#[derive(Clone, Debug, Serialize, Deserialize)]
354#[serde(tag = "kind", rename_all = "snake_case")]
355pub enum InspectionRequest {
356 Tree,
357 Query {
358 id: String,
359 },
360 Click {
361 id: String,
362 },
363 Wait {
364 id: String,
365 value: String,
366 timeout_ms: u64,
367 },
368 Resize {
369 width: u32,
370 height: u32,
371 timeout_ms: u64,
372 },
373 Focus {
374 id: String,
375 timeout_ms: u64,
376 },
377 Key {
378 key: String,
379 modifiers: NormalizedModifiers,
380 },
381 TypeText {
382 text: String,
383 },
384 Scroll {
385 id: Option<String>,
386 x: Option<f32>,
387 y: Option<f32>,
388 delta_x: f32,
389 delta_y: f32,
390 #[serde(default)]
391 modifiers: NormalizedModifiers,
392 },
393 Drag {
394 from_id: Option<String>,
395 from_x: Option<f32>,
396 from_y: Option<f32>,
397 to_id: Option<String>,
398 to_x: Option<f32>,
399 to_y: Option<f32>,
400 },
401 Screenshot {
402 name: String,
403 },
404}
405
406#[derive(Clone, Debug, Serialize, Deserialize)]
407pub struct AuthenticatedRequest {
408 pub token: String,
409 pub request: InspectionRequest,
410}
411
412#[derive(Clone, Debug, Serialize, Deserialize)]
413pub struct InspectionResponse {
414 pub ok: bool,
415 pub result: Option<serde_json::Value>,
416 pub error: Option<String>,
417}
418
419impl InspectionResponse {
420 fn success(value: impl Serialize) -> Self {
421 match serde_json::to_value(value) {
422 Ok(result) => Self {
423 ok: true,
424 result: Some(result),
425 error: None,
426 },
427 Err(error) => Self::failure(format!("serialize response: {error}")),
428 }
429 }
430
431 fn failure(error: impl Into<String>) -> Self {
432 Self {
433 ok: false,
434 result: None,
435 error: Some(error.into()),
436 }
437 }
438}
439
440pub struct InspectionServer {
442 address: SocketAddr,
443 token: String,
444}
445
446impl InspectionServer {
447 pub fn start(
448 port: u16,
449 registry: InspectionRegistry,
450 actions: async_channel::Sender<ActionRequest>,
451 ) -> Result<Self, String> {
452 let listener = TcpListener::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port))
453 .map_err(|error| format!("bind GPUI inspection listener: {error}"))?;
454 let address = listener
455 .local_addr()
456 .map_err(|error| format!("read GPUI inspection address: {error}"))?;
457 let token = uuid::Uuid::new_v4().simple().to_string();
458 let server_token = token.clone();
459 thread::Builder::new()
460 .name("gpui-mcp-inspection".to_owned())
461 .spawn(move || serve(listener, &server_token, registry, actions))
462 .map_err(|error| format!("start GPUI inspection listener: {error}"))?;
463 Ok(Self { address, token })
464 }
465
466 #[must_use]
467 pub fn address(&self) -> SocketAddr {
468 self.address
469 }
470
471 #[must_use]
472 pub fn token(&self) -> &str {
473 &self.token
474 }
475}
476
477fn serve(
478 listener: TcpListener,
479 token: &str,
480 registry: InspectionRegistry,
481 actions: async_channel::Sender<ActionRequest>,
482) {
483 for stream in listener.incoming() {
484 let Ok(stream) = stream else { continue };
485 let _ = stream.set_read_timeout(Some(Duration::from_secs(2)));
486 let _ = stream.set_write_timeout(Some(Duration::from_secs(2)));
487 let response = handle_stream(stream.try_clone().ok(), token, ®istry, &actions);
488 if let Ok(mut stream) = stream.try_clone() {
489 let _ = write_response(&mut stream, &response);
490 }
491 }
492}
493
494fn handle_stream(
495 stream: Option<TcpStream>,
496 token: &str,
497 registry: &InspectionRegistry,
498 actions: &async_channel::Sender<ActionRequest>,
499) -> InspectionResponse {
500 let Some(stream) = stream else {
501 return InspectionResponse::failure("clone inspection stream");
502 };
503 let mut line = String::new();
504 if let Err(error) = BufReader::new(stream)
505 .take(MAX_REQUEST_BYTES)
506 .read_line(&mut line)
507 {
508 return InspectionResponse::failure(format!("read request: {error}"));
509 }
510 let request: AuthenticatedRequest = match serde_json::from_str(&line) {
511 Ok(request) => request,
512 Err(error) => return InspectionResponse::failure(format!("invalid request: {error}")),
513 };
514 if request.token != token {
515 return InspectionResponse::failure("authentication failed");
516 }
517 dispatch(request.request, registry, actions)
518}
519
520fn dispatch(
521 request: InspectionRequest,
522 registry: &InspectionRegistry,
523 actions: &async_channel::Sender<ActionRequest>,
524) -> InspectionResponse {
525 match request {
526 InspectionRequest::Tree => InspectionResponse::success(registry.snapshot().as_ref()),
527 InspectionRequest::Query { id } => InspectionResponse::success(registry.node(&id)),
528 InspectionRequest::Wait {
529 id,
530 value,
531 timeout_ms,
532 } => {
533 let timeout = Duration::from_millis(timeout_ms.min(30_000));
534 match registry.wait_for_value(&id, &value, timeout) {
535 Some(node) => InspectionResponse::success(node),
536 None => InspectionResponse::failure(format!(
537 "timed out waiting for '{id}' to have value '{value}'"
538 )),
539 }
540 }
541 InspectionRequest::Click { id } => {
542 match dispatch_action(actions, InspectionAction::Click { id }) {
543 Ok(Ok(InspectionActionResult::Complete)) => {
544 InspectionResponse::success(serde_json::json!({ "clicked": true }))
545 }
546 Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
547 InspectionResponse::failure("GPUI click action returned a screenshot artifact")
548 }
549 Ok(Err(error)) => InspectionResponse::failure(error),
550 Err(_) => InspectionResponse::failure("GPUI action timed out"),
551 }
552 }
553 InspectionRequest::Resize {
554 width,
555 height,
556 timeout_ms,
557 } => {
558 if width == 0
559 || height == 0
560 || width > MAX_LOGICAL_WINDOW_SIZE
561 || height > MAX_LOGICAL_WINDOW_SIZE
562 {
563 return InspectionResponse::failure(format!(
564 "resize dimensions must be between 1 and {MAX_LOGICAL_WINDOW_SIZE} logical pixels"
565 ));
566 }
567 let Some(initial) = root_bounds(®istry.snapshot()) else {
568 return InspectionResponse::failure(
569 "GPUI root bounds are unavailable before resize",
570 );
571 };
572 match dispatch_action(actions, InspectionAction::Resize { width, height }) {
573 Ok(Ok(InspectionActionResult::Complete)) => {}
574 Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
575 return InspectionResponse::failure(
576 "GPUI resize action returned a screenshot artifact",
577 );
578 }
579 Ok(Err(error)) => return InspectionResponse::failure(error),
580 Err(_) => return InspectionResponse::failure("GPUI resize action timed out"),
581 }
582 let timeout = Duration::from_millis(timeout_ms.clamp(1, 30_000));
583 match registry.wait_for_root_bounds_settled(width, height, initial, timeout) {
584 Some(actual) => InspectionResponse::success(serde_json::json!({
585 "requested": { "width": width, "height": height },
586 "actual": { "width": actual.width, "height": actual.height },
587 "constrained": actual.width != width as f32 || actual.height != height as f32,
588 "settled": true
589 })),
590 None => InspectionResponse::failure("timed out waiting for GPUI resize to settle"),
591 }
592 }
593 InspectionRequest::Focus { id, timeout_ms } => {
594 match dispatch_action(actions, InspectionAction::Focus { id: id.clone() }) {
595 Ok(Ok(InspectionActionResult::Complete)) => {}
596 Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
597 return InspectionResponse::failure(
598 "GPUI focus action returned a screenshot artifact",
599 );
600 }
601 Ok(Err(error)) => return InspectionResponse::failure(error),
602 Err(_) => return InspectionResponse::failure("GPUI focus action timed out"),
603 }
604 let timeout = Duration::from_millis(timeout_ms.clamp(1, 30_000));
605 match registry.wait_for_focus(&id, true, timeout) {
606 Some(node) => InspectionResponse::success(node),
607 None => InspectionResponse::failure(format!(
608 "timed out waiting for '{id}' to receive focus"
609 )),
610 }
611 }
612 InspectionRequest::Key { key, modifiers } => {
613 if key.is_empty() || key.len() > MAX_KEY_NAME_BYTES {
614 return InspectionResponse::failure(format!(
615 "key must contain between 1 and {MAX_KEY_NAME_BYTES} bytes"
616 ));
617 }
618 match dispatch_action(actions, InspectionAction::Key { key, modifiers }) {
619 Ok(Ok(InspectionActionResult::Complete)) => {
620 InspectionResponse::success(serde_json::json!({ "pressed": true }))
621 }
622 Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
623 InspectionResponse::failure("GPUI key action returned a screenshot artifact")
624 }
625 Ok(Err(error)) => InspectionResponse::failure(error),
626 Err(_) => InspectionResponse::failure("GPUI key action timed out"),
627 }
628 }
629 InspectionRequest::TypeText { text } => {
630 if text.len() > MAX_TYPED_TEXT_BYTES {
631 return InspectionResponse::failure(format!(
632 "text must not exceed {MAX_TYPED_TEXT_BYTES} UTF-8 bytes"
633 ));
634 }
635 let characters = text.chars().count();
636 match dispatch_action(actions, InspectionAction::TypeText { text }) {
637 Ok(Ok(InspectionActionResult::Complete)) => {
638 InspectionResponse::success(serde_json::json!({
639 "typed": true,
640 "characters": characters
641 }))
642 }
643 Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
644 InspectionResponse::failure("GPUI typing action returned a screenshot artifact")
645 }
646 Ok(Err(error)) => InspectionResponse::failure(error),
647 Err(_) => InspectionResponse::failure("GPUI typing action timed out"),
648 }
649 }
650 InspectionRequest::Scroll {
651 id,
652 x,
653 y,
654 delta_x,
655 delta_y,
656 modifiers,
657 } => {
658 if !delta_x.is_finite() || !delta_y.is_finite() {
659 return InspectionResponse::failure("scroll deltas must be finite");
660 }
661 let position = match pointer_position(registry, id.as_deref(), x, y) {
662 Ok(position) => position,
663 Err(error) => return InspectionResponse::failure(error),
664 };
665 match dispatch_action(
666 actions,
667 InspectionAction::Scroll {
668 position,
669 delta_x,
670 delta_y,
671 modifiers,
672 },
673 ) {
674 Ok(Ok(InspectionActionResult::Complete)) => {
675 InspectionResponse::success(serde_json::json!({
676 "scrolled": true,
677 "position": position,
678 "delta": { "x": delta_x, "y": delta_y },
679 "modifiers": modifiers
680 }))
681 }
682 Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
683 InspectionResponse::failure("GPUI scroll action returned a screenshot artifact")
684 }
685 Ok(Err(error)) => InspectionResponse::failure(error),
686 Err(_) => InspectionResponse::failure("GPUI scroll action timed out"),
687 }
688 }
689 InspectionRequest::Drag {
690 from_id,
691 from_x,
692 from_y,
693 to_id,
694 to_x,
695 to_y,
696 } => {
697 let from = match pointer_position(registry, from_id.as_deref(), from_x, from_y) {
698 Ok(position) => position,
699 Err(error) => {
700 return InspectionResponse::failure(format!("invalid drag start: {error}"));
701 }
702 };
703 let to = match pointer_position(registry, to_id.as_deref(), to_x, to_y) {
704 Ok(position) => position,
705 Err(error) => {
706 return InspectionResponse::failure(format!("invalid drag end: {error}"));
707 }
708 };
709 match dispatch_action(actions, InspectionAction::Drag { from, to }) {
710 Ok(Ok(InspectionActionResult::Complete)) => {
711 InspectionResponse::success(serde_json::json!({
712 "dragged": true,
713 "from": from,
714 "to": to
715 }))
716 }
717 Ok(Ok(InspectionActionResult::Screenshot { .. })) => {
718 InspectionResponse::failure("GPUI drag action returned a screenshot artifact")
719 }
720 Ok(Err(error)) => InspectionResponse::failure(error),
721 Err(_) => InspectionResponse::failure("GPUI drag action timed out"),
722 }
723 }
724 InspectionRequest::Screenshot { name } => {
725 if name.is_empty()
726 || name.len() > MAX_SCREENSHOT_NAME_BYTES
727 || !name
728 .chars()
729 .all(|character| character.is_ascii_alphanumeric() || "-_.".contains(character))
730 || name.contains("..")
731 {
732 return InspectionResponse::failure(format!(
733 "screenshot name must contain 1-{MAX_SCREENSHOT_NAME_BYTES} ASCII letters, digits, '-', '_', or '.'"
734 ));
735 }
736 let Some(bounds) = root_bounds(®istry.snapshot()) else {
737 return InspectionResponse::failure(
738 "GPUI root bounds are unavailable before screenshot capture",
739 );
740 };
741 let width = bounds.width.round() as u32;
742 let height = bounds.height.round() as u32;
743 let directory = std::env::temp_dir().join("gpui-mcp");
744 if let Err(error) = std::fs::create_dir_all(&directory) {
745 return InspectionResponse::failure(format!(
746 "create screenshot artifact directory: {error}"
747 ));
748 }
749 let path = directory.join(format!("{name}-{}.png", uuid::Uuid::new_v4().simple()));
750 let path = path.to_string_lossy().into_owned();
751 match dispatch_action_with_timeout(
752 actions,
753 InspectionAction::Screenshot {
754 path: path.clone(),
755 width,
756 height,
757 },
758 SCREENSHOT_TIMEOUT,
759 ) {
760 Ok(Ok(InspectionActionResult::Screenshot {
761 path,
762 width,
763 height,
764 })) => InspectionResponse::success(serde_json::json!({
765 "captured": true,
766 "path": path,
767 "width": width,
768 "height": height,
769 "coordinate_scale": "logical"
770 })),
771 Ok(Ok(InspectionActionResult::Complete)) => {
772 InspectionResponse::failure("GPUI screenshot action returned no artifact")
773 }
774 Ok(Err(error)) => InspectionResponse::failure(error),
775 Err(_) => InspectionResponse::failure("GPUI screenshot action timed out"),
776 }
777 }
778 }
779}
780
781fn dispatch_action(
782 actions: &async_channel::Sender<ActionRequest>,
783 action: InspectionAction,
784) -> Result<Result<InspectionActionResult, String>, mpsc::RecvTimeoutError> {
785 dispatch_action_with_timeout(actions, action, ACTION_TIMEOUT)
786}
787
788fn dispatch_action_with_timeout(
789 actions: &async_channel::Sender<ActionRequest>,
790 action: InspectionAction,
791 timeout: Duration,
792) -> Result<Result<InspectionActionResult, String>, mpsc::RecvTimeoutError> {
793 let (response_tx, response_rx) = mpsc::sync_channel(1);
794 if let Err(error) = actions.try_send(ActionRequest {
795 action,
796 response: response_tx,
797 }) {
798 return Ok(Err(format!("GPUI action queue is unavailable: {error}")));
799 }
800 response_rx.recv_timeout(timeout)
801}
802
803fn write_response(stream: &mut TcpStream, response: &InspectionResponse) -> std::io::Result<()> {
804 serde_json::to_writer(&mut *stream, response)?;
805 stream.write_all(b"\n")?;
806 stream.flush()
807}
808
809pub fn request(
811 address: SocketAddr,
812 token: &str,
813 request: InspectionRequest,
814) -> Result<InspectionResponse, String> {
815 let mut stream = TcpStream::connect_timeout(&address, Duration::from_secs(2))
816 .map_err(|error| format!("connect to {address}: {error}"))?;
817 stream
818 .set_read_timeout(Some(Duration::from_secs(32)))
819 .map_err(|error| format!("configure inspection socket: {error}"))?;
820 serde_json::to_writer(
821 &mut stream,
822 &AuthenticatedRequest {
823 token: token.to_owned(),
824 request,
825 },
826 )
827 .map_err(|error| format!("encode inspection request: {error}"))?;
828 stream
829 .write_all(b"\n")
830 .map_err(|error| format!("write inspection request: {error}"))?;
831 let mut line = String::new();
832 BufReader::new(stream)
833 .read_line(&mut line)
834 .map_err(|error| format!("read inspection response: {error}"))?;
835 serde_json::from_str(&line).map_err(|error| format!("decode inspection response: {error}"))
836}
837
838#[cfg(test)]
839mod tests {
840 use super::*;
841
842 fn node(value: &str) -> SemanticNode {
843 focused_node(value, false)
844 }
845
846 fn focused_node(value: &str, focused: bool) -> SemanticNode {
847 SemanticNode {
848 id: "preview.plugin.editor".to_owned(),
849 role: "button".to_owned(),
850 label: "Plugin editor".to_owned(),
851 value: Some(value.to_owned()),
852 bounds: None,
853 focused,
854 actions: vec!["click".to_owned()],
855 }
856 }
857
858 fn root(width: f32, height: f32) -> SemanticNode {
859 SemanticNode {
860 id: "test.root".to_owned(),
861 role: "application".to_owned(),
862 label: "Preview".to_owned(),
863 value: None,
864 bounds: Some(SemanticBounds {
865 x: 0.0,
866 y: 0.0,
867 width,
868 height,
869 }),
870 focused: false,
871 actions: Vec::new(),
872 }
873 }
874
875 #[test]
876 fn publish_should_replace_latest_snapshot() {
877 let registry = InspectionRegistry::default();
878 registry.publish(SemanticTree {
879 revision: 7,
880 nodes: vec![node("closed")],
881 });
882 assert_eq!(registry.snapshot().revision, 7);
883 }
884
885 #[test]
886 fn wait_for_value_should_observe_a_later_publish() {
887 let registry = InspectionRegistry::default();
888 registry.publish(SemanticTree {
889 revision: 1,
890 nodes: vec![node("opening")],
891 });
892 let publisher = registry.clone();
893 thread::spawn(move || {
894 thread::sleep(Duration::from_millis(10));
895 publisher.publish(SemanticTree {
896 revision: 2,
897 nodes: vec![node("open")],
898 });
899 });
900 assert!(
901 registry
902 .wait_for_value("preview.plugin.editor", "open", Duration::from_secs(1))
903 .is_some()
904 );
905 }
906
907 #[test]
908 fn listener_should_reject_an_invalid_token() {
909 let registry = InspectionRegistry::default();
910 let (action_tx, _action_rx) = async_channel::bounded(1);
911 let server = InspectionServer::start(0, registry, action_tx).expect("start listener");
912 let response = request(server.address(), "wrong", InspectionRequest::Tree)
913 .expect("receive authentication response");
914 assert!(!response.ok);
915 }
916
917 #[test]
918 fn click_and_wait_should_follow_a_published_lifecycle() {
919 let registry = InspectionRegistry::default();
920 registry.publish(SemanticTree {
921 revision: 1,
922 nodes: vec![node("closed")],
923 });
924 let (action_tx, action_rx) = async_channel::bounded(1);
925 let server = InspectionServer::start(0, registry.clone(), action_tx)
926 .expect("start inspection listener");
927 thread::spawn(move || {
928 let action = action_rx.recv_blocking().expect("receive click action");
929 assert_eq!(
930 action.action,
931 InspectionAction::Click {
932 id: "preview.plugin.editor".to_owned()
933 }
934 );
935 registry.publish(SemanticTree {
936 revision: 2,
937 nodes: vec![node("open")],
938 });
939 action
940 .response
941 .send(Ok(InspectionActionResult::Complete))
942 .expect("acknowledge click");
943 });
944
945 let click = request(
946 server.address(),
947 server.token(),
948 InspectionRequest::Click {
949 id: "preview.plugin.editor".to_owned(),
950 },
951 )
952 .expect("click response");
953 assert!(click.ok);
954 let wait = request(
955 server.address(),
956 server.token(),
957 InspectionRequest::Wait {
958 id: "preview.plugin.editor".to_owned(),
959 value: "open".to_owned(),
960 timeout_ms: 1_000,
961 },
962 )
963 .expect("wait response");
964 assert!(wait.ok);
965 }
966
967 #[test]
968 fn resize_should_wait_for_settled_actual_bounds() {
969 let registry = InspectionRegistry::default();
970 registry.publish(SemanticTree {
971 revision: 1,
972 nodes: vec![root(1024.0, 640.0)],
973 });
974 let (action_tx, action_rx) = async_channel::bounded(1);
975 let server = InspectionServer::start(0, registry.clone(), action_tx)
976 .expect("start inspection listener");
977 thread::spawn(move || {
978 let action = action_rx.recv_blocking().expect("receive resize action");
979 assert_eq!(
980 action.action,
981 InspectionAction::Resize {
982 width: 1200,
983 height: 720
984 }
985 );
986 action
987 .response
988 .send(Ok(InspectionActionResult::Complete))
989 .expect("acknowledge resize");
990 registry.publish(SemanticTree {
991 revision: 1,
992 nodes: vec![root(1200.0, 720.0)],
993 });
994 });
995
996 let response = request(
997 server.address(),
998 server.token(),
999 InspectionRequest::Resize {
1000 width: 1200,
1001 height: 720,
1002 timeout_ms: 1_000,
1003 },
1004 )
1005 .expect("resize response");
1006 assert!(response.ok);
1007 let result = response.result.expect("resize result");
1008 assert_eq!(result["actual"]["width"], 1200.0);
1009 assert_eq!(result["actual"]["height"], 720.0);
1010 assert_eq!(result["constrained"], false);
1011 assert_eq!(result["settled"], true);
1012 }
1013
1014 #[test]
1015 fn resize_should_report_a_constrained_no_op() {
1016 let registry = InspectionRegistry::default();
1017 registry.publish(SemanticTree {
1018 revision: 1,
1019 nodes: vec![root(1024.0, 640.0)],
1020 });
1021 let (action_tx, action_rx) = async_channel::bounded(1);
1022 let server =
1023 InspectionServer::start(0, registry, action_tx).expect("start inspection listener");
1024 thread::spawn(move || {
1025 let action = action_rx.recv_blocking().expect("receive resize action");
1026 action
1027 .response
1028 .send(Ok(InspectionActionResult::Complete))
1029 .expect("acknowledge resize");
1030 });
1031
1032 let response = request(
1033 server.address(),
1034 server.token(),
1035 InspectionRequest::Resize {
1036 width: 800,
1037 height: 500,
1038 timeout_ms: 1_000,
1039 },
1040 )
1041 .expect("resize response");
1042 assert!(response.ok);
1043 assert_eq!(response.result.expect("resize result")["constrained"], true);
1044 }
1045
1046 #[test]
1047 fn resize_should_reject_invalid_dimensions_before_enqueuing() {
1048 let registry = InspectionRegistry::default();
1049 let (action_tx, action_rx) = async_channel::bounded(1);
1050 let response = dispatch(
1051 InspectionRequest::Resize {
1052 width: 0,
1053 height: 640,
1054 timeout_ms: 1_000,
1055 },
1056 ®istry,
1057 &action_tx,
1058 );
1059
1060 assert!(!response.ok);
1061 assert!(action_rx.is_empty());
1062 }
1063
1064 #[test]
1065 fn focus_should_wait_for_semantic_publication() {
1066 let registry = InspectionRegistry::default();
1067 registry.publish(SemanticTree {
1068 revision: 1,
1069 nodes: vec![focused_node("closed", false)],
1070 });
1071 let (action_tx, action_rx) = async_channel::bounded(1);
1072 let server = InspectionServer::start(0, registry.clone(), action_tx)
1073 .expect("start inspection listener");
1074 thread::spawn(move || {
1075 let action = action_rx.recv_blocking().expect("receive focus action");
1076 assert_eq!(
1077 action.action,
1078 InspectionAction::Focus {
1079 id: "preview.plugin.editor".to_owned()
1080 }
1081 );
1082 registry.publish(SemanticTree {
1083 revision: 1,
1084 nodes: vec![focused_node("closed", true)],
1085 });
1086 action
1087 .response
1088 .send(Ok(InspectionActionResult::Complete))
1089 .expect("acknowledge focus");
1090 });
1091
1092 let response = request(
1093 server.address(),
1094 server.token(),
1095 InspectionRequest::Focus {
1096 id: "preview.plugin.editor".to_owned(),
1097 timeout_ms: 1_000,
1098 },
1099 )
1100 .expect("focus response");
1101 assert!(response.ok);
1102 assert_eq!(response.result.expect("focused node")["focused"], true);
1103 }
1104
1105 #[test]
1106 fn keyboard_inputs_should_be_bounded_before_enqueuing() {
1107 let registry = InspectionRegistry::default();
1108 let (action_tx, action_rx) = async_channel::bounded(1);
1109 let empty_key = dispatch(
1110 InspectionRequest::Key {
1111 key: String::new(),
1112 modifiers: NormalizedModifiers::default(),
1113 },
1114 ®istry,
1115 &action_tx,
1116 );
1117 let oversized_text = dispatch(
1118 InspectionRequest::TypeText {
1119 text: "x".repeat(MAX_TYPED_TEXT_BYTES + 1),
1120 },
1121 ®istry,
1122 &action_tx,
1123 );
1124
1125 assert!(!empty_key.ok);
1126 assert!(!oversized_text.ok);
1127 assert!(action_rx.is_empty());
1128 }
1129
1130 #[test]
1131 fn pointer_targets_should_resolve_semantic_centers_or_coordinates() {
1132 let registry = InspectionRegistry::default();
1133 registry.publish(SemanticTree {
1134 revision: 1,
1135 nodes: vec![root(1024.0, 640.0)],
1136 });
1137
1138 assert_eq!(
1139 pointer_position(®istry, Some("test.root"), None, None),
1140 Ok(SemanticPoint { x: 512.0, y: 320.0 })
1141 );
1142 assert_eq!(
1143 pointer_position(®istry, None, Some(12.0), Some(34.0)),
1144 Ok(SemanticPoint { x: 12.0, y: 34.0 })
1145 );
1146 assert!(pointer_position(®istry, None, Some(12.0), None).is_err());
1147 assert!(pointer_position(®istry, Some("test.root"), Some(1.0), Some(2.0)).is_err());
1148 }
1149
1150 #[test]
1151 fn screenshot_should_use_root_logical_dimensions_and_generated_path() {
1152 let registry = InspectionRegistry::default();
1153 registry.publish(SemanticTree {
1154 revision: 1,
1155 nodes: vec![root(1024.0, 640.0)],
1156 });
1157 let (action_tx, action_rx) = async_channel::bounded(1);
1158 let server =
1159 InspectionServer::start(0, registry, action_tx).expect("start inspection listener");
1160 thread::spawn(move || {
1161 let action = action_rx
1162 .recv_blocking()
1163 .expect("receive screenshot action");
1164 let InspectionAction::Screenshot {
1165 path,
1166 width,
1167 height,
1168 } = action.action
1169 else {
1170 panic!("expected screenshot action")
1171 };
1172 assert_eq!((width, height), (1024, 640));
1173 assert!(path.ends_with(".png"));
1174 action
1175 .response
1176 .send(Ok(InspectionActionResult::Screenshot {
1177 path,
1178 width,
1179 height,
1180 }))
1181 .expect("acknowledge screenshot");
1182 });
1183
1184 let response = request(
1185 server.address(),
1186 server.token(),
1187 InspectionRequest::Screenshot {
1188 name: "logical-home".to_owned(),
1189 },
1190 )
1191 .expect("screenshot response");
1192 assert!(response.ok);
1193 let result = response.result.expect("screenshot result");
1194 assert_eq!(result["width"], 1024);
1195 assert_eq!(result["height"], 640);
1196 assert_eq!(result["coordinate_scale"], "logical");
1197 }
1198}