egui_mcp_client/
server.rs

1//! IPC server for handling MCP requests
2//!
3//! This server handles requests that require direct application access:
4//! - Screenshots
5//! - Coordinate-based input
6//! - Keyboard input
7//! - Scroll events
8
9use crate::{McpClient, PendingInput};
10use base64::Engine;
11use egui_mcp_protocol::{ProtocolError, Request, Response, read_request, write_response};
12use std::time::Duration;
13use tokio::net::{UnixListener, UnixStream};
14
15/// IPC server that listens for MCP requests
16pub struct IpcServer;
17
18impl IpcServer {
19    /// Run the IPC server
20    pub async fn run(client: McpClient) -> Result<(), ProtocolError> {
21        let socket_path = client.socket_path().await;
22
23        // Remove existing socket file if it exists
24        if socket_path.exists() {
25            std::fs::remove_file(&socket_path)?;
26        }
27
28        // Create parent directory if needed
29        if let Some(parent) = socket_path.parent() {
30            std::fs::create_dir_all(parent)?;
31        }
32
33        let listener = UnixListener::bind(&socket_path)?;
34        tracing::info!("IPC server listening on {:?}", socket_path);
35
36        loop {
37            match listener.accept().await {
38                Ok((stream, _addr)) => {
39                    let client = client.clone();
40                    tokio::spawn(async move {
41                        if let Err(e) = Self::handle_connection(stream, client).await {
42                            match e {
43                                ProtocolError::ConnectionClosed => {
44                                    tracing::debug!("Client disconnected");
45                                }
46                                _ => {
47                                    tracing::error!("Connection error: {}", e);
48                                }
49                            }
50                        }
51                    });
52                }
53                Err(e) => {
54                    tracing::error!("Accept error: {}", e);
55                }
56            }
57        }
58    }
59
60    /// Handle a single connection
61    async fn handle_connection(stream: UnixStream, client: McpClient) -> Result<(), ProtocolError> {
62        let (mut reader, mut writer) = stream.into_split();
63
64        loop {
65            let request = read_request(&mut reader).await?;
66            tracing::debug!("Received request: {:?}", request);
67
68            let response = Self::handle_request(&request, &client).await;
69            tracing::debug!("Sending response: {:?}", response);
70
71            write_response(&mut writer, &response).await?;
72        }
73    }
74
75    /// Handle a single request
76    async fn handle_request(request: &Request, client: &McpClient) -> Response {
77        match request {
78            Request::Ping => Response::Pong,
79
80            Request::TakeScreenshot => {
81                // Request a screenshot and get a receiver (event-driven)
82                let rx = client.request_screenshot().await;
83
84                // Wait for the screenshot with timeout (no polling needed)
85                match tokio::time::timeout(Duration::from_secs(5), rx).await {
86                    Ok(Ok(data)) => {
87                        let encoded = base64::engine::general_purpose::STANDARD.encode(&data);
88                        Response::Screenshot {
89                            data: encoded,
90                            format: "png".to_string(),
91                        }
92                    }
93                    Ok(Err(_)) => Response::Error {
94                        message: "Screenshot request was cancelled".to_string(),
95                    },
96                    Err(_) => Response::Error {
97                        message: "Screenshot timeout: the egui app did not provide a screenshot within 5 seconds".to_string(),
98                    },
99                }
100            }
101
102            Request::ClickAt { x, y, button } => {
103                client
104                    .queue_input(PendingInput::Click {
105                        x: *x,
106                        y: *y,
107                        button: *button,
108                    })
109                    .await;
110                Response::Success
111            }
112
113            Request::MoveMouse { x, y } => {
114                client
115                    .queue_input(PendingInput::MoveMouse { x: *x, y: *y })
116                    .await;
117                Response::Success
118            }
119
120            Request::KeyboardInput { key } => {
121                client
122                    .queue_input(PendingInput::Keyboard { key: key.clone() })
123                    .await;
124                Response::Success
125            }
126
127            Request::Scroll {
128                x,
129                y,
130                delta_x,
131                delta_y,
132            } => {
133                client
134                    .queue_input(PendingInput::Scroll {
135                        x: *x,
136                        y: *y,
137                        delta_x: *delta_x,
138                        delta_y: *delta_y,
139                    })
140                    .await;
141                Response::Success
142            }
143
144            Request::Drag {
145                start_x,
146                start_y,
147                end_x,
148                end_y,
149                button,
150            } => {
151                client
152                    .queue_input(PendingInput::Drag {
153                        start_x: *start_x,
154                        start_y: *start_y,
155                        end_x: *end_x,
156                        end_y: *end_y,
157                        button: *button,
158                    })
159                    .await;
160                Response::Success
161            }
162
163            Request::DoubleClick { x, y, button } => {
164                client
165                    .queue_input(PendingInput::DoubleClick {
166                        x: *x,
167                        y: *y,
168                        button: *button,
169                    })
170                    .await;
171                Response::Success
172            }
173
174            Request::TakeScreenshotRegion {
175                x,
176                y,
177                width,
178                height,
179            } => {
180                // Request a screenshot and get a receiver (event-driven)
181                let rx = client.request_screenshot().await;
182
183                // Wait for the screenshot with timeout (no polling needed)
184                match tokio::time::timeout(Duration::from_secs(5), rx).await {
185                    Ok(Ok(data)) => {
186                        // Crop the screenshot to the specified region
187                        match Self::crop_screenshot(&data, *x, *y, *width, *height) {
188                            Ok(cropped) => {
189                                let encoded =
190                                    base64::engine::general_purpose::STANDARD.encode(&cropped);
191                                Response::Screenshot {
192                                    data: encoded,
193                                    format: "png".to_string(),
194                                }
195                            }
196                            Err(e) => Response::Error {
197                                message: format!("Failed to crop screenshot: {}", e),
198                            },
199                        }
200                    }
201                    Ok(Err(_)) => Response::Error {
202                        message: "Screenshot request was cancelled".to_string(),
203                    },
204                    Err(_) => Response::Error {
205                        message: "Screenshot timeout: the egui app did not provide a screenshot within 5 seconds".to_string(),
206                    },
207                }
208            }
209
210            Request::HighlightElement {
211                x,
212                y,
213                width,
214                height,
215                color,
216                duration_ms,
217            } => {
218                let rect =
219                    egui::Rect::from_min_size(egui::pos2(*x, *y), egui::vec2(*width, *height));
220                let egui_color =
221                    egui::Color32::from_rgba_unmultiplied(color[0], color[1], color[2], color[3]);
222                let expires_at = if *duration_ms == 0 {
223                    None
224                } else {
225                    Some(std::time::Instant::now() + std::time::Duration::from_millis(*duration_ms))
226                };
227                client
228                    .add_highlight(crate::Highlight {
229                        rect,
230                        color: egui_color,
231                        expires_at,
232                    })
233                    .await;
234                Response::Success
235            }
236
237            Request::ClearHighlights => {
238                client.clear_highlights().await;
239                Response::Success
240            }
241
242            Request::GetLogs { level, limit } => {
243                let entries = client.get_logs(level.as_deref(), *limit).await;
244                Response::Logs { entries }
245            }
246
247            Request::ClearLogs => {
248                client.clear_logs().await;
249                Response::Success
250            }
251
252            Request::GetFrameStats => {
253                let stats = client.get_frame_stats().await;
254                Response::FrameStatsResponse { stats }
255            }
256
257            Request::StartPerfRecording { duration_ms } => {
258                client.start_perf_recording(*duration_ms).await;
259                Response::Success
260            }
261
262            Request::GetPerfReport => {
263                let report = client.get_perf_report().await;
264                Response::PerfReportResponse { report }
265            }
266        }
267    }
268
269    /// Crop a PNG screenshot to the specified region
270    fn crop_screenshot(
271        png_data: &[u8],
272        x: f32,
273        y: f32,
274        width: f32,
275        height: f32,
276    ) -> Result<Vec<u8>, String> {
277        use image::GenericImageView;
278        use std::io::Cursor;
279
280        let x = x as u32;
281        let y = y as u32;
282        let width = width as u32;
283        let height = height as u32;
284
285        // Load image from PNG data
286        let img = image::load_from_memory(png_data)
287            .map_err(|e| format!("Failed to load image: {}", e))?;
288
289        // Validate crop region
290        let (img_width, img_height) = img.dimensions();
291        if x >= img_width || y >= img_height {
292            return Err(format!(
293                "Crop region starts outside image bounds. Image: {}x{}, Region start: ({}, {})",
294                img_width, img_height, x, y
295            ));
296        }
297
298        // Clamp dimensions to image bounds
299        let clamped_w = width.min(img_width.saturating_sub(x));
300        let clamped_h = height.min(img_height.saturating_sub(y));
301
302        if clamped_w == 0 || clamped_h == 0 {
303            return Err("Crop region has zero width or height".to_string());
304        }
305
306        // Crop the image
307        let cropped = img.crop_imm(x, y, clamped_w, clamped_h);
308
309        // Encode back to PNG
310        let mut buf = Vec::new();
311        cropped
312            .write_to(&mut Cursor::new(&mut buf), image::ImageFormat::Png)
313            .map_err(|e| format!("Failed to encode PNG: {}", e))?;
314
315        Ok(buf)
316    }
317}
egui_mcp_client/server.rs

egui_mcp_client/
server.rs
