Skip to main content

computer_use_linux/
screenshot.rs

1use crate::diagnostics::hydrate_session_bus_env;
2use crate::x11_display::{is_native_x11_session, with_x11_display, X11_CAPTURE_TIMEOUT};
3use anyhow::{anyhow, bail, Context, Result};
4use base64::{engine::general_purpose::STANDARD, Engine};
5use futures_util::StreamExt;
6use image::codecs::jpeg::JpegEncoder;
7use image::imageops::FilterType;
8use schemars::JsonSchema;
9use serde::{Deserialize, Serialize};
10use std::{
11    collections::HashMap,
12    fs::{self, OpenOptions},
13    io::{Cursor, Read},
14    os::unix::fs::OpenOptionsExt,
15    path::{Path, PathBuf},
16    process::Stdio,
17    time::{Duration, SystemTime, UNIX_EPOCH},
18};
19use tokio::process::Command;
20use zbus::{
21    message::{Message, Type as MessageType},
22    names::{BusName, OwnedUniqueName, WellKnownName},
23    zvariant::{OwnedObjectPath, OwnedValue, Value},
24    MatchRule, MessageStream, Proxy,
25};
26
27const PORTAL_DESKTOP_SERVICE: &str = "org.freedesktop.portal.Desktop";
28const PORTAL_DESKTOP_PATH: &str = "/org/freedesktop/portal/desktop";
29const PORTAL_SCREENSHOT_INTERFACE: &str = "org.freedesktop.portal.Screenshot";
30const PORTAL_REQUEST_INTERFACE: &str = "org.freedesktop.portal.Request";
31const PORTAL_REQUEST_PATH_NAMESPACE: &str = "/org/freedesktop/portal/desktop/request";
32const MAX_SCREENSHOT_SOURCE_BYTES: u64 = 64 * 1024 * 1024;
33const MAX_SCREENSHOT_SOURCE_PIXELS: u64 = 128 * 1024 * 1024;
34
35pub const DEFAULT_SCREENSHOT_MAX_DIMENSION: u32 = 1920;
36pub const DEFAULT_SCREENSHOT_MAX_BYTES: usize = 2 * 1024 * 1024;
37pub const ABSOLUTE_SCREENSHOT_MAX_DIMENSION: u32 = 4096;
38pub const ABSOLUTE_SCREENSHOT_MAX_BYTES: usize = 4 * 1024 * 1024;
39pub const DEFAULT_SCREENSHOT_JPEG_QUALITY: u8 = 80;
40pub const MIN_SCREENSHOT_JPEG_QUALITY: u8 = 1;
41pub const MAX_SCREENSHOT_JPEG_QUALITY: u8 = 95;
42const MIN_SCREENSHOT_MAX_BYTES: usize = 1024;
43
44#[derive(Debug, Clone)]
45pub struct RawScreenshotCapture {
46    pub mime_type: String,
47    pub bytes: Vec<u8>,
48    pub source: String,
49    pub width: u32,
50    pub height: u32,
51}
52
53#[derive(Debug, Clone, Serialize, JsonSchema)]
54pub struct ScreenshotCapture {
55    pub mime_type: String,
56    pub data_url: String,
57    pub source: String,
58    /// Width of the returned image payload.
59    pub width: u32,
60    /// Height of the returned image payload.
61    pub height: u32,
62    /// Coordinate-space width before payload downscaling.
63    pub coordinate_width: u32,
64    /// Coordinate-space height before payload downscaling.
65    pub coordinate_height: u32,
66    /// Returned pixels per coordinate-space pixel.
67    pub scale: f32,
68    pub resized: bool,
69    pub bytes: usize,
70    pub original_bytes: usize,
71    pub max_bytes: usize,
72    pub format: ScreenshotOutputFormat,
73    pub quality: Option<u8>,
74}
75
76#[derive(Debug, Clone, Copy, Default)]
77pub struct ScreenshotPayloadOptions {
78    pub max_width: Option<u32>,
79    pub max_height: Option<u32>,
80    pub max_bytes: Option<usize>,
81    pub scale: Option<f32>,
82    pub format: Option<ScreenshotOutputFormat>,
83    pub quality: Option<u8>,
84}
85
86#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
87#[serde(rename_all = "lowercase")]
88pub enum ScreenshotOutputFormat {
89    Png,
90    Jpeg,
91}
92
93impl ScreenshotOutputFormat {
94    fn mime_type(self) -> &'static str {
95        match self {
96            Self::Png => "image/png",
97            Self::Jpeg => "image/jpeg",
98        }
99    }
100}
101
102#[derive(Debug, Clone, Copy)]
103struct ResolvedScreenshotPayloadOptions {
104    max_width: u32,
105    max_height: u32,
106    max_bytes: usize,
107    scale: f32,
108    format: ScreenshotOutputFormat,
109    quality: u8,
110}
111
112#[derive(Debug, Clone, PartialEq, Eq)]
113enum ScreenshotCleanup {
114    DeletePath(PathBuf),
115    Preserve,
116}
117
118impl ScreenshotPayloadOptions {
119    fn resolve(self) -> ResolvedScreenshotPayloadOptions {
120        let max_width = self
121            .max_width
122            .unwrap_or(DEFAULT_SCREENSHOT_MAX_DIMENSION)
123            .clamp(1, ABSOLUTE_SCREENSHOT_MAX_DIMENSION);
124        let max_height = self
125            .max_height
126            .unwrap_or(DEFAULT_SCREENSHOT_MAX_DIMENSION)
127            .clamp(1, ABSOLUTE_SCREENSHOT_MAX_DIMENSION);
128        let max_bytes = self
129            .max_bytes
130            .unwrap_or(DEFAULT_SCREENSHOT_MAX_BYTES)
131            .clamp(MIN_SCREENSHOT_MAX_BYTES, ABSOLUTE_SCREENSHOT_MAX_BYTES);
132        let scale = self
133            .scale
134            .filter(|value| value.is_finite() && *value > 0.0)
135            .unwrap_or(1.0)
136            .min(1.0);
137        let format = self.format.unwrap_or(ScreenshotOutputFormat::Png);
138        let quality = self
139            .quality
140            .unwrap_or(DEFAULT_SCREENSHOT_JPEG_QUALITY)
141            .clamp(MIN_SCREENSHOT_JPEG_QUALITY, MAX_SCREENSHOT_JPEG_QUALITY);
142
143        ResolvedScreenshotPayloadOptions {
144            max_width,
145            max_height,
146            max_bytes,
147            scale,
148            format,
149            quality,
150        }
151    }
152}
153
154/// Environment variable forcing a single capture backend, skipping the
155/// fallback chain. Accepts `gnome-shell`, `portal`, `x11`, or `gnome-screenshot`.
156const SCREENSHOT_BACKEND_ENV: &str = "COMPUTER_USE_LINUX_SCREENSHOT_BACKEND";
157
158#[derive(Debug, Clone, Copy, PartialEq, Eq)]
159enum ScreenshotBackend {
160    GnomeShell,
161    Portal,
162    X11,
163    GnomeScreenshot,
164}
165
166impl ScreenshotBackend {
167    fn parse(value: &str) -> Option<Self> {
168        match value.trim().to_ascii_lowercase().as_str() {
169            "gnome-shell" | "gnome_shell" | "shell" => Some(Self::GnomeShell),
170            "portal" | "xdg-portal" | "xdg_portal" => Some(Self::Portal),
171            "x11" | "x11-native" | "x11_native" | "xgetimage" => Some(Self::X11),
172            "gnome-screenshot" | "gnome_screenshot" => Some(Self::GnomeScreenshot),
173            _ => None,
174        }
175    }
176
177    async fn capture(self) -> Result<RawScreenshotCapture> {
178        match self {
179            Self::GnomeShell => capture_with_gnome_shell().await,
180            Self::Portal => capture_with_portal().await,
181            Self::X11 => capture_with_x11().await,
182            Self::GnomeScreenshot => capture_with_gnome_screenshot().await,
183        }
184    }
185}
186
187pub async fn capture_screenshot_raw() -> Result<RawScreenshotCapture> {
188    hydrate_session_bus_env();
189
190    // Explicit override: use exactly the requested backend, no fallback. Lets
191    // background/systemd contexts pin `gnome-screenshot` when the DBus paths are
192    // blocked, and aids debugging.
193    if let Some(forced) = forced_backend()? {
194        return forced.capture().await;
195    }
196
197    // The Shell and portal DBus paths fail for background processes (systemd
198    // user services, non-interactive parent shells): GNOME Shell's
199    // DBusSenderChecker rejects unknown bus names, and the portal cancels with
200    // response code 2 when there is no foreground window. `gnome-screenshot`
201    // claims an allowlisted bus name and works regardless, so it is the final
202    // fallback. See issue #20.
203    let gnome_error = match capture_with_gnome_shell().await {
204        Ok(capture) => return Ok(capture),
205        Err(error) => error,
206    };
207    let portal_error = match capture_with_portal().await {
208        Ok(capture) => return Ok(capture),
209        Err(error) => error,
210    };
211    // Native X11 only, and ahead of gnome-screenshot: gnome-screenshot 41 masks
212    // everything outside the GDK monitor geometry, which is 1/4 of the frame at
213    // window-scaling-factor 2 on MATE (issue #155). GetImage has no GDK layer.
214    let x11_error = match capture_with_x11().await {
215        Ok(capture) => return Ok(capture),
216        Err(error) => error,
217    };
218    let cli_error = match capture_with_gnome_screenshot().await {
219        Ok(capture) => return Ok(capture),
220        Err(error) => error,
221    };
222
223    Err(anyhow!(
224        "GNOME Shell screenshot failed: {gnome_error}; \
225         XDG portal screenshot failed: {portal_error}; \
226         native X11 screenshot failed: {x11_error}; \
227         gnome-screenshot fallback failed: {cli_error}"
228    ))
229}
230
231/// `GetImage` on the root window of a native X11 session. Pixels are device
232/// pixels, the space xdotool/XTEST input and X11 window origins use.
233async fn capture_with_x11() -> Result<RawScreenshotCapture> {
234    if !is_native_x11_session() {
235        bail!("not a native X11 session (needs DISPLAY on an X11, not Wayland, session)");
236    }
237    let image = with_x11_display(X11_CAPTURE_TIMEOUT, |display| display.capture_root()).await??;
238    let (width, height) = (image.width, image.height);
239    let bytes =
240        tokio::task::spawn_blocking(move || encode_rgb_png(image.width, image.height, image.rgb))
241            .await
242            .context("X11 screenshot encoder task failed")??;
243    Ok(RawScreenshotCapture {
244        mime_type: "image/png".to_string(),
245        bytes,
246        source: "x11".to_string(),
247        width,
248        height,
249    })
250}
251
252fn encode_rgb_png(width: u32, height: u32, rgb: Vec<u8>) -> Result<Vec<u8>> {
253    let buffer = image::RgbImage::from_raw(width, height, rgb)
254        .context("X11 root image did not match its dimensions")?;
255    let mut out = Vec::new();
256    image::DynamicImage::ImageRgb8(buffer)
257        .write_to(&mut Cursor::new(&mut out), image::ImageFormat::Png)
258        .context("failed to encode X11 screenshot PNG")?;
259    Ok(out)
260}
261
262fn forced_backend() -> Result<Option<ScreenshotBackend>> {
263    match std::env::var(SCREENSHOT_BACKEND_ENV) {
264        Ok(value) if !value.trim().is_empty() => {
265            ScreenshotBackend::parse(&value).map(Some).ok_or_else(|| {
266                anyhow!(
267                    "{SCREENSHOT_BACKEND_ENV}={value:?} is not a recognized backend \
268                     (expected gnome-shell, portal, x11, or gnome-screenshot)"
269                )
270            })
271        }
272        _ => Ok(None),
273    }
274}
275
276pub async fn capture_screenshot() -> Result<ScreenshotCapture> {
277    let raw = capture_screenshot_raw().await?;
278    prepare_screenshot_payload(raw, ScreenshotPayloadOptions::default())
279}
280
281pub fn prepare_screenshot_payload(
282    raw: RawScreenshotCapture,
283    options: ScreenshotPayloadOptions,
284) -> Result<ScreenshotCapture> {
285    if raw.bytes.is_empty() {
286        bail!("screenshot file was empty");
287    }
288    let (coordinate_width, coordinate_height) = png_dimensions(&raw.bytes)?;
289    let original_bytes = raw.bytes.len();
290    let options = options.resolve();
291    let (target_width, target_height) =
292        target_dimensions(coordinate_width, coordinate_height, options);
293
294    let (bytes, width, height) = if options.format == ScreenshotOutputFormat::Png
295        && target_width == coordinate_width
296        && target_height == coordinate_height
297        && original_bytes <= options.max_bytes
298    {
299        (raw.bytes, coordinate_width, coordinate_height)
300    } else {
301        encode_screenshot_to_fit_bytes(
302            &raw.bytes,
303            coordinate_width,
304            coordinate_height,
305            target_width,
306            target_height,
307            options,
308        )?
309    };
310
311    let encoded = STANDARD.encode(&bytes);
312    let scale = if coordinate_width == 0 {
313        1.0
314    } else {
315        width as f32 / coordinate_width as f32
316    };
317
318    Ok(ScreenshotCapture {
319        mime_type: options.format.mime_type().to_string(),
320        data_url: format!("data:{};base64,{encoded}", options.format.mime_type()),
321        source: raw.source,
322        width,
323        height,
324        coordinate_width,
325        coordinate_height,
326        scale,
327        resized: width != coordinate_width || height != coordinate_height,
328        bytes: bytes.len(),
329        original_bytes,
330        max_bytes: options.max_bytes,
331        format: options.format,
332        quality: (options.format == ScreenshotOutputFormat::Jpeg).then_some(options.quality),
333    })
334}
335
336async fn capture_with_gnome_shell() -> Result<RawScreenshotCapture> {
337    let connection = zbus::Connection::session()
338        .await
339        .context("failed to connect to session bus")?;
340    let proxy = Proxy::new(
341        &connection,
342        "org.gnome.Shell.Screenshot",
343        "/org/gnome/Shell/Screenshot",
344        "org.gnome.Shell.Screenshot",
345    )
346    .await
347    .context("failed to create GNOME Shell screenshot proxy")?;
348    let path = temp_png_path("gnome-shell");
349    let filename = path
350        .to_str()
351        .context("temporary screenshot path is not valid UTF-8")?;
352    let result = proxy.call("Screenshot", &(false, false, filename)).await;
353    let (success, filename_used): (bool, String) = match result {
354        Ok(result) => result,
355        Err(error) => {
356            cleanup_gnome_requested_path(&path);
357            return Err(error).context("GNOME Shell Screenshot call failed");
358        }
359    };
360
361    if !success {
362        cleanup_gnome_requested_path(&path);
363        bail!("GNOME Shell reported screenshot failure");
364    }
365
366    read_png_as_capture(
367        PathBuf::from(filename_used),
368        "gnome-shell",
369        ScreenshotCleanup::DeletePath(path),
370    )
371    .await
372}
373
374async fn capture_with_portal() -> Result<RawScreenshotCapture> {
375    let connection = zbus::Connection::session()
376        .await
377        .context("failed to connect to session bus")?;
378    let portal_owner = portal_desktop_owner(&connection).await?;
379    let token = request_token();
380    // Some portals rewrite the request handle, so subscribe before calling Screenshot
381    // and filter by the returned handle instead of subscribing after the call.
382    let mut response_stream = portal_response_stream(&connection, &portal_owner).await?;
383
384    let portal_proxy = Proxy::new(
385        &connection,
386        portal_owner.clone(),
387        PORTAL_DESKTOP_PATH,
388        PORTAL_SCREENSHOT_INTERFACE,
389    )
390    .await
391    .context("failed to create XDG portal screenshot proxy")?;
392    let mut options: HashMap<&str, Value<'_>> = HashMap::new();
393    options.insert("handle_token", Value::from(token.as_str()));
394    options.insert("interactive", Value::from(false));
395    let handle: OwnedObjectPath = portal_proxy
396        .call("Screenshot", &("", options))
397        .await
398        .context("XDG portal Screenshot call failed")?;
399
400    let (response_code, results) = tokio::time::timeout(
401        Duration::from_secs(20),
402        wait_for_portal_response(&mut response_stream, handle.as_str(), portal_owner.as_str()),
403    )
404    .await
405    .context("timed out waiting for XDG portal screenshot response")??;
406
407    if response_code != 0 {
408        bail!("XDG portal screenshot was denied or cancelled with response code {response_code}");
409    }
410
411    let uri_value = results
412        .get("uri")
413        .context("XDG portal screenshot response did not include a uri")?;
414    let uri: String = uri_value
415        .try_clone()
416        .context("failed to clone XDG portal screenshot uri")?
417        .try_into()
418        .context("XDG portal screenshot uri was not a string")?;
419    let path = file_uri_to_path(&uri)?;
420
421    read_png_as_capture(path, "xdg-desktop-portal", ScreenshotCleanup::Preserve).await
422}
423
424/// Upper bound on how long we wait for `gnome-screenshot` before killing it.
425/// Matches the portal timeout: a hung capture must not block the tool forever.
426const GNOME_SCREENSHOT_TIMEOUT: Duration = Duration::from_secs(20);
427
428async fn capture_with_gnome_screenshot() -> Result<RawScreenshotCapture> {
429    let path = temp_png_path("gnome-screenshot");
430    let filename = path
431        .to_str()
432        .context("temporary screenshot path is not valid UTF-8")?;
433
434    // `-f <file>` writes a full-screen PNG without prompting; no portal, no
435    // foreground window required. `tokio::process::Command` searches PATH and
436    // provides an async, non-polling wait.
437    let mut command = Command::new("gnome-screenshot");
438    command
439        .args(["-f", filename])
440        .stdin(Stdio::null())
441        .stdout(Stdio::null())
442        .stderr(Stdio::null());
443    let mut child = match crate::command_runner::spawn_retrying_busy(&mut command).await {
444        Ok(child) => child,
445        Err(error) => {
446            cleanup_gnome_requested_path(&path);
447            return Err(error).context("failed to spawn gnome-screenshot");
448        }
449    };
450
451    // A hung capture must not block the tool forever, so bound the wait and
452    // kill the child if it outlives the deadline.
453    let status = match tokio::time::timeout(GNOME_SCREENSHOT_TIMEOUT, child.wait()).await {
454        Ok(Ok(status)) => status,
455        Ok(Err(error)) => {
456            cleanup_gnome_requested_path(&path);
457            return Err(error).context("failed to wait for gnome-screenshot");
458        }
459        Err(_) => {
460            let _ = child.kill().await;
461            cleanup_gnome_requested_path(&path);
462            bail!("gnome-screenshot timed out");
463        }
464    };
465
466    if !status.success() {
467        cleanup_gnome_requested_path(&path);
468        bail!("gnome-screenshot exited with {status}");
469    }
470
471    read_png_as_capture(
472        path.clone(),
473        "gnome-screenshot",
474        ScreenshotCleanup::DeletePath(path),
475    )
476    .await
477}
478
479async fn portal_desktop_owner(connection: &zbus::Connection) -> Result<OwnedUniqueName> {
480    let dbus = zbus::fdo::DBusProxy::new(connection)
481        .await
482        .context("failed to create session-bus identity proxy")?;
483    let _: u32 = dbus
484        .start_service_by_name(WellKnownName::try_from(PORTAL_DESKTOP_SERVICE)?, 0)
485        .await
486        .context("failed to activate the XDG desktop portal")?;
487    dbus.get_name_owner(BusName::try_from(PORTAL_DESKTOP_SERVICE)?)
488        .await
489        .context("failed to resolve the XDG desktop portal owner")
490}
491
492async fn portal_response_stream(
493    connection: &zbus::Connection,
494    portal_owner: &OwnedUniqueName,
495) -> Result<MessageStream> {
496    let response_rule = MatchRule::builder()
497        .msg_type(MessageType::Signal)
498        .sender(portal_owner.clone())?
499        .interface(PORTAL_REQUEST_INTERFACE)?
500        .member("Response")?
501        .path_namespace(PORTAL_REQUEST_PATH_NAMESPACE)?
502        .build();
503
504    MessageStream::for_match_rule(response_rule, connection, None)
505        .await
506        .context("failed to subscribe to XDG portal screenshot responses")
507}
508
509async fn wait_for_portal_response(
510    response_stream: &mut MessageStream,
511    request_path: &str,
512    portal_owner: &str,
513) -> Result<(u32, HashMap<String, OwnedValue>)> {
514    loop {
515        let response = response_stream
516            .next()
517            .await
518            .context("XDG portal screenshot response stream ended")?
519            .context("XDG portal screenshot response stream failed")?;
520
521        if !portal_response_matches(&response, request_path, portal_owner) {
522            continue;
523        }
524
525        return response
526            .body()
527            .deserialize()
528            .context("failed to decode XDG portal screenshot response");
529    }
530}
531
532fn portal_response_matches(response: &Message, request_path: &str, portal_owner: &str) -> bool {
533    let header = response.header();
534    header
535        .sender()
536        .is_some_and(|sender| sender.as_str() == portal_owner)
537        && header
538            .path()
539            .is_some_and(|path| path.as_str() == request_path)
540}
541
542async fn read_png_as_capture(
543    path: PathBuf,
544    source: &str,
545    cleanup: ScreenshotCleanup,
546) -> Result<RawScreenshotCapture> {
547    let result = read_png_as_capture_inner(&path, source);
548    if let ScreenshotCleanup::DeletePath(path) = cleanup {
549        let _ = fs::remove_file(path);
550    }
551    result
552}
553
554fn read_png_as_capture_inner(path: &Path, source: &str) -> Result<RawScreenshotCapture> {
555    let file = OpenOptions::new()
556        .read(true)
557        .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK)
558        .open(path)
559        .with_context(|| format!("failed to open screenshot file {}", path.display()))?;
560    let metadata = file
561        .metadata()
562        .with_context(|| format!("failed to inspect screenshot file {}", path.display()))?;
563    if !metadata.file_type().is_file() {
564        bail!("screenshot path is not a regular file: {}", path.display());
565    }
566    if metadata.len() > MAX_SCREENSHOT_SOURCE_BYTES {
567        bail!(
568            "screenshot file is {} bytes, over the {}-byte source limit",
569            metadata.len(),
570            MAX_SCREENSHOT_SOURCE_BYTES
571        );
572    }
573    let mut bytes = Vec::with_capacity(metadata.len() as usize);
574    file.take(MAX_SCREENSHOT_SOURCE_BYTES + 1)
575        .read_to_end(&mut bytes)
576        .with_context(|| format!("failed to read screenshot file {}", path.display()))?;
577    if bytes.len() as u64 > MAX_SCREENSHOT_SOURCE_BYTES {
578        bail!(
579            "screenshot file grew beyond the {}-byte source limit",
580            MAX_SCREENSHOT_SOURCE_BYTES
581        );
582    }
583    if bytes.is_empty() {
584        bail!("screenshot file was empty: {}", path.display());
585    }
586    let (width, height) = png_dimensions(&bytes)?;
587    Ok(RawScreenshotCapture {
588        mime_type: "image/png".to_string(),
589        bytes,
590        source: source.to_string(),
591        width,
592        height,
593    })
594}
595
596fn target_dimensions(
597    width: u32,
598    height: u32,
599    options: ResolvedScreenshotPayloadOptions,
600) -> (u32, u32) {
601    let width_scale = options.max_width as f64 / width as f64;
602    let height_scale = options.max_height as f64 / height as f64;
603    let scale = f64::from(options.scale)
604        .min(width_scale)
605        .min(height_scale)
606        .min(1.0);
607
608    let target_width = ((width as f64 * scale).round() as u32).clamp(1, width);
609    let target_height = ((height as f64 * scale).round() as u32).clamp(1, height);
610    (target_width, target_height)
611}
612
613fn encode_screenshot_to_fit_bytes(
614    raw: &[u8],
615    original_width: u32,
616    original_height: u32,
617    mut target_width: u32,
618    mut target_height: u32,
619    options: ResolvedScreenshotPayloadOptions,
620) -> Result<(Vec<u8>, u32, u32)> {
621    let img = image::load_from_memory_with_format(raw, image::ImageFormat::Png)
622        .context("failed to decode screenshot PNG for encoding")?;
623
624    loop {
625        let bytes = if options.format == ScreenshotOutputFormat::Png
626            && target_width == original_width
627            && target_height == original_height
628        {
629            raw.to_vec()
630        } else {
631            let output = if target_width == original_width && target_height == original_height {
632                img.clone()
633            } else {
634                img.resize_exact(target_width, target_height, FilterType::Lanczos3)
635            };
636            encode_image(&output, options)?
637        };
638
639        if bytes.len() <= options.max_bytes {
640            return Ok((bytes, target_width, target_height));
641        }
642
643        if target_width == 1 && target_height == 1 {
644            bail!(
645                "screenshot payload is {} bytes at 1x1, over max_bytes {}",
646                bytes.len(),
647                options.max_bytes
648            );
649        }
650
651        (target_width, target_height) = next_dimensions_for_byte_cap(
652            target_width,
653            target_height,
654            bytes.len(),
655            options.max_bytes,
656        );
657    }
658}
659
660fn encode_image(
661    img: &image::DynamicImage,
662    options: ResolvedScreenshotPayloadOptions,
663) -> Result<Vec<u8>> {
664    let mut out = Vec::new();
665    match options.format {
666        ScreenshotOutputFormat::Png => {
667            img.write_to(&mut Cursor::new(&mut out), image::ImageFormat::Png)
668                .context("failed to encode screenshot PNG")?;
669        }
670        ScreenshotOutputFormat::Jpeg => {
671            let rgb = img.to_rgb8();
672            JpegEncoder::new_with_quality(&mut out, options.quality)
673                .encode_image(&rgb)
674                .context("failed to encode screenshot JPEG")?;
675        }
676    }
677    Ok(out)
678}
679
680fn next_dimensions_for_byte_cap(
681    width: u32,
682    height: u32,
683    encoded_bytes: usize,
684    max_bytes: usize,
685) -> (u32, u32) {
686    let shrink = ((max_bytes as f64 / encoded_bytes as f64).sqrt() * 0.9).clamp(0.1, 0.95);
687    let mut next_width = ((width as f64 * shrink).floor() as u32).max(1);
688    let mut next_height = ((height as f64 * shrink).floor() as u32).max(1);
689
690    if next_width >= width && width > 1 {
691        next_width = width - 1;
692    }
693    if next_height >= height && height > 1 {
694        next_height = height - 1;
695    }
696
697    (next_width, next_height)
698}
699
700fn cleanup_gnome_requested_path(path: &Path) {
701    let _ = fs::remove_file(path);
702}
703
704fn png_dimensions(bytes: &[u8]) -> Result<(u32, u32)> {
705    const PNG_SIGNATURE: &[u8; 8] = b"\x89PNG\r\n\x1a\n";
706    if bytes.len() < 24 || &bytes[..8] != PNG_SIGNATURE || &bytes[12..16] != b"IHDR" {
707        bail!("screenshot file was not a valid PNG");
708    }
709    let width = u32::from_be_bytes(bytes[16..20].try_into().unwrap());
710    let height = u32::from_be_bytes(bytes[20..24].try_into().unwrap());
711    if width == 0 || height == 0 {
712        bail!("screenshot PNG had invalid dimensions {width}x{height}");
713    }
714    let pixels = u64::from(width) * u64::from(height);
715    if pixels > MAX_SCREENSHOT_SOURCE_PIXELS {
716        bail!(
717            "screenshot PNG has {pixels} pixels, over the {MAX_SCREENSHOT_SOURCE_PIXELS}-pixel source limit"
718        );
719    }
720    Ok((width, height))
721}
722
723fn file_uri_to_path(uri: &str) -> Result<PathBuf> {
724    let Some(rest) = uri.strip_prefix("file://") else {
725        bail!("unsupported screenshot uri: {uri}");
726    };
727    let path = PathBuf::from(percent_decode(rest));
728    if !path.is_absolute() {
729        bail!("screenshot uri did not contain an absolute file path: {uri}");
730    }
731    Ok(path)
732}
733
734fn percent_decode(value: &str) -> String {
735    let bytes = value.as_bytes();
736    let mut decoded = Vec::with_capacity(bytes.len());
737    let mut index = 0;
738
739    while index < bytes.len() {
740        if bytes[index] == b'%' && index + 2 < bytes.len() {
741            if let Ok(hex) = std::str::from_utf8(&bytes[index + 1..index + 3]) {
742                if let Ok(byte) = u8::from_str_radix(hex, 16) {
743                    decoded.push(byte);
744                    index += 3;
745                    continue;
746                }
747            }
748        }
749
750        decoded.push(bytes[index]);
751        index += 1;
752    }
753
754    String::from_utf8_lossy(&decoded).into_owned()
755}
756
757fn temp_png_path(source: &str) -> PathBuf {
758    std::env::temp_dir().join(format!(
759        "computer-use-linux-{source}-{}.png",
760        unique_suffix()
761    ))
762}
763
764fn request_token() -> String {
765    format!("computer_use_linux_{}", unique_suffix().replace('-', "_"))
766}
767
768fn unique_suffix() -> String {
769    let nanos = SystemTime::now()
770        .duration_since(UNIX_EPOCH)
771        .map(|duration| duration.as_nanos())
772        .unwrap_or_default();
773    format!("{}-{nanos}", std::process::id())
774}
775
776#[cfg(test)]
777mod tests {
778    use super::*;
779
780    fn test_path(name: &str) -> PathBuf {
781        std::env::temp_dir().join(format!(
782            "computer-use-linux-screenshot-test-{name}-{}",
783            unique_suffix()
784        ))
785    }
786
787    fn valid_png(width: u32, height: u32) -> Vec<u8> {
788        let mut png = Vec::new();
789        png.extend_from_slice(b"\x89PNG\r\n\x1a\n");
790        png.extend_from_slice(&13_u32.to_be_bytes());
791        png.extend_from_slice(b"IHDR");
792        png.extend_from_slice(&width.to_be_bytes());
793        png.extend_from_slice(&height.to_be_bytes());
794        png.extend_from_slice(&[8, 6, 0, 0, 0]);
795        png
796    }
797
798    fn solid_png(width: u32, height: u32) -> Vec<u8> {
799        let img = image::RgbaImage::from_pixel(width, height, image::Rgba([24, 96, 160, 255]));
800        encode_test_png(img)
801    }
802
803    fn noisy_png(width: u32, height: u32) -> Vec<u8> {
804        let mut img = image::RgbaImage::new(width, height);
805        for (x, y, pixel) in img.enumerate_pixels_mut() {
806            let r = ((x * 31 + y * 17) % 256) as u8;
807            let g = ((x * 13 + y * 47) % 256) as u8;
808            let b = ((x * 97 + y * 7) % 256) as u8;
809            *pixel = image::Rgba([r, g, b, 255]);
810        }
811        encode_test_png(img)
812    }
813
814    fn encode_test_png(img: image::RgbaImage) -> Vec<u8> {
815        let mut out = Vec::new();
816        image::DynamicImage::ImageRgba8(img)
817            .write_to(&mut Cursor::new(&mut out), image::ImageFormat::Png)
818            .unwrap();
819        out
820    }
821
822    fn raw_capture(bytes: Vec<u8>) -> RawScreenshotCapture {
823        let (width, height) = png_dimensions(&bytes).unwrap();
824        RawScreenshotCapture {
825            mime_type: "image/png".to_string(),
826            bytes,
827            source: "test".to_string(),
828            width,
829            height,
830        }
831    }
832
833    #[test]
834    fn decodes_file_uri_percent_escapes() {
835        assert_eq!(
836            file_uri_to_path("file:///tmp/Codex%20Screenshot.png").unwrap(),
837            PathBuf::from("/tmp/Codex Screenshot.png")
838        );
839    }
840
841    #[test]
842    fn file_uri_requires_an_absolute_path() {
843        assert!(file_uri_to_path("file://relative/path.png").is_err());
844    }
845
846    #[test]
847    fn portal_response_requires_the_pinned_sender_and_exact_path() {
848        let request_path = "/org/freedesktop/portal/desktop/request/1_7/token";
849        let response = Message::signal(request_path, PORTAL_REQUEST_INTERFACE, "Response")
850            .unwrap()
851            .sender(":1.42")
852            .unwrap()
853            .build(&())
854            .unwrap();
855
856        assert!(portal_response_matches(&response, request_path, ":1.42"));
857        assert!(!portal_response_matches(&response, request_path, ":1.99"));
858        assert!(!portal_response_matches(
859            &response,
860            "/org/freedesktop/portal/desktop/request/1_7/other",
861            ":1.42"
862        ));
863    }
864
865    #[test]
866    fn parses_known_backend_names() {
867        assert_eq!(
868            ScreenshotBackend::parse("gnome-shell"),
869            Some(ScreenshotBackend::GnomeShell)
870        );
871        assert_eq!(
872            ScreenshotBackend::parse("  Portal "),
873            Some(ScreenshotBackend::Portal)
874        );
875        assert_eq!(
876            ScreenshotBackend::parse("GNOME_SCREENSHOT"),
877            Some(ScreenshotBackend::GnomeScreenshot)
878        );
879        assert_eq!(
880            ScreenshotBackend::parse("x11"),
881            Some(ScreenshotBackend::X11)
882        );
883        assert_eq!(
884            ScreenshotBackend::parse(" X11-Native "),
885            Some(ScreenshotBackend::X11)
886        );
887        assert_eq!(ScreenshotBackend::parse("nonsense"), None);
888    }
889
890    #[test]
891    fn forced_backend_reads_env_override() {
892        // Only this test touches SCREENSHOT_BACKEND_ENV, so no cross-test race.
893        std::env::set_var(SCREENSHOT_BACKEND_ENV, "gnome-screenshot");
894        assert_eq!(
895            forced_backend().unwrap(),
896            Some(ScreenshotBackend::GnomeScreenshot)
897        );
898
899        std::env::set_var(SCREENSHOT_BACKEND_ENV, "   ");
900        assert_eq!(forced_backend().unwrap(), None);
901
902        std::env::set_var(SCREENSHOT_BACKEND_ENV, "bogus");
903        let error = forced_backend().unwrap_err();
904        assert!(error.to_string().contains("not a recognized backend"));
905
906        std::env::remove_var(SCREENSHOT_BACKEND_ENV);
907        assert_eq!(forced_backend().unwrap(), None);
908    }
909
910    #[test]
911    fn request_token_is_portal_safe() {
912        let token = request_token();
913        assert!(token.starts_with("computer_use_linux_"));
914        assert!(token.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'));
915    }
916
917    #[test]
918    fn reads_png_dimensions_from_ihdr() {
919        let png = valid_png(3840, 1080);
920
921        assert_eq!(png_dimensions(&png).unwrap(), (3840, 1080));
922    }
923
924    #[test]
925    fn default_payload_downscales_long_edge() {
926        let capture =
927            prepare_screenshot_payload(raw_capture(solid_png(4000, 1000)), Default::default())
928                .unwrap();
929
930        assert_eq!((capture.width, capture.height), (1920, 480));
931        assert_eq!(
932            (capture.coordinate_width, capture.coordinate_height),
933            (4000, 1000)
934        );
935        assert!(capture.resized);
936        assert!(capture.bytes <= DEFAULT_SCREENSHOT_MAX_BYTES);
937        assert!(capture.data_url.starts_with("data:image/png;base64,"));
938    }
939
940    #[test]
941    fn larger_bounded_request_can_keep_more_detail() {
942        let capture = prepare_screenshot_payload(
943            raw_capture(solid_png(3000, 1000)),
944            ScreenshotPayloadOptions {
945                max_width: Some(3000),
946                max_height: Some(3000),
947                max_bytes: Some(DEFAULT_SCREENSHOT_MAX_BYTES),
948                ..Default::default()
949            },
950        )
951        .unwrap();
952
953        assert_eq!((capture.width, capture.height), (3000, 1000));
954        assert_eq!(
955            (capture.coordinate_width, capture.coordinate_height),
956            (3000, 1000)
957        );
958        assert!(!capture.resized);
959    }
960
961    #[test]
962    fn byte_cap_downscales_until_payload_fits() {
963        let capture = prepare_screenshot_payload(
964            raw_capture(noisy_png(512, 512)),
965            ScreenshotPayloadOptions {
966                max_width: Some(512),
967                max_height: Some(512),
968                max_bytes: Some(20_000),
969                ..Default::default()
970            },
971        )
972        .unwrap();
973
974        assert!(capture.bytes <= 20_000);
975        assert!(capture.width < 512);
976        assert_eq!(
977            (capture.coordinate_width, capture.coordinate_height),
978            (512, 512)
979        );
980        assert!(capture.resized);
981    }
982
983    #[test]
984    fn jpeg_format_compresses_when_requested() {
985        let capture = prepare_screenshot_payload(
986            raw_capture(noisy_png(512, 512)),
987            ScreenshotPayloadOptions {
988                max_width: Some(512),
989                max_height: Some(512),
990                max_bytes: Some(DEFAULT_SCREENSHOT_MAX_BYTES),
991                format: Some(ScreenshotOutputFormat::Jpeg),
992                quality: Some(60),
993                ..Default::default()
994            },
995        )
996        .unwrap();
997
998        assert_eq!(capture.mime_type, "image/jpeg");
999        assert_eq!(capture.format, ScreenshotOutputFormat::Jpeg);
1000        assert_eq!(capture.quality, Some(60));
1001        assert_eq!((capture.width, capture.height), (512, 512));
1002        assert_eq!(
1003            (capture.coordinate_width, capture.coordinate_height),
1004            (512, 512)
1005        );
1006        assert!(capture.bytes < capture.original_bytes);
1007        assert!(capture.data_url.starts_with("data:image/jpeg;base64,"));
1008    }
1009
1010    #[tokio::test]
1011    async fn portal_capture_preserves_valid_returned_path() {
1012        let path = test_path("portal-valid");
1013        fs::write(&path, valid_png(1, 1)).unwrap();
1014
1015        let capture = read_png_as_capture(
1016            path.clone(),
1017            "xdg-desktop-portal",
1018            ScreenshotCleanup::Preserve,
1019        )
1020        .await
1021        .unwrap();
1022
1023        assert_eq!(capture.source, "xdg-desktop-portal");
1024        assert!(path.exists());
1025        let _ = fs::remove_file(path);
1026    }
1027
1028    #[tokio::test]
1029    async fn portal_capture_preserves_invalid_returned_path() {
1030        let path = test_path("portal-invalid");
1031        fs::write(&path, b"").unwrap();
1032
1033        let error = read_png_as_capture(
1034            path.clone(),
1035            "xdg-desktop-portal",
1036            ScreenshotCleanup::Preserve,
1037        )
1038        .await
1039        .unwrap_err();
1040
1041        assert!(error.to_string().contains("screenshot file was empty"));
1042        assert!(path.exists());
1043        let _ = fs::remove_file(path);
1044    }
1045
1046    #[tokio::test]
1047    async fn screenshot_reader_rejects_symlinks() {
1048        let target = test_path("symlink-target");
1049        let link = test_path("symlink-link");
1050        fs::write(&target, valid_png(1, 1)).unwrap();
1051        std::os::unix::fs::symlink(&target, &link).unwrap();
1052
1053        let error = read_png_as_capture(link.clone(), "test", ScreenshotCleanup::Preserve)
1054            .await
1055            .unwrap_err();
1056
1057        assert!(error.to_string().contains("failed to open screenshot file"));
1058        let _ = fs::remove_file(link);
1059        let _ = fs::remove_file(target);
1060    }
1061
1062    #[tokio::test]
1063    async fn screenshot_reader_rejects_oversized_source_files() {
1064        let path = test_path("oversized-source");
1065        let file = fs::File::create(&path).unwrap();
1066        file.set_len(MAX_SCREENSHOT_SOURCE_BYTES + 1).unwrap();
1067
1068        let error = read_png_as_capture(path.clone(), "test", ScreenshotCleanup::Preserve)
1069            .await
1070            .unwrap_err();
1071
1072        assert!(error.to_string().contains("over the"));
1073        let _ = fs::remove_file(path);
1074    }
1075
1076    #[test]
1077    fn png_dimensions_reject_decompression_bomb_dimensions() {
1078        let png = valid_png(u32::MAX, u32::MAX);
1079
1080        assert!(png_dimensions(&png)
1081            .unwrap_err()
1082            .to_string()
1083            .contains("pixel source limit"));
1084    }
1085
1086    #[tokio::test]
1087    async fn gnome_capture_deletes_backend_temp_path_on_success() {
1088        let path = test_path("gnome-valid");
1089        fs::write(&path, valid_png(1, 1)).unwrap();
1090
1091        let capture = read_png_as_capture(
1092            path.clone(),
1093            "gnome-shell",
1094            ScreenshotCleanup::DeletePath(path.clone()),
1095        )
1096        .await
1097        .unwrap();
1098
1099        assert_eq!(capture.source, "gnome-shell");
1100        assert!(!path.exists());
1101    }
1102
1103    #[tokio::test]
1104    async fn gnome_capture_deletes_backend_temp_path_on_parse_failure() {
1105        let path = test_path("gnome-invalid");
1106        fs::write(&path, b"").unwrap();
1107
1108        let error = read_png_as_capture(
1109            path.clone(),
1110            "gnome-shell",
1111            ScreenshotCleanup::DeletePath(path.clone()),
1112        )
1113        .await
1114        .unwrap_err();
1115
1116        assert!(error.to_string().contains("screenshot file was empty"));
1117        assert!(!path.exists());
1118    }
1119
1120    #[test]
1121    fn gnome_failure_cleanup_removes_requested_temp_path() {
1122        let path = test_path("gnome-pre-read-failure");
1123        fs::write(&path, b"partial").unwrap();
1124
1125        cleanup_gnome_requested_path(&path);
1126
1127        assert!(!path.exists());
1128    }
1129
1130    #[tokio::test]
1131    async fn gnome_deletes_requested_temp_path_and_preserves_unexpected_returned_path() {
1132        let requested = test_path("gnome-requested");
1133        let returned = test_path("gnome-returned");
1134        fs::write(&requested, b"partial").unwrap();
1135        fs::write(&returned, valid_png(1, 1)).unwrap();
1136
1137        let capture = read_png_as_capture(
1138            returned.clone(),
1139            "gnome-shell",
1140            ScreenshotCleanup::DeletePath(requested.clone()),
1141        )
1142        .await
1143        .unwrap();
1144
1145        assert_eq!(capture.source, "gnome-shell");
1146        assert!(!requested.exists());
1147        assert!(returned.exists());
1148        let _ = fs::remove_file(returned);
1149    }
1150}