#![cfg(target_os = "linux")]
use crate::backend::WindowTarget;
use crate::error::ComputerError;
use crate::target::WindowBounds;
pub(crate) const SOURCE_PORTAL: &str = "portal";
pub(crate) const TIER_NAME: &str = "portal";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SourceKind {
Monitor,
Window,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct StreamInfo {
pub kind: SourceKind,
pub position: Option<(i32, i32)>,
pub size: Option<(i32, i32)>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SourcePlan {
Window {
index: usize,
},
Monitor {
index: usize,
crop: CropRect,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct CropRect {
pub x: i32,
pub y: i32,
pub width: u32,
pub height: u32,
}
pub(crate) fn plan_source(
streams: &[StreamInfo],
window: WindowBounds,
) -> Result<SourcePlan, ComputerError> {
if streams.is_empty() {
return Err(ComputerError::DisplayUnavailable(
"the XDG ScreenCast portal returned no streams; the compositor offered nothing to record"
.into(),
));
}
if let Some(index) = streams.iter().position(|s| s.kind == SourceKind::Window) {
return Ok(SourcePlan::Window { index });
}
let centre = window_centre(window);
let mut unknown_extent: Option<SourcePlan> = None;
for (index, stream) in streams.iter().enumerate() {
if stream.kind != SourceKind::Monitor {
continue;
}
let (Some(position), Some(size)) = (stream.position, stream.size) else {
unknown_extent.get_or_insert(SourcePlan::Monitor {
index,
crop: CropRect {
x: 0,
y: 0,
width: window.width.max(1),
height: window.height.max(1),
},
});
continue;
};
if !rect_contains((position.0, position.1, size.0, size.1), centre) {
continue;
}
let crop = crop_to_stream(window, (position.0, position.1), (size.0, size.1))?;
return Ok(SourcePlan::Monitor { index, crop });
}
unknown_extent.ok_or_else(|| {
ComputerError::WindowNotFound(format!(
"no monitor returned by the XDG ScreenCast portal contains the window at ({}, {}) {}x{}",
window.x, window.y, window.width, window.height
))
})
}
fn window_centre(window: WindowBounds) -> (i32, i32) {
(
window.x + (window.width / 2) as i32,
window.y + (window.height / 2) as i32,
)
}
fn rect_contains(rect: (i32, i32, i32, i32), point: (i32, i32)) -> bool {
let (ox, oy, w, h) = rect;
if w <= 0 || h <= 0 {
return false;
}
let (px, py) = point;
px >= ox && px < ox + w && py >= oy && py < oy + h
}
pub(crate) fn crop_to_stream(
window: WindowBounds,
origin: (i32, i32),
size: (i32, i32),
) -> Result<CropRect, ComputerError> {
let left = window.x - origin.0;
let top = window.y - origin.1;
let right = left + window.width as i32;
let bottom = top + window.height as i32;
let left = left.clamp(0, size.0);
let top = top.clamp(0, size.1);
let right = right.clamp(0, size.0);
let bottom = bottom.clamp(0, size.1);
if right <= left || bottom <= top {
return Err(ComputerError::WindowNotFound(format!(
"window at ({}, {}) {}x{} does not overlap the {}x{} monitor at ({}, {})",
window.x, window.y, window.width, window.height, size.0, size.1, origin.0, origin.1
)));
}
Ok(CropRect {
x: left,
y: top,
width: (right - left) as u32,
height: (bottom - top) as u32,
})
}
pub(crate) fn frame_scale(logical_width: u32, pixel_width: u32) -> f64 {
if logical_width == 0 || pixel_width == 0 {
return 1.0;
}
f64::from(pixel_width) / f64::from(logical_width)
}
pub(crate) fn crop_pixels(
crop: CropRect,
scale: f64,
frame_width: u32,
frame_height: u32,
) -> Option<(u32, u32, u32, u32)> {
if frame_width == 0 || frame_height == 0 || !scale.is_finite() || scale <= 0.0 {
return None;
}
let limit_w = u64::from(frame_width);
let limit_h = u64::from(frame_height);
let x = ((f64::from(crop.x) * scale).round().max(0.0) as u64).min(limit_w);
let y = ((f64::from(crop.y) * scale).round().max(0.0) as u64).min(limit_h);
let width = ((f64::from(crop.width) * scale).round().max(1.0) as u64).min(limit_w - x);
let height = ((f64::from(crop.height) * scale).round().max(1.0) as u64).min(limit_h - y);
if width == 0 || height == 0 {
return None;
}
Some((x as u32, y as u32, width as u32, height as u32))
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum PortalFailure {
Cancelled,
Denied,
NoPortal,
}
pub(crate) fn map_failure(failure: &PortalFailure) -> ComputerError {
match failure {
PortalFailure::Cancelled => ComputerError::PermissionRequired(
"the XDG Desktop Portal dialog was dismissed; grant Screen Sharing (or Input) for \
this application in the system settings, then run the command again. The agent \
cannot click that dialog for you"
.into(),
),
PortalFailure::Denied => ComputerError::PermissionRequired(
"the XDG Desktop Portal refused the request; the ScreenCast or RemoteDesktop session \
was declined"
.into(),
),
PortalFailure::NoPortal => ComputerError::Unsupported(
"no XDG Desktop Portal implementation is running on this Wayland session, so the \
ScreenCast and RemoteDesktop portals are unavailable"
.into(),
),
}
}
#[cfg(feature = "portal")]
impl From<&ashpd::desktop::ResponseError> for PortalFailure {
fn from(error: &ashpd::desktop::ResponseError) -> Self {
match error {
ashpd::desktop::ResponseError::Cancelled => PortalFailure::Cancelled,
ashpd::desktop::ResponseError::Other => PortalFailure::Denied,
}
}
}
pub(crate) fn compiled() -> bool {
cfg!(feature = "portal")
}
pub(crate) fn wayland_session() -> bool {
std::env::var("WAYLAND_DISPLAY")
.ok()
.filter(|value| !value.trim().is_empty())
.is_some()
}
#[cfg_attr(not(feature = "portal"), allow(dead_code))]
fn is_terminal(error: &ComputerError) -> bool {
matches!(error, ComputerError::PermissionRequired(_))
}
#[cfg(feature = "portal")]
pub(crate) fn capture_window(
target: &WindowTarget,
) -> Result<crate::types::RawCapture, ComputerError> {
if target.bounds.width == 0 || target.bounds.height == 0 {
return Err(ComputerError::InvalidArguments(
"window bounds are empty".into(),
));
}
match capture_portal(target) {
Ok(raw) => Ok(raw),
Err(error) if is_terminal(&error) => Err(error),
Err(error) => fallback_crop(target, &error),
}
}
#[cfg(not(feature = "portal"))]
pub(crate) fn capture_window(
target: &WindowTarget,
) -> Result<crate::types::RawCapture, ComputerError> {
if target.bounds.width == 0 || target.bounds.height == 0 {
return Err(ComputerError::InvalidArguments(
"window bounds are empty".into(),
));
}
let mut raw = crate::backend::capture::crop_window(target.bounds)?;
raw.warnings.push(
"built without the `portal` feature: cropped the desktop image, so covered pixels may be \
wrong"
.into(),
);
Ok(raw)
}
#[cfg(feature = "portal")]
fn fallback_crop(
target: &WindowTarget,
cause: &ComputerError,
) -> Result<crate::types::RawCapture, ComputerError> {
let mut raw = crate::backend::capture::crop_window(target.bounds)?;
raw.warnings.push(format!(
"the XDG ScreenCast portal could not deliver a window frame ({cause}); cropped the \
desktop image instead, so covered pixels may be wrong"
));
Ok(raw)
}
#[cfg(feature = "portal")]
fn capture_portal(target: &WindowTarget) -> Result<crate::types::RawCapture, ComputerError> {
let bounds = target.bounds;
crate::backend::linux::runtime::block_on(pull_one_frame(bounds))?
}
#[cfg(feature = "portal")]
async fn pull_one_frame(window: WindowBounds) -> Result<crate::types::RawCapture, ComputerError> {
use ashpd::desktop::screencast::{
CursorMode, Screencast, SelectSourcesOptions, SourceType, StartCastOptions, Streams,
};
use ashpd::enumflags2::BitFlags;
let screencast = Screencast::new().await.map_err(portal_error)?;
let session = screencast
.create_session(Default::default())
.await
.map_err(portal_error)?;
screencast
.select_sources(
&session,
SelectSourcesOptions::default()
.set_cursor_mode(CursorMode::Embedded)
.set_sources(Some(
BitFlags::from(SourceType::Monitor) | BitFlags::from(SourceType::Window),
))
.set_multiple(false)
.set_persist_mode(ashpd::desktop::PersistMode::DoNot),
)
.await
.map_err(portal_error)?
.response()
.map_err(portal_error)?;
let streams: Streams = screencast
.start(&session, None, StartCastOptions::default())
.await
.map_err(portal_error)?
.response()
.map_err(portal_error)?;
let described: Vec<StreamInfo> = streams
.streams()
.iter()
.map(|stream| StreamInfo {
kind: match stream.source_type() {
Some(SourceType::Window) => SourceKind::Window,
_ => SourceKind::Monitor,
},
position: stream.position(),
size: stream.size(),
})
.collect();
let plan = plan_source(&described, window)?;
let (index, crop) = match plan {
SourcePlan::Window { index } => (index, None),
SourcePlan::Monitor { index, crop } => (index, Some(crop)),
};
let stream = streams.streams().get(index).ok_or_else(|| {
ComputerError::DisplayUnavailable(
"the ScreenCast portal returned an unusable stream index".into(),
)
})?;
let node_id = stream.pipe_wire_node_id();
let fd = screencast
.open_pipe_wire_remote(&session, Default::default())
.await
.map_err(portal_error)?;
let (frame_width, frame_height, rgba) = grab_pipewire_frame(fd, node_id)?;
let (image, logical_width, logical_height, scale, warnings) = match crop {
None => {
let scale = frame_scale(window.width, frame_width);
let geo = crate::image_util::window_placement(
window.x,
window.y,
window.width,
window.height,
dpi_from_scale(scale),
);
let warnings =
crate::image_util::size_mismatch_warning(frame_width, frame_height, &geo)
.into_iter()
.collect();
(
(frame_width, frame_height, rgba),
window.width.max(1),
window.height.max(1),
scale,
warnings,
)
}
Some(crop) => {
let stream_size = described[index].size.unwrap_or((
crop.x.saturating_add(crop.width as i32),
crop.y.saturating_add(crop.height as i32),
));
let scale = frame_scale(stream_size.0.max(1) as u32, frame_width);
let (x, y, width, height) = crop_pixels(crop, scale, frame_width, frame_height)
.ok_or_else(|| {
ComputerError::ExecutionFailed(
"the window crop falls outside the monitor frame the portal returned"
.into(),
)
})?;
let warnings = vec![format!(
"the ScreenCast portal returned a monitor, so the window rect was cropped out of \
it ({crop:?} in stream-local logical points at scale {scale})"
)];
(
crop_image(rgba, frame_width, x, y, width, height),
crop.width.max(1),
crop.height.max(1),
scale,
warnings,
)
}
};
let image = image::RgbaImage::from_raw(image.0, image.1, image.2).ok_or_else(|| {
ComputerError::ExecutionFailed("portal frame size does not match its pixel buffer".into())
})?;
Ok(crate::types::RawCapture {
frame: crate::types::Frame::from_image(image, crate::types::normalize_scale(scale)),
origin_x: window.x,
origin_y: window.y,
logical_width,
logical_height,
capture_source: SOURCE_PORTAL.into(),
occluded: false,
warnings,
})
}
#[cfg(feature = "portal")]
type CroppedImage = (u32, u32, Vec<u8>);
#[cfg(feature = "portal")]
fn crop_image(
rgba: Vec<u8>,
frame_width: u32,
x: u32,
y: u32,
width: u32,
height: u32,
) -> CroppedImage {
let row_bytes = width as usize * 4;
let start = y as usize * frame_width as usize * 4 + x as usize * 4;
let end = start + row_bytes * height as usize;
if end > rgba.len() || start > end {
return (width, height, Vec::new());
}
(width, height, rgba[start..end].to_vec())
}
#[cfg(feature = "portal")]
fn dpi_from_scale(scale: f64) -> u32 {
(scale * f64::from(96)).round().max(1.0) as u32
}
#[cfg(feature = "portal")]
fn portal_error(error: ashpd::Error) -> ComputerError {
match error {
ashpd::Error::Response(response) => map_failure(&PortalFailure::from(&response)),
ashpd::Error::PortalNotFound(_) => map_failure(&PortalFailure::NoPortal),
other => ComputerError::ExecutionFailed(format!("XDG Desktop Portal call failed: {other}")),
}
}
#[cfg(feature = "portal")]
const CAPTURE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
#[cfg(feature = "portal")]
fn grab_pipewire_frame(
fd: std::os::fd::OwnedFd,
node_id: u32,
) -> Result<(u32, u32, Vec<u8>), ComputerError> {
use pipewire::properties::properties;
use pipewire::spa;
use std::cell::RefCell;
use std::rc::Rc;
pipewire::init();
let mainloop = pipewire::main_loop::MainLoopRc::new(None)
.map_err(|error| pipewire_error("main loop", error))?;
let context = pipewire::context::ContextRc::new(&mainloop, None)
.map_err(|error| pipewire_error("context", error))?;
let core = context
.connect_fd_rc(fd, None)
.map_err(|error| pipewire_error("connect", error))?;
let stream = pipewire::stream::StreamRc::new(
core.clone(),
"computer-use-screencast",
properties! {
*pipewire::keys::MEDIA_TYPE => "Video",
*pipewire::keys::MEDIA_CATEGORY => "Capture",
*pipewire::keys::MEDIA_ROLE => "Screen",
"target.object" => node_id.to_string(),
},
)
.map_err(|error| pipewire_error("stream", error))?;
struct Shared {
format: spa::param::video::VideoInfoRaw,
frame: Option<(u32, u32, Vec<u8>)>,
}
let shared = Rc::new(RefCell::new(Shared {
format: spa::param::video::VideoInfoRaw::new(),
frame: None,
}));
let on_format = Rc::clone(&shared);
let on_process = Rc::clone(&shared);
let quit_on_frame = mainloop.clone();
let _listener = stream
.add_local_listener_with_user_data(())
.param_changed(move |_, _, id, param| {
let Some(param) = param else { return };
if id != spa::param::ParamType::Format.as_raw() {
return;
}
let Ok((media_type, media_subtype)) = spa::param::format_utils::parse_format(param)
else {
return;
};
if media_type != spa::param::format::MediaType::Video
|| media_subtype != spa::param::format::MediaSubtype::Raw
{
return;
}
let mut format = spa::param::video::VideoInfoRaw::new();
if format.parse(param).is_err() {
return;
}
on_format.borrow_mut().format = format;
})
.process(move |stream, _| {
if on_process.borrow().frame.is_some() {
return;
}
let mut buffer = match stream.dequeue_buffer() {
Some(buffer) => buffer,
None => return,
};
let format = on_process.borrow().format;
let size = format.size();
let width = size.width;
let height = size.height;
if width == 0 || height == 0 {
return;
}
let Some(image) = read_frame(&mut buffer, width, height, format.format()) else {
return;
};
on_process.borrow_mut().frame = Some(image);
quit_on_frame.quit();
})
.register()
.map_err(|error| pipewire_error("listener", error))?;
let mut params: [&pipewire::spa::pod::Pod; 0] = [];
stream
.connect(
spa::utils::Direction::Input,
None,
pipewire::stream::StreamFlags::AUTOCONNECT
| pipewire::stream::StreamFlags::MAP_BUFFERS
| pipewire::stream::StreamFlags::RT_PROCESS,
&mut params,
)
.map_err(|error| pipewire_error("stream connect", error))?;
let timer_weak = mainloop.downgrade();
let timer = mainloop.loop_().add_timer(move |_| {
if let Some(loop_) = timer_weak.upgrade() {
loop_.quit();
}
});
let _ = timer.update_timer(Some(CAPTURE_TIMEOUT), None);
mainloop.run();
let frame = shared.borrow_mut().frame.take();
frame.ok_or_else(|| {
ComputerError::ExecutionFailed(
"the ScreenCast portal produced no video frame within 5s".into(),
)
})
}
#[cfg(feature = "portal")]
fn read_frame(
buffer: &mut pipewire::buffer::Buffer<'_>,
width: u32,
height: u32,
format: pipewire::spa::param::video::VideoFormat,
) -> Option<(u32, u32, Vec<u8>)> {
use pipewire::spa::param::video::VideoFormat;
let bytes_per_pixel = match format {
VideoFormat::BGRx | VideoFormat::BGRA | VideoFormat::xBGR | VideoFormat::ABGR => 4,
VideoFormat::RGBx | VideoFormat::RGBA | VideoFormat::xRGB | VideoFormat::ARGB => 4,
VideoFormat::RGB | VideoFormat::BGR => 3,
_ => return None,
};
let swizzle = matches!(
format,
VideoFormat::BGRx
| VideoFormat::BGRA
| VideoFormat::xBGR
| VideoFormat::ABGR
| VideoFormat::BGR
);
let datas = buffer.datas_mut();
let data = datas.first_mut()?;
let (size, stride) = {
let chunk = data.chunk();
(
chunk.size() as usize,
chunk.stride().unsigned_abs() as usize,
)
};
let bytes = data.data()?;
if size == 0 || bytes.len() < size || stride == 0 {
return None;
}
let row_bytes = width as usize * bytes_per_pixel;
let rows = (size / stride).min(height as usize);
let mut out = Vec::with_capacity(width as usize * height as usize * 4);
for row in 0..rows {
let start = row * stride;
let end = start + row_bytes;
if end > bytes.len() {
return None;
}
let line = &bytes[start..end];
if swizzle {
out.reserve(line.len() / bytes_per_pixel * 4);
for pixel in line.chunks_exact(bytes_per_pixel) {
if bytes_per_pixel == 4 {
out.extend_from_slice(&[pixel[2], pixel[1], pixel[0], 0xff]);
} else {
out.extend_from_slice(&[pixel[2], pixel[1], pixel[0]]);
}
}
} else {
out.extend_from_slice(line);
}
}
while out.len() < width as usize * height as usize * 4 {
out.push(0);
}
Some((width, height, out))
}
#[cfg(feature = "portal")]
fn pipewire_error(what: &str, error: pipewire::Error) -> ComputerError {
ComputerError::ExecutionFailed(format!("pipewire {what}: {error}"))
}
#[cfg(feature = "portal")]
pub(crate) fn perform_event_tier(
job: &crate::tier::InputJob,
) -> Result<crate::tier::InputReport, ComputerError> {
if !wayland_session() {
return Err(ComputerError::Unsupported(
"the XDG RemoteDesktop portal only applies to a Wayland session; this one is X11, where \
the XTest tier is used"
.into(),
));
}
if job.require_background {
return Err(ComputerError::ExecutionFailed(
"--require-background refuses the RemoteDesktop tier because the portal moves the \
real pointer"
.into(),
));
}
crate::backend::linux::runtime::block_on(run_remote_desktop(job.kind.clone()))?
}
#[cfg(feature = "portal")]
async fn run_remote_desktop(
kind: crate::tier::InputKind,
) -> Result<crate::tier::InputReport, ComputerError> {
use crate::tier::InputKind;
use ashpd::desktop::{
remote_desktop::{
Axis, DeviceType, KeyState, NotifyKeyboardKeysymOptions,
NotifyPointerAxisDiscreteOptions, NotifyPointerButtonOptions, RemoteDesktop,
SelectDevicesOptions,
},
screencast::{CursorMode, Screencast, SelectSourcesOptions, SourceType},
};
use ashpd::enumflags2::BitFlags;
let remote = RemoteDesktop::new().await.map_err(portal_error)?;
let session = remote
.create_session(Default::default())
.await
.map_err(portal_error)?;
remote
.select_devices(
&session,
SelectDevicesOptions::default().set_devices(DeviceType::Pointer | DeviceType::Keyboard),
)
.await
.map_err(portal_error)?
.response()
.map_err(portal_error)?;
let screencast = Screencast::new().await.map_err(portal_error)?;
screencast
.select_sources(
&session,
SelectSourcesOptions::default()
.set_cursor_mode(CursorMode::Hidden)
.set_sources(Some(BitFlags::from(SourceType::Monitor))),
)
.await
.map_err(portal_error)?
.response()
.map_err(portal_error)?;
let response = remote
.start(&session, None, Default::default())
.await
.map_err(portal_error)?
.response()
.map_err(portal_error)?;
if !response.devices().contains(DeviceType::Pointer) {
return Err(map_failure(&PortalFailure::Denied));
}
let node = response
.streams()
.first()
.map(|stream| stream.pipe_wire_node_id())
.ok_or_else(|| {
ComputerError::ExecutionFailed(
"the RemoteDesktop portal granted input but returned no stream, so there is no \
coordinate space for pointer motion"
.into(),
)
})?;
let mut warnings = Vec::new();
if !try_eis(&remote, &session).await {
warnings.push(EIS_UNAVAILABLE_NOTE.to_string());
}
match kind {
InputKind::Move { x, y } => {
notify_absolute(&remote, &session, node, x, y).await?;
}
InputKind::Click {
x,
y,
button,
count,
} => {
notify_absolute(&remote, &session, node, x, y).await?;
let code = evdev_button(button);
for index in 0..count {
if index > 0 {
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
remote
.notify_pointer_button(
&session,
code,
KeyState::Pressed,
NotifyPointerButtonOptions::default(),
)
.await
.map_err(portal_error)?;
remote
.notify_pointer_button(
&session,
code,
KeyState::Released,
NotifyPointerButtonOptions::default(),
)
.await
.map_err(portal_error)?;
}
}
InputKind::Scroll { x, y, dx, dy } => {
if let (Some(x), Some(y)) = (x, y) {
notify_absolute(&remote, &session, node, x, y).await?;
}
if dy != 0 {
remote
.notify_pointer_axis_discrete(
&session,
Axis::Vertical,
dy,
NotifyPointerAxisDiscreteOptions::default(),
)
.await
.map_err(portal_error)?;
}
if dx != 0 {
remote
.notify_pointer_axis_discrete(
&session,
Axis::Horizontal,
dx,
NotifyPointerAxisDiscreteOptions::default(),
)
.await
.map_err(portal_error)?;
}
}
InputKind::Key { strokes } => {
if !response.devices().contains(DeviceType::Keyboard) {
return Err(map_failure(&PortalFailure::Denied));
}
for stroke in strokes {
let keysym = keysym_for(&stroke.key).ok_or_else(|| {
ComputerError::Unsupported(format!(
"no X11 keysym is known for `{}`; the RemoteDesktop portal speaks keysyms, \
not XTest keycodes",
stroke.key
))
})?;
for modifier in &stroke.modifiers {
let sym = keysym_for(modifier.as_str()).ok_or_else(|| {
ComputerError::Unsupported(format!("no keysym for modifier {modifier:?}"))
})?;
remote
.notify_keyboard_keysym(
&session,
sym,
KeyState::Pressed,
NotifyKeyboardKeysymOptions::default(),
)
.await
.map_err(portal_error)?;
}
remote
.notify_keyboard_keysym(
&session,
keysym,
KeyState::Pressed,
NotifyKeyboardKeysymOptions::default(),
)
.await
.map_err(portal_error)?;
remote
.notify_keyboard_keysym(
&session,
keysym,
KeyState::Released,
NotifyKeyboardKeysymOptions::default(),
)
.await
.map_err(portal_error)?;
for modifier in stroke.modifiers.iter().rev() {
let sym = keysym_for(modifier.as_str()).ok_or_else(|| {
ComputerError::Unsupported(format!("no keysym for modifier {modifier:?}"))
})?;
remote
.notify_keyboard_keysym(
&session,
sym,
KeyState::Released,
NotifyKeyboardKeysymOptions::default(),
)
.await
.map_err(portal_error)?;
}
}
}
InputKind::Drag { .. }
| InputKind::MouseDown { .. }
| InputKind::MouseUp { .. }
| InputKind::HoldKey { .. }
| InputKind::Type { .. } => {
return Err(ComputerError::Unsupported(
"the RemoteDesktop portal tier covers pointer motion, buttons, scroll and key \
combinations; drag, button hold, hold-key and free text typing are not sent \
through it"
.into(),
))
}
}
Ok(crate::tier::InputReport {
tier: TIER_NAME,
cursor_moved: true,
focus_restored: None,
warnings,
scroll: None,
})
}
#[cfg(feature = "portal")]
async fn notify_absolute(
remote: &RemoteDesktopProxy,
session: &ashpd::desktop::Session<RemoteDesktopProxy>,
node: u32,
x: i32,
y: i32,
) -> Result<(), ComputerError> {
use ashpd::desktop::remote_desktop::NotifyPointerMotionAbsoluteOptions;
remote
.notify_pointer_motion_absolute(
session,
node,
f64::from(x),
f64::from(y),
NotifyPointerMotionAbsoluteOptions::default(),
)
.await
.map_err(portal_error)
}
#[cfg(feature = "portal")]
pub(crate) fn evdev_button(button: crate::types::MouseButton) -> i32 {
match button {
crate::types::MouseButton::Left => 0x110,
crate::types::MouseButton::Right => 0x111,
crate::types::MouseButton::Middle => 0x112,
crate::types::MouseButton::Back => 0x115,
crate::types::MouseButton::Forward => 0x116,
}
}
#[cfg(feature = "portal")]
pub(crate) fn keysym_for(name: &str) -> Option<i32> {
let lower = name.trim().to_ascii_lowercase();
if let Some(rest) = lower.strip_prefix("xk_") {
let rest = rest.strip_prefix("0x").unwrap_or(rest);
return i32::from_str_radix(rest, 16).ok();
}
let sym = match lower.as_str() {
"return" | "enter" => 0xff0d,
"escape" | "esc" => 0xff1b,
"tab" => 0xff09,
"space" => 0x0020,
"backspace" => 0xff08,
"delete" | "del" => 0xffff,
"home" => 0xff50,
"end" => 0xff57,
"pageup" | "page_up" | "pgup" => 0xff55,
"pagedown" | "page_down" | "pgdn" => 0xff56,
"left" | "arrowleft" => 0xff51,
"right" | "arrowright" => 0xff53,
"up" | "arrowup" => 0xff52,
"down" | "arrowdown" => 0xff54,
"insert" => 0xff63,
"plus" => 0x02b,
"minus" => 0x02d,
"shift" => 0xffe1,
"lock" => 0xffe5,
"control" | "ctrl" => 0xffe3,
"alt" => 0xffe9,
"super" | "meta" | "command" => 0xffeb,
"capslock" => 0xffe2,
"printscreen" => 0xff61,
"menu" => 0xff67,
"f1" => 0xffbe,
"f2" => 0xffbf,
"f3" => 0xffc0,
"f4" => 0xffc1,
"f5" => 0xffc2,
"f6" => 0xffc3,
"f7" => 0xffc4,
"f8" => 0xffc5,
"f9" => 0xffc6,
"f10" => 0xffc7,
"f11" => 0xffc8,
"f12" => 0xffc9,
other => {
let mut chars = other.chars();
return match (chars.next(), chars.next()) {
(Some(one), None) => Some(one as i32),
_ => None,
};
}
};
Some(sym)
}
#[cfg(feature = "portal")]
use ashpd::desktop::remote_desktop::RemoteDesktop as RemoteDesktopProxy;
#[cfg(feature = "portal")]
async fn try_eis(
remote: &RemoteDesktopProxy,
session: &ashpd::desktop::Session<RemoteDesktopProxy>,
) -> bool {
use ashpd::desktop::remote_desktop::ConnectToEISOptions;
use reis::ei::{handshake::ContextType, Context as EiContext};
let Ok(fd) = remote
.connect_to_eis(session, ConnectToEISOptions::default())
.await
else {
return false;
};
let socket = std::os::unix::net::UnixStream::from(fd);
let Ok(context) = EiContext::new(socket) else {
return false;
};
reis::handshake::ei_handshake_blocking(&context, "computer-use", ContextType::Sender).is_ok()
}
#[cfg(feature = "portal")]
pub(crate) const EIS_UNAVAILABLE_NOTE: &str =
"the compositor did not offer an EIS socket, so the portal's absolute pointer motion is used; \
libei relative motion is unavailable";
#[cfg(test)]
mod tests {
use super::*;
fn bounds(x: i32, y: i32, width: u32, height: u32) -> WindowBounds {
WindowBounds {
x,
y,
width,
height,
}
}
fn monitor(position: (i32, i32), size: (i32, i32)) -> StreamInfo {
StreamInfo {
kind: SourceKind::Monitor,
position: Some(position),
size: Some(size),
}
}
#[test]
fn a_window_stream_beats_a_containing_monitor() {
let streams = [
monitor((0, 0), (1920, 1080)),
StreamInfo {
kind: SourceKind::Window,
position: None,
size: Some((800, 600)),
},
];
assert_eq!(
plan_source(&streams, bounds(100, 100, 400, 300)).unwrap(),
SourcePlan::Window { index: 1 }
);
}
#[test]
fn a_monitor_is_cropped_when_no_window_stream_exists() {
let streams = [monitor((1920, 0), (1920, 1080))];
match plan_source(&streams, bounds(1800, 100, 400, 300)).unwrap() {
SourcePlan::Monitor { index, crop } => {
assert_eq!(index, 0);
assert_eq!(
crop,
CropRect {
x: 0,
y: 100,
width: 280,
height: 300
}
);
}
other => panic!("expected a cropped monitor, got {other:?}"),
}
}
#[test]
fn a_monitor_with_unknown_extent_is_only_a_last_resort() {
let streams = [
monitor((5000, 5000), (100, 100)),
StreamInfo {
kind: SourceKind::Monitor,
position: None,
size: None,
},
];
match plan_source(&streams, bounds(10, 10, 200, 100)).unwrap() {
SourcePlan::Monitor { index, crop } => {
assert_eq!(index, 1);
assert_eq!(
crop,
CropRect {
x: 0,
y: 0,
width: 200,
height: 100
}
);
}
other => panic!("expected the unknown-extent monitor, got {other:?}"),
}
}
#[test]
fn no_streams_is_display_unavailable_not_window_not_found() {
assert_eq!(
plan_source(&[], bounds(0, 0, 10, 10)).unwrap_err().code(),
"display_unavailable"
);
}
#[test]
fn a_window_off_every_monitor_is_window_not_found() {
let streams = [monitor((0, 0), (800, 600))];
assert_eq!(
plan_source(&streams, bounds(5000, 5000, 100, 100))
.unwrap_err()
.code(),
"window_not_found"
);
}
#[test]
fn a_window_stream_is_accepted_without_a_position() {
let streams = [StreamInfo {
kind: SourceKind::Window,
position: None,
size: None,
}];
assert_eq!(
plan_source(&streams, bounds(0, 0, 1, 1)).unwrap(),
SourcePlan::Window { index: 0 }
);
}
#[test]
fn rect_containment_excludes_the_far_edge() {
assert!(rect_contains((0, 0, 100, 100), (0, 0)));
assert!(rect_contains((0, 0, 100, 100), (99, 99)));
assert!(!rect_contains((0, 0, 100, 100), (100, 50)));
assert!(!rect_contains((0, 0, 0, 100), (0, 50)));
assert!(!rect_contains((0, 0, 100, 100), (-1, 50)));
}
#[test]
fn crop_translates_into_stream_local_points() {
let crop = crop_to_stream(bounds(100, 50, 200, 100), (0, 0), (1920, 1080)).unwrap();
assert_eq!(
crop,
CropRect {
x: 100,
y: 50,
width: 200,
height: 100
}
);
}
#[test]
fn crop_accounts_for_a_monitor_origin_on_a_second_display() {
let crop = crop_to_stream(bounds(2000, 50, 200, 100), (1920, 0), (1920, 1080)).unwrap();
assert_eq!(
crop,
CropRect {
x: 80,
y: 50,
width: 200,
height: 100
}
);
}
#[test]
fn crop_clamps_a_window_hanging_off_the_top_left() {
let crop = crop_to_stream(bounds(-50, -20, 100, 100), (0, 0), (1920, 1080)).unwrap();
assert_eq!(
crop,
CropRect {
x: 0,
y: 0,
width: 50,
height: 80
}
);
}
#[test]
fn crop_clamps_a_window_overrunning_the_bottom_right() {
let crop = crop_to_stream(bounds(1900, 1000, 400, 400), (0, 0), (1920, 1080)).unwrap();
assert_eq!(
crop,
CropRect {
x: 1900,
y: 1000,
width: 20,
height: 80
}
);
}
#[test]
fn crop_of_a_window_entirely_off_the_monitor_is_refused() {
assert_eq!(
crop_to_stream(bounds(3000, 0, 100, 100), (0, 0), (1920, 1080))
.unwrap_err()
.code(),
"window_not_found"
);
}
#[test]
fn frame_scale_is_the_device_pixel_ratio() {
assert_eq!(frame_scale(1920, 3840), 2.0);
assert_eq!(frame_scale(1920, 1920), 1.0);
assert_eq!(frame_scale(0, 3840), 1.0);
assert_eq!(frame_scale(1920, 0), 1.0);
}
#[test]
fn crop_pixels_scales_into_device_pixels() {
let crop = CropRect {
x: 80,
y: 50,
width: 200,
height: 100,
};
assert_eq!(
crop_pixels(crop, 2.0, 3840, 2160),
Some((160, 100, 400, 200))
);
assert_eq!(crop_pixels(crop, 1.0, 1920, 1080), Some((80, 50, 200, 100)));
}
#[test]
fn crop_pixels_clamps_to_the_buffer_instead_of_overrunning() {
let crop = CropRect {
x: 80,
y: 50,
width: 200,
height: 100,
};
assert_eq!(crop_pixels(crop, 2.0, 300, 200), Some((160, 100, 140, 100)));
assert_eq!(crop_pixels(crop, 2.0, 0, 200), None);
assert_eq!(crop_pixels(crop, 0.0, 300, 200), None);
assert_eq!(crop_pixels(crop, f64::NAN, 300, 200), None);
assert_eq!(crop_pixels(crop, -1.0, 300, 200), None);
assert_eq!(
crop_pixels(
CropRect {
x: 5000,
y: 0,
width: 10,
height: 10
},
1.0,
800,
600
),
None
);
}
#[test]
fn a_cancelled_dialog_is_permission_required() {
let error = map_failure(&PortalFailure::Cancelled);
assert_eq!(error.code(), "permission_required");
assert!(error.to_string().contains("cannot click"));
}
#[test]
fn a_denied_dialog_is_permission_required() {
assert_eq!(
map_failure(&PortalFailure::Denied).code(),
"permission_required"
);
}
#[test]
fn a_missing_portal_is_unsupported() {
assert_eq!(map_failure(&PortalFailure::NoPortal).code(), "unsupported");
}
#[test]
fn a_cancelled_dialog_is_never_downgraded_to_a_fallback() {
assert!(is_terminal(&map_failure(&PortalFailure::Cancelled)));
assert!(is_terminal(&map_failure(&PortalFailure::Denied)));
assert!(!is_terminal(&map_failure(&PortalFailure::NoPortal)));
assert!(!is_terminal(&ComputerError::ExecutionFailed(
"no frame".into()
)));
}
#[test]
fn compiled_matches_the_feature_flag() {
assert_eq!(compiled(), cfg!(feature = "portal"));
}
#[test]
#[cfg(feature = "portal")]
fn the_eis_note_explains_the_fallback() {
assert!(EIS_UNAVAILABLE_NOTE.contains("EIS"));
assert!(EIS_UNAVAILABLE_NOTE.contains("relative"));
}
#[cfg(feature = "portal")]
#[test]
fn evdev_button_codes_are_not_x11_button_numbers() {
use crate::types::MouseButton;
assert_eq!(evdev_button(MouseButton::Left), 0x110);
assert_eq!(evdev_button(MouseButton::Right), 0x111);
assert_eq!(evdev_button(MouseButton::Middle), 0x112);
assert_eq!(evdev_button(MouseButton::Back), 0x115);
assert_eq!(evdev_button(MouseButton::Forward), 0x116);
}
#[cfg(feature = "portal")]
#[test]
fn keysyms_cover_the_names_the_agent_emits() {
let table = [
("return", 0xff0d),
("enter", 0xff0d),
("escape", 0xff1b),
("tab", 0xff09),
("space", 0x0020),
("backspace", 0xff08),
("delete", 0xffff),
("home", 0xff50),
("end", 0xff57),
("pageup", 0xff55),
("page_down", 0xff56),
("left", 0xff51),
("right", 0xff53),
("up", 0xff52),
("down", 0xff54),
("insert", 0xff63),
("f1", 0xffbe),
("f12", 0xffc9),
("shift", 0xffe1),
("control", 0xffe3),
("alt", 0xffe9),
("super", 0xffeb),
];
for (name, sym) in table {
assert_eq!(keysym_for(name), Some(sym), "keysym for {name:?}");
}
assert_eq!(keysym_for("command"), Some(0xffeb));
assert_eq!(keysym_for("meta"), Some(0xffeb));
assert_eq!(keysym_for("a"), Some('a' as i32));
assert_eq!(keysym_for("A"), Some('a' as i32));
assert_eq!(keysym_for("Z"), Some('z' as i32));
assert_eq!(keysym_for("7"), Some('7' as i32));
assert_eq!(keysym_for("xk_0x1234"), Some(0x1234));
assert_eq!(keysym_for("xk_1b"), Some(0x1b));
assert_eq!(keysym_for("xk_zz"), None);
assert_eq!(keysym_for(""), None);
assert_eq!(keysym_for("nosuchkey"), None);
}
#[cfg(feature = "portal")]
#[test]
fn cropping_a_buffer_takes_exactly_the_requested_pixels() {
let rgba: Vec<u8> = vec![
1, 1, 1, 255, 2, 2, 2, 255, 3, 3, 3, 255, 4, 4, 4, 255,
];
let (width, height, out) = crop_image(rgba.clone(), 2, 1, 1, 1, 1);
assert_eq!((width, height), (1, 1));
assert_eq!(out, vec![4, 4, 4, 255]);
let (_, _, full) = crop_image(rgba, 2, 0, 0, 2, 2);
assert_eq!(full.len(), 16);
}
#[cfg(feature = "portal")]
#[test]
fn cropping_out_of_range_yields_an_empty_buffer_not_a_panic() {
let rgba = vec![0u8; 16];
let (_, _, out) = crop_image(rgba, 2, 0, 0, 8, 8);
assert!(out.is_empty());
}
}