use anyhow::Result;
use pixelactions_core::flow::Axis;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Button {
Left,
}
pub trait Injector {
fn move_to(&mut self, x: f64, y: f64) -> Result<()>;
fn click(&mut self, button: Button) -> Result<()>;
fn double_click(&mut self, button: Button) -> Result<()>;
fn press(&mut self, button: Button) -> Result<()>;
fn release(&mut self, button: Button) -> Result<()>;
fn text(&mut self, text: &str) -> Result<()>;
fn chord(&mut self, chord: &str) -> Result<()>;
fn scroll(&mut self, amount: i32, axis: Axis) -> Result<()>;
fn cursor(&mut self) -> Result<(f64, f64)>;
fn probe(&mut self) -> Result<()>;
}
#[cfg(test)]
#[derive(Debug, Clone, PartialEq)]
pub struct Recording {
pub events: Vec<String>,
pub cursor: (f64, f64),
}
#[cfg(test)]
impl Default for Recording {
fn default() -> Self {
Self {
events: Vec::new(),
cursor: (400.0, 300.0),
}
}
}
#[cfg(test)]
impl Injector for Recording {
fn move_to(&mut self, x: f64, y: f64) -> Result<()> {
self.events.push(format!("move {x:.0},{y:.0}"));
Ok(())
}
fn click(&mut self, _button: Button) -> Result<()> {
self.events.push("click".into());
Ok(())
}
fn double_click(&mut self, _button: Button) -> Result<()> {
self.events.push("double_click".into());
Ok(())
}
fn press(&mut self, _button: Button) -> Result<()> {
self.events.push("press".into());
Ok(())
}
fn release(&mut self, _button: Button) -> Result<()> {
self.events.push("release".into());
Ok(())
}
fn text(&mut self, text: &str) -> Result<()> {
self.events.push(format!("text {text}"));
Ok(())
}
fn chord(&mut self, chord: &str) -> Result<()> {
self.events.push(format!("chord {chord}"));
Ok(())
}
fn scroll(&mut self, amount: i32, axis: Axis) -> Result<()> {
let way = match axis {
Axis::Vertical => "v",
Axis::Horizontal => "h",
};
self.events.push(format!("scroll {way}{amount}"));
Ok(())
}
fn cursor(&mut self) -> Result<(f64, f64)> {
Ok(self.cursor)
}
fn probe(&mut self) -> Result<()> {
self.events.push("probe".into());
Ok(())
}
}
#[cfg(target_os = "macos")]
pub use platform::RealInjector;
#[cfg(target_os = "linux")]
pub use wayland::WaylandInjector;
#[cfg(target_os = "linux")]
pub use x11::X11Injector;
#[cfg(target_os = "linux")]
pub fn session_server() -> pixelactions_core::display::Server {
pixelactions_core::display::detect(
std::env::var("XDG_SESSION_TYPE").ok().as_deref(),
std::env::var("WAYLAND_DISPLAY").ok().as_deref(),
std::env::var("DISPLAY").ok().as_deref(),
)
}
#[cfg(target_os = "linux")]
pub fn availability() -> Result<(), String> {
use pixelactions_core::display::Server;
match session_server() {
Server::Wayland => {}
Server::X11 => {
return X11Injector::new()
.map(|_| ())
.map_err(|error| format!("{error:#}"));
}
Server::Unknown => {
return Err(
"no desktop session was found — neither XDG_SESSION_TYPE, WAYLAND_DISPLAY \
nor DISPLAY names one. Synthesizing input needs a display server to send \
it to; `plan` works without one"
.to_string(),
);
}
}
let capabilities = crate::portal::capabilities()
.map_err(|error| format!("cannot ask the desktop portal what it supports: {error:#}"))?;
if !capabilities.usable() {
return Err(format!(
"this compositor cannot grant input: the portal offers RemoteDesktop version {} \
(needs 2 or newer for ConnectToEIS), device types {:#b} (needs keyboard and \
pointer), ScreenCast version {}. GNOME and KDE implement this; wlroots \
compositors do not yet",
capabilities.remote_desktop_version,
capabilities.device_types,
capabilities.screen_cast_version
));
}
Ok(())
}
#[cfg(not(target_os = "linux"))]
pub fn availability() -> Result<(), String> {
if cfg!(target_os = "macos") {
return Ok(());
}
Err("input synthesis is not implemented for this platform yet — `plan` works everywhere".into())
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
mod keys {
use anyhow::{Context, Result, anyhow};
use enigo::Key;
use pixelactions_core::chord::NAMED_KEYS;
pub fn key_for(token: &str) -> Result<Key> {
let key = match token.to_ascii_lowercase().as_str() {
"cmd" | "command" | "meta" | "super" => Key::Meta,
"ctrl" | "control" => Key::Control,
"alt" | "option" | "opt" => Key::Alt,
"shift" => Key::Shift,
"tab" => Key::Tab,
"enter" | "return" => Key::Return,
"esc" | "escape" => Key::Escape,
"space" => Key::Space,
"backspace" | "delete" => Key::Backspace,
"up" => Key::UpArrow,
"down" => Key::DownArrow,
"left" => Key::LeftArrow,
"right" => Key::RightArrow,
other => {
let mut chars = other.chars();
let first = chars.next().context("empty key in chord")?;
if chars.next().is_some() {
return Err(anyhow!(
"unknown key {other:?} in chord — use a single character or one of: {}",
NAMED_KEYS.join(", ")
));
}
Key::Unicode(first)
}
};
Ok(key)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn modifier_names_map_the_way_a_human_writes_them() {
for name in ["cmd", "command", "meta", "super", "SUPER"] {
assert_eq!(key_for(name).expect(name), Key::Meta, "{name}");
}
for name in ["ctrl", "control"] {
assert_eq!(key_for(name).expect(name), Key::Control, "{name}");
}
for name in ["alt", "option", "opt"] {
assert_eq!(key_for(name).expect(name), Key::Alt, "{name}");
}
}
#[test]
fn every_promised_name_resolves() {
for name in NAMED_KEYS {
let key = key_for(name).unwrap_or_else(|error| panic!("{name}: {error}"));
assert_ne!(
key,
Key::Unicode(name.chars().next().expect("non-empty")),
"{name} fell through to its first character instead of naming a key"
);
}
}
#[test]
fn a_single_character_becomes_itself() {
assert_eq!(key_for("s").expect("s"), Key::Unicode('s'));
assert_eq!(key_for("7").expect("7"), Key::Unicode('7'));
assert_eq!(key_for("ü").expect("ü"), Key::Unicode('ü'));
}
#[test]
fn an_unknown_multi_character_key_is_refused_and_says_what_is_allowed() {
let error = key_for("fnord").expect_err("not a key");
let message = error.to_string();
assert!(message.contains("fnord"), "{message}");
assert!(message.contains("shift"), "lists the names: {message}");
assert!(key_for("").is_err());
}
}
}
#[cfg(target_os = "linux")]
mod wayland {
use anyhow::{Result, anyhow, bail};
use pixelactions_core::flow::Axis;
use pixelactions_core::stream::place;
use pixelcoords_core::session::MonitorRecord;
use super::{Button, Injector};
use crate::{eis, portal};
pub struct WaylandInjector {
sender: eis::Sender,
monitors: Vec<MonitorRecord>,
_grant: portal::Grant,
}
impl WaylandInjector {
pub fn new(monitors: &[MonitorRecord]) -> Result<Self> {
let mut grant = portal::grant()?;
let sender = eis::Sender::connect(grant.take_socket()?)?;
Ok(Self {
sender,
monitors: monitors.to_vec(),
_grant: grant,
})
}
pub fn can_type(&self) -> bool {
self.sender.can_type()
}
pub fn regions(&self) -> &[pixelactions_core::stream::Region] {
self.sender.regions()
}
}
impl Injector for WaylandInjector {
fn move_to(&mut self, x: f64, y: f64) -> Result<()> {
let placement = place(
&self.monitors,
self.sender.regions(),
x.round() as i32,
y.round() as i32,
)
.map_err(|error| anyhow!("cannot place the pointer: {error}"))?;
self.sender.move_to(placement)
}
fn click(&mut self, button: Button) -> Result<()> {
self.press(button)?;
self.release(button)
}
fn double_click(&mut self, button: Button) -> Result<()> {
self.click(button)?;
std::thread::sleep(std::time::Duration::from_millis(40));
self.click(button)
}
fn press(&mut self, button: Button) -> Result<()> {
match button {
Button::Left => self.sender.button(true),
}
}
fn release(&mut self, button: Button) -> Result<()> {
match button {
Button::Left => self.sender.button(false),
}
}
fn text(&mut self, text: &str) -> Result<()> {
self.sender.text(text)
}
fn chord(&mut self, chord: &str) -> Result<()> {
self.sender.chord(chord)
}
fn scroll(&mut self, amount: i32, axis: Axis) -> Result<()> {
self.sender.scroll(amount, axis)
}
fn cursor(&mut self) -> Result<(f64, f64)> {
bail!(
"Wayland exposes no way to ask where the pointer is — the same isolation \
that makes input injection require your consent also hides the pointer \
from other programs. The corner kill switch therefore has nothing to \
watch on this platform. Set failsafe = false in the flow to run without \
it, deliberately"
)
}
fn probe(&mut self) -> Result<()> {
if self.sender.regions().is_empty() {
bail!("the compositor granted input but described no region to aim in");
}
Ok(())
}
}
}
#[cfg(target_os = "linux")]
mod x11 {
use anyhow::{Result, anyhow, bail};
use enigo::{
Axis as EnigoAxis, Button as EnigoButton, Coordinate, Direction, Enigo, Keyboard, Mouse,
Settings,
};
use pixelactions_core::flow::Axis;
use super::{Button, Injector, keys::key_for};
const PROBE_SETTLE: std::time::Duration = std::time::Duration::from_millis(40);
pub struct X11Injector {
enigo: Enigo,
}
impl X11Injector {
pub fn new() -> Result<Self> {
let display = std::env::var("DISPLAY").unwrap_or_default();
let enigo = Enigo::new(&Settings::default()).map_err(|error| {
anyhow!(
"cannot connect to the X server at DISPLAY={display:?}: {error}. \
Check that DISPLAY names the session you meant, that the server is \
running, and that this user is allowed to connect to it (`xhost` \
restrictions and a different user's session are the usual causes)"
)
})?;
Ok(Self { enigo })
}
fn location(&mut self) -> Result<(i32, i32)> {
self.enigo
.location()
.map_err(|error| anyhow!("cannot read the cursor position: {error}"))
}
fn nudge(&mut self, from: (i32, i32), step: i32) -> Result<bool> {
self.enigo
.move_mouse(from.0 + step, from.1, Coordinate::Abs)
.map_err(|error| anyhow!("cannot move the cursor: {error}"))?;
std::thread::sleep(PROBE_SETTLE);
let after = self.location()?;
let _ = self.enigo.move_mouse(from.0, from.1, Coordinate::Abs);
Ok(after != from)
}
}
fn to_enigo(button: Button) -> EnigoButton {
match button {
Button::Left => EnigoButton::Left,
}
}
fn root_point(x: f64, y: f64) -> Result<(i32, i32)> {
let (px, py) = (x.round() as i32, y.round() as i32);
if px < 0 || py < 0 {
bail!(
"({x:.0}, {y:.0}) is not a point on this X screen: XTEST addresses the root \
window, whose coordinates start at (0, 0) and span every output, so a \
negative one cannot be expressed. Re-mark the region with pixelcoords on \
this session"
);
}
Ok((px, py))
}
impl Injector for X11Injector {
fn move_to(&mut self, x: f64, y: f64) -> Result<()> {
let (px, py) = root_point(x, y)?;
self.enigo
.move_mouse(px, py, Coordinate::Abs)
.map_err(|error| anyhow!("move to ({px}, {py}) failed: {error}"))
}
fn click(&mut self, button: Button) -> Result<()> {
self.enigo
.button(to_enigo(button), Direction::Click)
.map_err(|error| anyhow!("click failed: {error}"))
}
fn double_click(&mut self, button: Button) -> Result<()> {
self.click(button)?;
std::thread::sleep(std::time::Duration::from_millis(40));
self.click(button)
}
fn press(&mut self, button: Button) -> Result<()> {
self.enigo
.button(to_enigo(button), Direction::Press)
.map_err(|error| anyhow!("press failed: {error}"))
}
fn release(&mut self, button: Button) -> Result<()> {
self.enigo
.button(to_enigo(button), Direction::Release)
.map_err(|error| anyhow!("release failed: {error}"))
}
fn text(&mut self, text: &str) -> Result<()> {
self.enigo
.text(text)
.map_err(|error| anyhow!("typing failed: {error}"))
}
fn chord(&mut self, chord: &str) -> Result<()> {
let (modifiers, key) = pixelactions_core::chord::split(chord)?;
let mut held = Vec::new();
for token in &modifiers {
let modifier = key_for(token)?;
self.enigo
.key(modifier, Direction::Press)
.map_err(|error| anyhow!("holding {token} failed: {error}"))?;
held.push(modifier);
}
let result = self
.enigo
.key(key_for(key)?, Direction::Click)
.map_err(|error| anyhow!("pressing {key} failed: {error}"));
for modifier in held.into_iter().rev() {
let _ = self.enigo.key(modifier, Direction::Release);
}
result
}
fn scroll(&mut self, amount: i32, axis: Axis) -> Result<()> {
let axis = match axis {
Axis::Vertical => EnigoAxis::Vertical,
Axis::Horizontal => EnigoAxis::Horizontal,
};
self.enigo
.scroll(amount, axis)
.map_err(|error| anyhow!("cannot scroll: {error}"))
}
fn cursor(&mut self) -> Result<(f64, f64)> {
let (x, y) = self.location()?;
Ok((f64::from(x), f64::from(y)))
}
fn probe(&mut self) -> Result<()> {
let from = self.location()?;
for step in [1, -1] {
if self.nudge(from, step)? {
return Ok(());
}
}
Err(anyhow!(
"the cursor did not move: the X server processed the XTEST event and the \
pointer stayed at ({}, {}). Something holds a pointer grab, or this server \
was built without the XTEST extension",
from.0,
from.1
))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_fractional_point_rounds_to_the_nearest_pixel() {
assert_eq!(root_point(10.4, 20.6).expect("on screen"), (10, 21));
assert_eq!(root_point(0.0, 0.0).expect("the origin is a pixel"), (0, 0));
assert_eq!(root_point(0.4, 0.4).expect("on screen"), (0, 0));
}
#[test]
fn a_negative_point_is_refused_by_name_not_clamped() {
for (x, y) in [(-1.0, 100.0), (100.0, -1.0), (-1920.0, -1080.0)] {
let error = root_point(x, y).expect_err("off the root window");
let message = error.to_string();
assert!(message.contains("root window"), "{message}");
assert!(
message.contains(&format!("({x:.0}, {y:.0})")),
"names the point it refused: {message}"
);
}
}
}
}
#[cfg(target_os = "macos")]
mod platform {
use anyhow::{Result, anyhow};
use enigo::{
Axis as EnigoAxis, Button as EnigoButton, Coordinate, Direction, Enigo, Keyboard, Mouse,
Settings,
};
use pixelactions_core::flow::Axis;
use super::{Button, Injector, keys::key_for};
pub struct RealInjector {
enigo: Enigo,
}
impl RealInjector {
pub fn new() -> Result<Self> {
let enigo = Enigo::new(&Settings::default()).map_err(|e| {
anyhow!(
"cannot synthesize input: {e}. On macOS this usually means the \
Accessibility permission is missing — grant it under System Settings \
> Privacy & Security > Accessibility for the terminal running \
pixelactions, then try again"
)
})?;
Ok(Self { enigo })
}
}
fn to_enigo(button: Button) -> EnigoButton {
match button {
Button::Left => EnigoButton::Left,
}
}
impl Injector for RealInjector {
fn move_to(&mut self, x: f64, y: f64) -> Result<()> {
self.enigo
.move_mouse(x as i32, y as i32, Coordinate::Abs)
.map_err(|e| anyhow!("move to ({x:.0}, {y:.0}) failed: {e}"))
}
fn click(&mut self, button: Button) -> Result<()> {
self.enigo
.button(to_enigo(button), Direction::Click)
.map_err(|e| anyhow!("click failed: {e}"))
}
fn double_click(&mut self, button: Button) -> Result<()> {
self.click(button)?;
std::thread::sleep(std::time::Duration::from_millis(40));
self.click(button)
}
fn press(&mut self, button: Button) -> Result<()> {
self.enigo
.button(to_enigo(button), Direction::Press)
.map_err(|e| anyhow!("press failed: {e}"))
}
fn release(&mut self, button: Button) -> Result<()> {
self.enigo
.button(to_enigo(button), Direction::Release)
.map_err(|e| anyhow!("release failed: {e}"))
}
fn text(&mut self, text: &str) -> Result<()> {
self.enigo
.text(text)
.map_err(|e| anyhow!("typing failed: {e}"))
}
fn scroll(&mut self, amount: i32, axis: Axis) -> Result<()> {
let axis = match axis {
Axis::Vertical => EnigoAxis::Vertical,
Axis::Horizontal => EnigoAxis::Horizontal,
};
self.enigo
.scroll(amount, axis)
.map_err(|e| anyhow!("cannot scroll: {e}"))
}
fn cursor(&mut self) -> Result<(f64, f64)> {
let (x, y) = self
.enigo
.location()
.map_err(|e| anyhow!("cannot read the cursor position: {e}"))?;
Ok((f64::from(x), f64::from(y)))
}
fn probe(&mut self) -> Result<()> {
let (x, y) = self
.enigo
.location()
.map_err(|e| anyhow!("cannot read the cursor position: {e}"))?;
let target = (x + 1, y);
self.enigo
.move_mouse(target.0, target.1, Coordinate::Abs)
.map_err(|e| anyhow!("cannot move the cursor: {e}"))?;
std::thread::sleep(std::time::Duration::from_millis(60));
let after = self
.enigo
.location()
.map_err(|e| anyhow!("cannot read the cursor position: {e}"))?;
let _ = self.enigo.move_mouse(x, y, Coordinate::Abs);
if after == (x, y) {
return Err(anyhow!(
"the cursor did not move — macOS accepted the event and discarded it, \
which is what happens without the Accessibility permission. Grant it \
under System Settings > Privacy & Security > Accessibility for the \
application running pixelactions (your terminal, if you launched it \
from one), then quit and reopen that application"
));
}
Ok(())
}
fn chord(&mut self, chord: &str) -> Result<()> {
let (modifiers, key) = pixelactions_core::chord::split(chord)?;
let mut held = Vec::new();
for token in &modifiers {
let modifier = key_for(token)?;
self.enigo
.key(modifier, Direction::Press)
.map_err(|e| anyhow!("holding {token} failed: {e}"))?;
held.push(modifier);
}
let result = self
.enigo
.key(key_for(key)?, Direction::Click)
.map_err(|e| anyhow!("pressing {key} failed: {e}"));
for modifier in held.into_iter().rev() {
let _ = self.enigo.key(modifier, Direction::Release);
}
result
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_recording_injector_captures_order() {
let mut injector = Recording::default();
injector.move_to(10.4, 20.6).expect("recorded");
injector.click(Button::Left).expect("recorded");
injector.text("hi").expect("recorded");
assert_eq!(injector.events, vec!["move 10,21", "click", "text hi"]);
}
}