#![cfg(target_os = "macos")]
#![allow(
clippy::borrow_as_ptr,
clippy::cast_lossless,
clippy::cast_possible_truncation,
clippy::cast_possible_wrap,
clippy::cast_precision_loss,
clippy::cast_sign_loss,
clippy::ptr_as_ptr,
clippy::ref_as_ptr,
clippy::semicolon_if_nothing_returned
)]
use super::core::{
self, AppInfo, Backend, Capture, CaptureBudget, ElementAct, Impact, Locator, Modifier,
MouseButton, Observation, RawInput, ScrollDirection, SurfaceGeometry, TargetSpec, Truncation,
UiNode, UnreadWindows, WindowInfo, WindowKey, WindowListing,
};
use crate::util::{UnwrapPoison, with_block_in_place};
use anyhow::anyhow;
use block2::RcBlock;
use dispatch2::{DispatchRetained, DispatchSemaphore, DispatchTime};
use objc2::AnyThread;
use objc2::rc::Retained;
use objc2_app_kit::{NSApplicationActivationPolicy, NSRunningApplication, NSWorkspace};
use objc2_application_services::{
self, AXError, AXIsProcessTrusted, AXIsProcessTrustedWithOptions, AXUIElement, AXValue,
AXValueType, kAXTrustedCheckOptionPrompt,
};
use objc2_core_foundation::{
CFArray, CFBoolean, CFDictionary, CFRange, CFRetained, CFString, CFType, CFURL, CGPoint,
CGRect, CGSize, ConcreteType, Type,
};
use objc2_core_graphics::{
CGBitmapContextCreate, CGBitmapContextGetData, CGColorSpace, CGContext, CGDirectDisplayID,
CGDisplayBounds, CGError, CGEvent, CGEventField, CGEventFlags, CGEventSource,
CGEventSourceStateID, CGEventTapLocation, CGEventType, CGGetActiveDisplayList, CGImage,
CGImageAlphaInfo, CGImageByteOrderInfo, CGMouseButton, CGPreflightPostEventAccess,
CGPreflightScreenCaptureAccess, CGRequestPostEventAccess, CGRequestScreenCaptureAccess,
CGScrollEventUnit,
};
use objc2_foundation::{NSArray, NSError, NSString, NSURL};
use objc2_screen_capture_kit::{
SCContentFilter, SCScreenshotManager, SCShareableContent, SCStreamConfiguration,
SCStreamErrorCode, SCWindow,
};
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, Mutex, Once};
use std::time::Duration;
pub(crate) static MACOS_BACKEND: MacOsBackend = MacOsBackend;
pub(crate) struct MacOsBackend;
fn install_messaging_timeout() {
static INSTALLED: Once = Once::new();
INSTALLED.call_once(|| {
let err = unsafe {
AXUIElement::new_system_wide()
.set_messaging_timeout(core::EXCHANGE_DEADLINE.as_secs_f32())
};
if err != AXError::Success {
tracing::warn!(
limit_secs = core::EXCHANGE_DEADLINE.as_secs(),
"computer use: could not set the AX messaging timeout ({err:?}) — an \
exchange that never answers is then unbounded"
);
}
});
}
fn catching_foreign<T>(
what: &str,
f: impl FnOnce() -> Result<T, anyhow::Error>,
) -> Result<T, anyhow::Error> {
objc2::exception::catch(std::panic::AssertUnwindSafe(f))
.map_err(|exc| {
let detail = exc
.as_ref()
.map_or_else(|| "an unnamed exception".to_string(), ToString::to_string);
core::framework_failed(what, format!("the system raised an exception: {detail}"))
})
.and_then(std::convert::identity)
}
fn contained<T>(
what: &str,
body: impl FnOnce() -> Result<T, anyhow::Error>,
) -> Result<T, anyhow::Error> {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
catching_foreign(what, body)
}))
.unwrap_or_else(|payload| {
Err(anyhow!(
"{what} panicked: {}",
crate::util::panic_message(payload.as_ref())
))
})
}
fn with_platform<T>(
what: &str,
f: impl FnOnce() -> Result<T, anyhow::Error>,
) -> Result<T, anyhow::Error> {
with_block_in_place(|| {
objc2::rc::autoreleasepool(|_pool| {
catching_foreign(what, || {
install_messaging_timeout();
f()
})
})
})
}
const AX_ROLE: &str = "AXRole";
const AX_TITLE: &str = "AXTitle";
const AX_VALUE: &str = "AXValue";
const AX_SUBROLE: &str = "AXSubrole";
const AX_POSITION: &str = "AXPosition";
const AX_SIZE: &str = "AXSize";
const AX_CHILDREN: &str = "AXChildren";
const AX_WINDOWS: &str = "AXWindows";
const AX_FOCUSED_WINDOW: &str = "AXFocusedWindow";
const AX_FOCUSED: &str = "AXFocused";
const AX_PRESS: &str = "AXPress";
const AX_OPEN: &str = "AXOpen";
const AX_URL: &str = "AXURL";
const AX_RAISE: &str = "AXRaise";
const AX_MINIMIZED: &str = "AXMinimized";
#[inline]
fn axc(s: &'static str) -> CFRetained<CFString> {
CFString::from_static_str(s)
}
#[derive(Clone, Copy)]
enum Grant {
Accessibility,
Input,
ScreenRecording,
}
impl Grant {
fn what(self) -> &'static str {
match self {
Self::Accessibility => "Accessibility",
Self::Input => "permission to send input (clicks, typing and key chords)",
Self::ScreenRecording => "Screen Recording",
}
}
fn pane(self) -> &'static str {
match self {
Self::Accessibility | Self::Input => "Accessibility",
Self::ScreenRecording => "Screen Recording",
}
}
fn held(self) -> bool {
unsafe {
match self {
Self::Accessibility => AXIsProcessTrusted(),
Self::Input => CGPreflightPostEventAccess(),
Self::ScreenRecording => CGPreflightScreenCaptureAccess(),
}
}
}
fn asked(self) -> &'static AtomicBool {
match self {
Self::Accessibility => &ACCESSIBILITY_ASKED,
Self::Input => &INPUT_ASKED,
Self::ScreenRecording => &SCREEN_RECORDING_ASKED,
}
}
fn request(self) {
let (done, wait) = std::sync::mpsc::channel();
std::thread::spawn(move || {
if let Err(e) = contained("ask for the grant", || {
self.prompt();
Ok(())
}) {
tracing::warn!("computer use: {e:#}");
}
let _ = done.send(());
});
let _ = wait.recv_timeout(PROMPT_WAIT);
}
fn prompt(self) {
unsafe {
match self {
Self::Accessibility => {
let options = CFDictionary::<CFString, CFBoolean>::from_slices(
&[kAXTrustedCheckOptionPrompt],
&[CFBoolean::new(true)],
);
let _ = AXIsProcessTrustedWithOptions(Some(options.as_opaque()));
}
Self::Input => {
let _ = CGRequestPostEventAccess();
}
Self::ScreenRecording => {
let _ = CGRequestScreenCaptureAccess();
}
}
}
}
}
const PROMPT_WAIT: Duration = Duration::from_secs(20);
static ACCESSIBILITY_ASKED: AtomicBool = AtomicBool::new(false);
static INPUT_ASKED: AtomicBool = AtomicBool::new(false);
static SCREEN_RECORDING_ASKED: AtomicBool = AtomicBool::new(false);
fn ensure(grant: Grant) -> anyhow::Result<()> {
if grant.held() {
return Ok(());
}
if !grant.asked().swap(true, Ordering::Relaxed) {
tracing::warn!(
"computer use: asking the system to prompt — {}",
permission_error(grant)
);
grant.request();
}
if grant.held() {
return Ok(());
}
Err(permission_error(grant))
}
fn permission_error(grant: Grant) -> anyhow::Error {
let app = responsible_app();
let consequence = match grant {
Grant::Accessibility => {
"Nothing was observed or acted on. The grant covers every process started from that \
terminal, this daemon's own shell commands included."
}
Grant::Input => {
"Clicks, typing and key chords stay unavailable; reading the accessibility tree does \
not need this grant."
}
Grant::ScreenRecording => {
"Restart mahbot before it applies — macOS gives this grant only to a newly started \
process. Stills stay unavailable meanwhile; observe and act keep working. macOS asks \
for this grant again periodically by design — grant it again the same way. Every \
process started from that terminal shares it."
}
};
core::permission_denied(format!(
"{} is not granted to {app}. macOS attributes privacy grants to the terminal this \
daemon was started from, not to the binary itself — open System Settings → Privacy & \
Security → {} and enable {app}. {consequence}",
grant.what(),
grant.pane(),
))
}
fn responsible_app() -> String {
let non_empty = |key: &str| {
std::env::var(key)
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
};
if let Some(bundle_id) = non_empty("__CFBundleIdentifier") {
let resolved = objc2::rc::autoreleasepool(|_pool| {
let id = NSString::from_str(&bundle_id);
catching_foreign("resolve the responsible application", || {
Ok(
NSRunningApplication::runningApplicationsWithBundleIdentifier(&id)
.iter()
.find_map(|app| {
app.localizedName()
.map(|name| name.to_string())
.filter(|name| !name.is_empty())
}),
)
})
})
.inspect_err(|e| tracing::warn!("computer use: {e:#}"))
.ok()
.flatten();
return resolved.unwrap_or(bundle_id);
}
non_empty("TERM_PROGRAM").unwrap_or_else(|| "the terminal you started mahbot from".to_string())
}
fn sck_available() -> bool {
objc2::available!(macos = 14.0, ..)
}
fn screen_capture_needs_14() -> anyhow::Error {
core::taxonomy_error(
core::ERR_UNSUPPORTED,
"screen capture needs macOS 14 or newer; observing and acting through the accessibility \
tree keep working",
)
}
const SELF_AX_WAIT: Duration = Duration::from_secs(5);
#[derive(Debug)]
struct NeedsAxThread;
impl std::fmt::Display for NeedsAxThread {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("mahbot's own window is answered on the main thread")
}
}
impl std::error::Error for NeedsAxThread {}
fn is_ax_thread(err: &anyhow::Error) -> bool {
err.downcast_ref::<NeedsAxThread>().is_some()
}
fn ensure_ax_thread(pid: u32) -> Result<(), anyhow::Error> {
if pid == std::process::id() && objc2::MainThreadMarker::new().is_none() {
return Err(anyhow::Error::new(NeedsAxThread));
}
Ok(())
}
#[derive(Debug)]
struct AxBusy(u64);
impl std::fmt::Display for AxBusy {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"mahbot's own window did not answer within {}s — its main thread is busy, so the call \
gave up instead of hanging",
self.0
)
}
}
impl std::error::Error for AxBusy {}
fn degraded_read(e: anyhow::Error) -> anyhow::Error {
match e.downcast_ref::<AxBusy>() {
Some(busy) => core::taxonomy_error(core::ERR_DEGRADED, format!("{busy}; retry the read")),
None => e,
}
}
fn on_ax_thread<T: Send + 'static>(
pid: u32,
f: impl FnOnce() -> Result<T, anyhow::Error> + Send + 'static,
) -> Result<T, anyhow::Error> {
if pid != std::process::id() || objc2::MainThreadMarker::new().is_some() {
return f();
}
let (tx, rx) = std::sync::mpsc::sync_channel(1);
dispatch2::DispatchQueue::main().exec_async(move || {
let result = contained("access to mahbot's own window", f);
let _ = tx.send(result);
});
match rx.recv_timeout(SELF_AX_WAIT) {
Ok(result) => result,
Err(_) => Err(anyhow::Error::new(AxBusy(SELF_AX_WAIT.as_secs()))),
}
}
fn with_target<T: Send + 'static>(
target: &TargetSpec,
f: impl Fn(&TargetSpec) -> Result<T, anyhow::Error> + Send + 'static,
) -> Result<T, anyhow::Error> {
match f(target) {
Err(e) if is_ax_thread(&e) => {
let owned = target.clone();
on_ax_thread(std::process::id(), move || f(&owned))
}
other => other,
}
}
fn attribute_value(el: &AXUIElement, attr: &CFString) -> Result<CFRetained<CFType>, AXError> {
let mut value: *const CFType = std::ptr::null();
let err = unsafe {
el.copy_attribute_value(
attr,
std::ptr::NonNull::new(&mut value as *mut *const CFType).unwrap(),
)
};
if err != AXError::Success {
return Err(err);
}
match std::ptr::NonNull::new(value.cast_mut()) {
Some(p) => Ok(unsafe { CFRetained::from_raw(p) }),
None => Err(AXError::NoValue),
}
}
fn attribute_text(el: &AXUIElement, attr: &CFString, max: usize) -> Option<String> {
let val = attribute_value(el, attr).ok()?;
let string = val.downcast_ref::<CFString>()?;
let length = string.length();
if length <= 0 {
return Some(String::new());
}
let wanted = length.min(2 * (max as isize + 2));
let mut units = vec![0u16; wanted as usize];
unsafe {
string.characters(CFRange::new(0, wanted), units.as_mut_ptr());
}
Some(String::from_utf16_lossy(&units))
}
fn attribute_name(el: &AXUIElement, attr: &CFString) -> Option<String> {
attribute_text(el, attr, core::MAX_NAME_CHARS).map(|text| core::cap_label(&text))
}
fn attribute_point(el: &AXUIElement, attr: &CFString) -> Option<CGPoint> {
let val = attribute_value(el, attr).ok()?;
let axval = val.downcast_ref::<AXValue>()?;
let mut p = CGPoint::new(0.0, 0.0);
let ok = unsafe {
axval.value(
AXValueType::CGPoint,
std::ptr::NonNull::new(&mut p as *mut _ as *mut _).unwrap(),
)
};
ok.then_some(p)
}
fn attribute_size(el: &AXUIElement, attr: &CFString) -> Option<CGSize> {
let val = attribute_value(el, attr).ok()?;
let axval = val.downcast_ref::<AXValue>()?;
let mut s = CGSize::new(0.0, 0.0);
let ok = unsafe {
axval.value(
AXValueType::CGSize,
std::ptr::NonNull::new(&mut s as *mut _ as *mut _).unwrap(),
)
};
ok.then_some(s)
}
fn attribute_bool(el: &AXUIElement, attr: &CFString) -> Option<bool> {
let val = attribute_value(el, attr).ok()?;
Some(val.downcast_ref::<CFBoolean>()?.value())
}
fn typed_members<T: ConcreteType>(array: &CFArray) -> Vec<CFRetained<T>> {
let members: CFRetained<CFArray<CFType>> =
unsafe { CFRetained::cast_unchecked(array.retain()) };
members
.iter()
.filter_map(|member| member.downcast::<T>().ok())
.collect()
}
fn raw_action_names(el: &AXUIElement) -> Result<Option<Vec<String>>, AXError> {
let mut names: *const CFArray = std::ptr::null();
let err = unsafe {
el.copy_action_names(std::ptr::NonNull::new(&mut names as *mut *const CFArray).unwrap())
};
if err != AXError::Success {
return Err(err);
}
let Some(raw) = std::ptr::NonNull::new(names.cast_mut()) else {
return Ok(Some(Vec::new()));
};
let object: CFRetained<CFType> =
unsafe { CFRetained::cast_unchecked(CFRetained::from_raw(raw)) };
let Ok(array) = object.downcast::<CFArray>() else {
return Ok(None);
};
let names: Vec<String> = typed_members::<CFString>(&array)
.iter()
.map(ToString::to_string)
.collect();
if names.is_empty() && !array.is_empty() {
return Ok(None);
}
Ok(Some(names))
}
fn attribute_children(el: &AXUIElement) -> Vec<CFRetained<AXUIElement>> {
let Ok(val) = attribute_value(el, &axc(AX_CHILDREN)) else {
return Vec::new();
};
let Ok(array) = val.downcast::<CFArray>() else {
return Vec::new();
};
typed_members::<AXUIElement>(&array)
}
fn normalize_action_name(name: &str) -> String {
name.strip_prefix("AX").unwrap_or(name).to_lowercase()
}
fn ax_error_shared(err: AXError, ctx: impl std::fmt::Display) -> Option<anyhow::Error> {
match err {
AXError::CannotComplete => Some(core::taxonomy_error(
core::ERR_DEGRADED,
format!("transient AX failure, retry: {ctx}"),
)),
AXError::InvalidUIElement => Some(core::taxonomy_error(
core::ERR_STALE_ELEMENT,
format!("window closed or changed — re-enumerate: {ctx}"),
)),
AXError::APIDisabled => Some(permission_error(Grant::Accessibility)),
_ => None,
}
}
fn ax_error(err: AXError, ctx: impl std::fmt::Display) -> anyhow::Error {
if let Some(mapped) = ax_error_shared(err, &ctx) {
return mapped;
}
match err {
AXError::AttributeUnsupported
| AXError::ActionUnsupported
| AXError::NoValue
| AXError::NotImplemented
| AXError::ParameterizedAttributeUnsupported => core::taxonomy_error(
core::ERR_UNSUPPORTED,
format!(
"unsupported accessibility operation: {ctx} — act on it by coordinates or use \
another element"
),
),
_ => anyhow!("AX error {err:?}: {ctx}"),
}
}
fn ax_window_error(err: AXError, ctx: impl std::fmt::Display) -> anyhow::Error {
ax_error_shared(err, &ctx).unwrap_or_else(|| {
core::taxonomy_error(
core::ERR_NOT_MATCHED,
format!("no windows enumerated — re-enumerate: {ctx}"),
)
})
}
const MAX_DEPTH: usize = 25;
fn is_secure_field(role: &str, subrole: Option<&str>) -> bool {
role.to_lowercase().contains("secure")
|| subrole.is_some_and(|s| s.to_lowercase().contains("secure"))
}
fn walk_element(
element: &AXUIElement,
origin: (f64, f64),
depth: usize,
handles: &mut Vec<CFRetained<AXUIElement>>,
nodes: &mut usize,
truncation: &mut Truncation,
budget: &core::Budget,
) -> anyhow::Result<Option<UiNode>> {
budget.check()?;
if *nodes >= core::MAX_TREE_NODES {
truncation.hit_node_ceiling(core::MAX_TREE_NODES);
return Ok(None);
}
*nodes += 1;
handles.push(element.retain());
budget.check()?;
let role = attribute_name(element, &axc(AX_ROLE)).unwrap_or_else(|| "unknown".to_string());
let subrole = attribute_name(element, &axc(AX_SUBROLE));
budget.check()?;
let value = if is_secure_field(&role, subrole.as_deref()) {
None
} else {
attribute_text(element, &axc(AX_VALUE), core::MAX_VALUE_CHARS)
.map(|value| truncation.keep_value(&value))
};
budget.check()?;
let frame = match (
attribute_point(element, &axc(AX_POSITION)),
attribute_size(element, &axc(AX_SIZE)),
) {
(Some(p), Some(s)) => Some((p.x - origin.0, p.y - origin.1, s.width, s.height)),
_ => None,
};
budget.check()?;
let actions = raw_action_names(element)
.ok()
.flatten()
.unwrap_or_default()
.iter()
.map(|a| normalize_action_name(a))
.collect();
budget.check()?;
let focused = attribute_bool(element, &axc(AX_FOCUSED)).unwrap_or(false);
let mut children = Vec::new();
if depth < MAX_DEPTH {
budget.check()?;
for child in attribute_children(element) {
if let Some(child_node) = walk_element(
&child,
origin,
depth + 1,
handles,
nodes,
truncation,
budget,
)? {
children.push(child_node);
}
}
} else if !attribute_children(element).is_empty() {
truncation.cut_depth(MAX_DEPTH);
}
Ok(Some(UiNode {
role,
name: attribute_text(element, &axc(AX_TITLE), core::MAX_NAME_CHARS)
.map(|name| truncation.keep_name(&name)),
value,
frame,
actions,
focused,
children,
}))
}
fn build_tree(
root: &AXUIElement,
origin: (f64, f64),
budget: &core::Budget,
) -> anyhow::Result<(UiNode, Vec<CFRetained<AXUIElement>>, Truncation)> {
let mut handles = Vec::new();
let mut nodes = 0usize;
let mut truncation = Truncation::default();
let node = walk_element(
root,
origin,
0,
&mut handles,
&mut nodes,
&mut truncation,
budget,
)?
.ok_or_else(|| {
core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"{} did not answer for its accessibility root, so nothing was read — {}",
budget.target(),
core::unanswered_next_step()
),
)
})?;
Ok((node, handles, truncation))
}
fn resolved_app_name(target: &TargetSpec, app: &AXUIElement) -> Result<String, anyhow::Error> {
match target.app() {
Some((_, name)) => Ok(name.to_string()),
None => Ok(app_display_name(app_pid(app)?, app)),
}
}
fn resolved_surface_label(
target: &TargetSpec,
app: &AXUIElement,
title: Option<&str>,
) -> Result<String, anyhow::Error> {
Ok(core::surface_label(&resolved_app_name(target, app)?, title))
}
fn observe_blocking(target: &TargetSpec) -> Result<Observation, anyhow::Error> {
let mut budget = core::Budget::new(core::OBSERVE_BUDGET, "observation", target.describe());
let (window, app) = resolve_window(target, &budget)?;
budget.check()?;
let app_name = resolved_app_name(target, &app)?;
let title = attribute_text(&window, &axc(AX_TITLE), core::MAX_NAME_CHARS);
let label = core::name_surface(&mut budget, &app_name, title.as_deref());
refuse_minimized(&window)?;
budget.check()?;
let origin =
window_origin(&window).map_err(|_| core::unanswered(&label, "for its position"))?;
let surface =
window_frame(&window, &budget).map_err(|_| core::unanswered(&label, "for its frame"))?;
budget.check()?;
let (root, _handles, mut truncation) = build_tree(&window, origin, &budget)?;
let title = title.map(|title| truncation.keep_name(&title));
Ok(Observation {
truncation,
app_name,
window_title: title,
surface,
root,
})
}
struct HeldWindow {
element: CFRetained<AXUIElement>,
owner: String,
}
unsafe impl Send for HeldWindow {}
static HELD_WINDOWS: LazyLock<Mutex<BTreeMap<u64, HeldWindow>>> =
LazyLock::new(|| Mutex::new(BTreeMap::new()));
static NEXT_WINDOW_TOKEN: AtomicU64 = AtomicU64::new(1);
const MAX_HELD_WINDOWS: usize = core::MAX_STORED_TARGETS;
fn window_key(window: &AXUIElement, owner: &str) -> WindowKey {
let mut held = HELD_WINDOWS.lock().unwrap_poison();
let known = held
.iter()
.find(|(_, existing)| existing.owner == owner && &*existing.element == window)
.map(|(token, _)| *token);
let token = if let Some(token) = known {
token
} else {
let token = NEXT_WINDOW_TOKEN.fetch_add(1, Ordering::Relaxed);
held.insert(
token,
HeldWindow {
element: window.retain(),
owner: owner.to_string(),
},
);
let mine: Vec<u64> = held
.iter()
.filter(|(_, entry)| entry.owner == owner)
.map(|(token, _)| *token)
.collect();
for token in mine
.iter()
.take(mine.len().saturating_sub(MAX_HELD_WINDOWS))
{
held.remove(token);
}
token
};
WindowKey::new(format!("ax:{token}"))
}
fn window_by_key(
key: &WindowKey,
app_name: &str,
) -> Result<CFRetained<AXUIElement>, anyhow::Error> {
let token = key
.as_str()
.strip_prefix("ax:")
.and_then(|raw| raw.parse::<u64>().ok())
.ok_or_else(|| core::surface_gone(app_name))?;
let window = {
let held = HELD_WINDOWS.lock().unwrap_poison();
held.get(&token)
.map(|held| held.element.retain())
.ok_or_else(|| core::surface_gone(app_name))?
};
match attribute_value(&window, &axc(AX_ROLE)) {
Ok(_) => Ok(window),
Err(AXError::InvalidUIElement) => Err(core::surface_gone(app_name)),
Err(_) => Err(core::unanswered(
&core::surface_label(app_name, None),
"for its role",
)),
}
}
pub(crate) fn forget_agent_windows(agent: &str) {
let mut held = HELD_WINDOWS.lock().unwrap_poison();
held.retain(|_, entry| entry.owner != agent);
}
fn refuse_minimized(window: &AXUIElement) -> Result<(), anyhow::Error> {
if !attribute_bool(window, &axc(AX_MINIMIZED)).unwrap_or(false) {
return Ok(());
}
Err(core::minimized_window(
&window_title(window).unwrap_or_default(),
))
}
fn frontmost_app() -> Result<CFRetained<AXUIElement>, anyhow::Error> {
let app = NSWorkspace::sharedWorkspace()
.frontmostApplication()
.ok_or_else(|| {
core::taxonomy_error(
core::ERR_NOT_MATCHED,
"no frontmost application — enumerate apps/windows first",
)
})?;
Ok(unsafe { AXUIElement::new_application(app.processIdentifier()) })
}
fn focused_window_of_app(app: &AXUIElement) -> Option<CFRetained<AXUIElement>> {
attribute_value(app, &axc(AX_FOCUSED_WINDOW))
.ok()?
.downcast::<AXUIElement>()
.ok()
}
fn running_app_pid(name: &str) -> Option<u32> {
let needle = name.to_lowercase();
NSWorkspace::sharedWorkspace()
.runningApplications()
.iter()
.find(|app| {
app.activationPolicy() == NSApplicationActivationPolicy::Regular
&& app
.localizedName()
.is_some_and(|n| n.to_string().to_lowercase() == needle)
})
.map(|app| app.processIdentifier() as u32)
}
fn app_display_name(pid: u32, app: &AXUIElement) -> String {
NSWorkspace::sharedWorkspace()
.runningApplications()
.iter()
.find(|a| a.processIdentifier() == pid as i32)
.and_then(|a| a.localizedName().map(|n| n.to_string()))
.filter(|n| !n.is_empty())
.unwrap_or_else(|| attribute_name(app, &axc(AX_TITLE)).unwrap_or_default())
}
fn app_element(pid: Option<u32>, name: &str) -> Result<CFRetained<AXUIElement>, anyhow::Error> {
let pid = match pid {
Some(p) => p,
None => running_app_pid(name).ok_or_else(|| core::surface_gone(name))?,
};
Ok(unsafe { AXUIElement::new_application(pid as libc::pid_t) })
}
fn read_windows(
app: &AXUIElement,
app_name: &str,
) -> Result<Vec<CFRetained<AXUIElement>>, anyhow::Error> {
let val = attribute_value(app, &axc(AX_WINDOWS)).map_err(|e| {
if e == AXError::CannotComplete {
core::unanswered(&core::surface_label(app_name, None), "for its window list")
} else {
ax_window_error(e, "enumerate windows")
}
})?;
let array: CFRetained<CFArray> = val
.downcast::<CFArray>()
.map_err(|_| anyhow!("AXWindows was not a CFArray"))?;
Ok(typed_members::<AXUIElement>(&array))
}
fn windows_of_app(
pid: u32,
app_name: &str,
owner: &str,
budget: &core::Budget,
) -> Result<Vec<WindowInfo>, anyhow::Error> {
let app = unsafe { AXUIElement::new_application(pid as libc::pid_t) };
budget.check()?;
let windows = read_windows(&app, app_name)?;
let mut out = Vec::new();
for window in &windows {
budget.check()?;
let frame = match window_frame(window, budget) {
Ok(frame) => frame,
Err(e) if budget.expired() => return Err(e),
Err(_) => continue,
};
budget.check()?;
let title = window_title(window).unwrap_or_default();
if title.is_empty() && (frame.width <= 0.0 || frame.height <= 0.0) {
continue;
}
out.push(WindowInfo {
app_pid: Some(pid),
app_name: app_name.to_string(),
title,
key: window_key(window, owner),
surface: frame,
});
}
Ok(out)
}
fn resolve_window(
target: &TargetSpec,
budget: &core::Budget,
) -> Result<(CFRetained<AXUIElement>, CFRetained<AXUIElement>), anyhow::Error> {
budget.check()?;
match target {
TargetSpec::Focused => {
let app = frontmost_app()?;
ensure_ax_thread(app_pid(&app)?)?;
let Some(window) = focused_window_of_app(&app) else {
return Err(match attribute_value(&app, &axc(AX_ROLE)) {
Ok(_) => core::taxonomy_error(
core::ERR_NOT_MATCHED,
"no focused window (desktop frontmost/screensaver) — use \
'apps'/'windows' to pick a surface",
),
Err(_) => core::unanswered(
&format!(
"the frontmost application ({})",
app_display_name(app_pid(&app)?, &app)
),
"for its focused window",
),
});
};
Ok((window, app))
}
TargetSpec::App {
app_pid: pinned,
app_name,
} => {
let app = app_element(*pinned, app_name)?;
let pid = app_pid(&app)?;
ensure_ax_thread(pid)?;
budget.check()?;
let window = match focused_window_of_app(&app) {
Some(window) => window,
None => read_windows(&app, app_name)?
.into_iter()
.next()
.ok_or_else(|| core::surface_gone(app_name))?,
};
Ok((window, app))
}
TargetSpec::Window {
app_pid: pinned,
app_name,
key,
..
} => {
let app = app_element(*pinned, app_name)?;
ensure_ax_thread(app_pid(&app)?)?;
Ok((window_by_key(key, app_name)?, app))
}
TargetSpec::Screen => Err(core::screen_observe_error()),
}
}
fn window_origin(window: &AXUIElement) -> Result<(f64, f64), anyhow::Error> {
let p = attribute_point(window, &axc(AX_POSITION))
.ok_or_else(|| anyhow!("window position unavailable"))?;
Ok((p.x, p.y))
}
fn window_frame(
window: &AXUIElement,
budget: &core::Budget,
) -> Result<SurfaceGeometry, anyhow::Error> {
budget.check()?;
let p = attribute_point(window, &axc(AX_POSITION))
.ok_or_else(|| anyhow!("window position unavailable"))?;
budget.check()?;
let s =
attribute_size(window, &axc(AX_SIZE)).ok_or_else(|| anyhow!("window size unavailable"))?;
Ok(SurfaceGeometry {
x: p.x,
y: p.y,
width: s.width,
height: s.height,
})
}
fn app_pid(app: &AXUIElement) -> Result<u32, anyhow::Error> {
let mut pid: libc::pid_t = 0;
let err = unsafe { app.pid(std::ptr::NonNull::new(&mut pid).unwrap()) };
if err != AXError::Success {
return Err(ax_error(err, "read application pid"));
}
Ok(pid as u32)
}
fn cg_mouse_button(button: MouseButton) -> CGMouseButton {
match button {
MouseButton::Left => CGMouseButton::Left,
MouseButton::Right => CGMouseButton::Right,
MouseButton::Middle => CGMouseButton::Center,
}
}
fn cg_type_for(button: MouseButton, down: bool) -> CGEventType {
match (button, down) {
(MouseButton::Left, true) => CGEventType::LeftMouseDown,
(MouseButton::Left, false) => CGEventType::LeftMouseUp,
(MouseButton::Right, true) => CGEventType::RightMouseDown,
(MouseButton::Right, false) => CGEventType::RightMouseUp,
(MouseButton::Middle, true) => CGEventType::OtherMouseDown,
(MouseButton::Middle, false) => CGEventType::OtherMouseUp,
}
}
fn cg_flags(modifiers: &[Modifier]) -> CGEventFlags {
modifiers.iter().fold(CGEventFlags::empty(), |acc, m| {
acc | match m {
Modifier::Cmd => CGEventFlags::MaskCommand,
Modifier::Ctrl => CGEventFlags::MaskControl,
Modifier::Alt => CGEventFlags::MaskAlternate,
Modifier::Shift => CGEventFlags::MaskShift,
}
})
}
fn post_event(event: &CGEvent) {
CGEvent::post(CGEventTapLocation::HIDEventTap, Some(event));
}
fn source() -> Option<CFRetained<CGEventSource>> {
CGEventSource::new(CGEventSourceStateID::CombinedSessionState)
}
fn mouse_event(
source: Option<&CGEventSource>,
kind: CGEventType,
point: CGPoint,
button: CGMouseButton,
) -> Result<CFRetained<CGEvent>, anyhow::Error> {
CGEvent::new_mouse_event(source, kind, point, button)
.ok_or_else(|| anyhow!("failed to create a mouse event"))
}
fn keyboard_event(
source: Option<&CGEventSource>,
code: u16,
down: bool,
) -> Result<CFRetained<CGEvent>, anyhow::Error> {
CGEvent::new_keyboard_event(source, code, down)
.ok_or_else(|| anyhow!("failed to create a keyboard event"))
}
fn click(
point: CGPoint,
button: MouseButton,
double: bool,
modifiers: &[Modifier],
) -> Result<Impact, anyhow::Error> {
let src = source();
let flags = cg_flags(modifiers);
let state = if double { 2 } else { 1 };
let mut events = Vec::with_capacity(if double { 4 } else { 2 });
for _ in 0..if double { 2 } else { 1 } {
for down in [true, false] {
let ev = mouse_event(
src.as_deref(),
cg_type_for(button, down),
point,
cg_mouse_button(button),
)?;
CGEvent::set_flags(Some(&ev), flags);
CGEvent::set_integer_value_field(Some(&ev), CGEventField::MouseEventClickState, state);
events.push(ev);
}
}
for ev in &events {
post_event(ev);
}
Ok(Impact::Done)
}
fn sleep_millis(ms: u64) {
std::thread::sleep(std::time::Duration::from_millis(ms));
}
fn ascii_key(c: char) -> Option<i64> {
Some(match c {
'0' => 29,
'1' => 18,
'2' => 19,
'3' => 20,
'4' => 21,
'5' => 23,
'6' => 22,
'7' => 26,
'8' => 28,
'9' => 25,
'a' => 0,
's' => 1,
'd' => 2,
'f' => 3,
'h' => 4,
'g' => 5,
'z' => 6,
'x' => 7,
'c' => 8,
'v' => 9,
'b' => 11,
'q' => 12,
'w' => 13,
'e' => 14,
'r' => 15,
'y' => 16,
't' => 17,
'i' => 34,
'o' => 31,
'p' => 35,
'l' => 37,
'j' => 38,
'k' => 40,
'n' => 45,
'm' => 46,
'u' => 32,
_ => return None,
})
}
fn key_code(key: &str) -> Option<i64> {
let k = key.to_lowercase();
if k.len() == 1
&& let Some(c) = k.chars().next().and_then(ascii_key)
{
return Some(c);
}
Some(match k.as_str() {
"return" | "enter" => 36,
"tab" => 48,
"space" => 49,
"delete" => 51, "forwarddelete" => 117,
"escape" => 53,
"home" => 115,
"end" => 119,
"pageup" => 116,
"pagedown" => 121,
"up" => 126,
"down" => 125,
"left" => 123,
"right" => 124,
"f1" => 122,
"f2" => 120,
"f3" => 99,
"f4" => 118,
"f5" => 96,
"f6" => 97,
"f7" => 98,
"f8" => 100,
"f9" => 101,
"f10" => 109,
"f11" => 103,
"f12" => 111,
"minus" => 27,
"equal" => 24,
"comma" => 43,
"period" => 47,
"slash" => 44,
"semicolon" => 41,
"quote" => 39,
"backslash" => 42,
"backtick" => 50,
"leftbracket" => 33,
"rightbracket" => 30,
"capslock" => 57,
_ => return None,
})
}
const KNOWN_KEYS: &str = "return, enter, tab, space, delete, forwarddelete, escape, home, end, \
pageup, pagedown, up, down, left, right, f1..f12, minus, equal, comma, period, slash, \
semicolon, quote, backslash, backtick, leftbracket, rightbracket, capslock, a-z, 0-9";
fn lookup_key(key: &str) -> Result<i64, anyhow::Error> {
key_code(key).ok_or_else(|| anyhow!("unknown key '{key}' — supported keys: {KNOWN_KEYS}"))
}
fn press_key(key: &str) -> Result<(), anyhow::Error> {
let code = lookup_key(key)?;
let src = source();
let down = keyboard_event(src.as_deref(), code as u16, true)?;
let up = keyboard_event(src.as_deref(), code as u16, false)?;
post_event(&down);
post_event(&up);
Ok(())
}
fn type_text(text: &str) -> Result<Impact, anyhow::Error> {
fn type_char(src: Option<&CGEventSource>, ch: char) -> Result<(), anyhow::Error> {
let mut buf = [0u16; 2];
let encoded = ch.encode_utf16(&mut buf);
let down = keyboard_event(src, 0, true)?;
let up = keyboard_event(src, 0, false)?;
for event in [&down, &up] {
unsafe {
CGEvent::keyboard_set_unicode_string(
Some(event),
encoded.len() as u64,
encoded.as_ptr(),
)
};
}
post_event(&down);
post_event(&up);
Ok(())
}
let src = source();
let total = text.chars().count();
for (typed, ch) in text.chars().enumerate() {
let made = if ch == '\n' {
press_key("return")
} else {
type_char(src.as_deref(), ch)
};
match made {
Ok(()) => sleep_millis(1),
Err(e) if typed == 0 => return Err(e),
Err(e) => {
return Ok(Impact::Unconfirmed(format!(
"{typed} of {total} characters were typed before the next keyboard event \
could not be created ({e})"
)));
}
}
}
Ok(Impact::Done)
}
fn key_chord(chord: &str) -> Result<Impact, anyhow::Error> {
let (modifiers, key) = core::parse_key_chord(chord)?;
let code = lookup_key(&key)?;
let src = source();
let flags = cg_flags(&modifiers);
let down = keyboard_event(src.as_deref(), code as u16, true)?;
let up = keyboard_event(src.as_deref(), code as u16, false)?;
CGEvent::set_flags(Some(&down), flags);
CGEvent::set_flags(Some(&up), CGEventFlags::empty());
post_event(&down);
post_event(&up);
Ok(Impact::Done)
}
fn scroll(
point: Option<CGPoint>,
direction: ScrollDirection,
amount: u32,
) -> Result<Impact, anyhow::Error> {
let src = source();
let ticks = i32::try_from(amount).unwrap_or(i32::MAX);
let (w1, w2) = match direction {
ScrollDirection::Down => (ticks, 0),
ScrollDirection::Up => (-ticks, 0),
ScrollDirection::Right => (0, ticks),
ScrollDirection::Left => (0, -ticks),
};
let ev =
CGEvent::new_scroll_wheel_event2(src.as_deref(), CGScrollEventUnit::Line, 2, w1, w2, 0)
.ok_or_else(|| anyhow!("failed to create a scroll event"))?;
if let Some(p) = point {
CGEvent::set_location(Some(&ev), p);
}
post_event(&ev);
Ok(Impact::Done)
}
fn drag(from: CGPoint, to: CGPoint) -> Result<Impact, anyhow::Error> {
let src = source();
let mut events = Vec::with_capacity(14);
events.push(mouse_event(
src.as_deref(),
CGEventType::LeftMouseDown,
from,
CGMouseButton::Left,
)?);
for i in 1..=12 {
let t = i as f64 / 12.0;
events.push(mouse_event(
src.as_deref(),
CGEventType::LeftMouseDragged,
CGPoint::new(from.x + (to.x - from.x) * t, from.y + (to.y - from.y) * t),
CGMouseButton::Left,
)?);
}
events.push(mouse_event(
src.as_deref(),
CGEventType::LeftMouseUp,
to,
CGMouseButton::Left,
)?);
for event in &events {
post_event(event);
sleep_millis(2);
}
Ok(Impact::Done)
}
fn active_display_ids() -> Vec<CGDirectDisplayID> {
let mut displays = [0u32; 32];
let mut count = 0u32;
let err =
unsafe { CGGetActiveDisplayList(displays.len() as u32, displays.as_mut_ptr(), &mut count) };
if err != CGError::Success {
return Vec::new();
}
displays[..count as usize].to_vec()
}
fn screen_surface() -> Result<SurfaceGeometry, anyhow::Error> {
let ids = active_display_ids();
if ids.is_empty() {
anyhow::bail!("no active displays found");
}
let mut min_x = f64::INFINITY;
let mut min_y = f64::INFINITY;
let mut max_x = f64::NEG_INFINITY;
let mut max_y = f64::NEG_INFINITY;
for id in ids {
let b = CGDisplayBounds(id);
min_x = min_x.min(b.origin.x);
min_y = min_y.min(b.origin.y);
max_x = max_x.max(b.origin.x + b.size.width);
max_y = max_y.max(b.origin.y + b.size.height);
}
Ok(SurfaceGeometry {
x: min_x,
y: min_y,
width: max_x - min_x,
height: max_y - min_y,
})
}
fn surface_geometry_blocking(
target: &TargetSpec,
budget: &core::Budget,
) -> Result<SurfaceGeometry, anyhow::Error> {
if matches!(target, TargetSpec::Screen) {
return screen_surface();
}
let (window, app) = resolve_window(target, budget)?;
budget.check()?;
let label = resolved_surface_label(target, &app, window_title(&window).as_deref())?;
budget.check()?;
refuse_minimized(&window)?;
window_frame(&window, budget).map_err(|_| core::unanswered(&label, "for its frame"))
}
fn surface_geometry_with(
target: &TargetSpec,
budget: core::Budget,
) -> Result<SurfaceGeometry, anyhow::Error> {
if !matches!(target, TargetSpec::Screen) {
ensure(Grant::Accessibility)?;
}
with_target(target, move |target| {
surface_geometry_blocking(target, &budget)
})
}
fn raise_window(window: &AXUIElement) -> Result<bool, anyhow::Error> {
let Some(names) = raw_action_names(window).ok().flatten() else {
return Ok(false);
};
if !names.iter().any(|a| a == AX_RAISE) {
return Ok(false);
}
let err = unsafe { window.perform_action(&axc(AX_RAISE)) };
if err != AXError::Success {
return Err(ax_error(err, "raise window"));
}
Ok(true)
}
fn activate_window(target: &TargetSpec, budget: &core::Budget) -> Result<bool, anyhow::Error> {
let (TargetSpec::Focused | TargetSpec::App { .. } | TargetSpec::Window { .. }) = target else {
return Ok(false);
};
let (window, _app) = match resolve_window(target, budget) {
Ok(resolved) => resolved,
Err(e) if !is_ax_thread(&e) => return Ok(false),
Err(e) => return Err(e),
};
raise_window(&window)
}
fn window_receives_input(window: &AXUIElement) -> bool {
let Ok(app) = frontmost_app() else {
return false;
};
focused_window_of_app(&app).is_some_and(|focused| &*focused == window)
}
fn ensure_window_receives(target: &TargetSpec, budget: &core::Budget) -> Result<(), anyhow::Error> {
for attempt in 0..=core::RAISE_SETTLE_STEPS {
if attempt > 0 {
sleep_millis(core::RAISE_SETTLE_MS);
}
budget.check()?;
let probe = budget.clone();
let receives = with_target(target, move |target| {
let (window, _app) = resolve_window(target, &probe)?;
Ok(window_receives_input(&window))
})?;
if receives {
return Ok(());
}
}
Err(core::not_brought_forward(target.describe()))
}
fn bring_window_forward(
window: &AXUIElement,
surface: &str,
budget: &core::Budget,
) -> Result<(), anyhow::Error> {
raise_window(window)?;
for attempt in 0..=core::RAISE_SETTLE_STEPS {
if attempt > 0 {
sleep_millis(core::RAISE_SETTLE_MS);
}
budget.check()?;
if window_receives_input(window) {
return Ok(());
}
}
Err(core::not_brought_forward(surface))
}
fn check_input_arguments(input: &RawInput) -> Result<(), anyhow::Error> {
core::check_raw_input(input)?;
match input {
RawInput::KeyChord { chord } => lookup_key(&core::parse_key_chord(chord)?.1).map(|_| ()),
_ => Ok(()),
}
}
fn raw_input_blocking(target: &TargetSpec, input: &RawInput) -> Result<Impact, anyhow::Error> {
let pinned = core::pinned_input(target, input);
let raises = !matches!(input, RawInput::KeyChord { .. });
let lookup = core::Budget::new(core::LIST_BUDGET, "surface lookup", target.describe());
let geometry = lookup.clone();
if pinned {
let raise = lookup.clone();
with_target(target, move |target| {
let (window, _app) = resolve_window(target, &raise)?;
raise_window(&window).map(|_| ())
})?;
with_target(target, move |target| {
ensure_window_receives(target, &lookup)
})?;
} else if raises {
let raise = lookup.clone();
if with_target(target, move |target| activate_window(target, &raise))? {
sleep_millis((core::RAISE_SETTLE_STEPS as u64) * core::RAISE_SETTLE_MS);
}
}
match input {
RawInput::Click {
point,
button,
double,
modifiers,
} => {
let (x, y) = core::normalized_to_surface(
point.0,
point.1,
&surface_geometry_with(target, geometry)?,
)?;
click(CGPoint::new(x, y), *button, *double, modifiers)
}
RawInput::TypeText { text } => type_text(text),
RawInput::KeyChord { chord } => key_chord(chord),
RawInput::Scroll {
point,
direction,
amount,
} => {
let p = if let Some((nx, ny)) = point {
let (x, y) = core::normalized_to_surface(
*nx,
*ny,
&surface_geometry_with(target, geometry)?,
)?;
Some(CGPoint::new(x, y))
} else {
None
};
scroll(p, *direction, *amount)
}
RawInput::Drag { from, to } => {
let surface = surface_geometry_with(target, geometry)?;
let (fx, fy) = core::normalized_to_surface(from.0, from.1, &surface)?;
let (tx, ty) = core::normalized_to_surface(to.0, to.1, &surface)?;
drag(CGPoint::new(fx, fy), CGPoint::new(tx, ty))
}
}
}
struct SendShareable(Retained<SCShareableContent>);
unsafe impl Send for SendShareable {}
struct SendImage(Retained<CGImage>);
unsafe impl Send for SendImage {}
fn sc_error(err: &NSError) -> anyhow::Error {
let code = err.code();
if code == SCStreamErrorCode::UserDeclined.0 || code == SCStreamErrorCode::MissingEntitlements.0
{
permission_error(Grant::ScreenRecording)
} else if code == SCStreamErrorCode::NoWindowList.0 {
core::taxonomy_error(
core::ERR_NOT_MATCHED,
"no window found — re-enumerate windows",
)
} else {
anyhow!(
"screen capture failed (code {code}): {}",
err.localizedDescription()
)
}
}
fn sc_wait<T: Send>(
slot: &Arc<Mutex<Option<Result<T, anyhow::Error>>>>,
semaphore: &Arc<DispatchRetained<DispatchSemaphore>>,
what: &str,
) -> Result<T, anyhow::Error> {
if semaphore.wait(DispatchTime::NOW.time(5_000_000_000)) != 0 {
return Err(anyhow!("timed out waiting for {what}"));
}
slot.lock()
.unwrap_poison()
.take()
.ok_or_else(|| anyhow!("{what} handler never invoked"))?
}
fn sc_complete<T>(
slot: &Mutex<Option<Result<T, anyhow::Error>>>,
semaphore: &DispatchSemaphore,
what: &str,
body: impl FnOnce() -> Result<T, anyhow::Error>,
) {
let outcome = contained(what, body);
if let Err(e) = &outcome {
tracing::debug!("computer use: {e:#}");
}
*slot.lock().unwrap_poison() = Some(outcome);
semaphore.signal();
}
fn get_shareable_content() -> Result<Retained<SCShareableContent>, anyhow::Error> {
let semaphore = Arc::new(DispatchSemaphore::new(0));
let sem2 = Arc::clone(&semaphore);
let slot: Arc<Mutex<Option<Result<SendShareable, anyhow::Error>>>> = Arc::new(Mutex::new(None));
let slot2 = Arc::clone(&slot);
let block = RcBlock::new(move |content: *mut SCShareableContent, err: *mut NSError| {
sc_complete(&slot2, &sem2, "shareable content", || {
if !err.is_null() {
return Err(sc_error(unsafe { &*err }));
}
if content.is_null() {
return Err(anyhow!("no shareable content"));
}
let content = unsafe { Retained::retain(content) }
.ok_or_else(|| anyhow!("null shareable content"))?;
Ok(SendShareable(content))
});
});
unsafe {
SCShareableContent::getShareableContentExcludingDesktopWindows_onScreenWindowsOnly_completionHandler(false, false, &block);
}
sc_wait::<SendShareable>(&slot, &semaphore, "shareable content").map(|SendShareable(c)| c)
}
fn capture_filter_to_rgba(
filter: &SCContentFilter,
px_w: usize,
px_h: usize,
ignore_shadows: bool,
budget: &CaptureBudget,
) -> Result<Capture, anyhow::Error> {
let (px_w, px_h) = core::capture_size("the target", px_w, px_h)?;
let (px_w, px_h) = (px_w as usize, px_h as usize);
let config = unsafe { SCStreamConfiguration::new() };
unsafe {
config.setWidth(px_w);
config.setHeight(px_h);
config.setShowsCursor(false);
config.setIgnoreShadowsSingleWindow(ignore_shadows);
config.setIgnoreShadowsDisplay(ignore_shadows);
}
let semaphore = Arc::new(DispatchSemaphore::new(0));
let sem2 = Arc::clone(&semaphore);
let slot: Arc<Mutex<Option<Result<SendImage, anyhow::Error>>>> = Arc::new(Mutex::new(None));
let slot2 = Arc::clone(&slot);
let block = RcBlock::new(move |image: *mut CGImage, err: *mut NSError| {
sc_complete(&slot2, &sem2, "screenshot", || {
if !err.is_null() {
return Err(sc_error(unsafe { &*err }));
}
if image.is_null() {
return Err(anyhow!("no image in the screenshot"));
}
let image = unsafe { Retained::retain(image) }
.ok_or_else(|| anyhow!("null screenshot image"))?;
Ok(SendImage(image))
});
});
unsafe {
SCScreenshotManager::captureImageWithFilter_configuration_completionHandler(
filter,
&config,
Some(&block),
);
}
let SendImage(image) = sc_wait::<SendImage>(&slot, &semaphore, "screenshot")?;
capture_image_to_rgba(&image, budget)
}
fn capture_image_to_rgba(
image: &CGImage,
budget: &CaptureBudget,
) -> Result<Capture, anyhow::Error> {
let w = CGImage::width(Some(image));
let h = CGImage::height(Some(image));
let (px_w, px_h) = core::capture_size("a captured frame", w, h)?;
let space = CGColorSpace::new_device_rgb()
.ok_or_else(|| anyhow!("failed to create RGB color space"))?;
let bitmap_info = CGImageAlphaInfo::PremultipliedLast.0 | CGImageByteOrderInfo::Order32Big.0;
let ctx = unsafe {
CGBitmapContextCreate(
std::ptr::null_mut(),
w,
h,
8,
w * 4,
Some(&space),
bitmap_info,
)
}
.ok_or_else(|| anyhow!("failed to create bitmap context"))?;
CGContext::draw_image(
Some(&ctx),
CGRect::new(CGPoint::new(0.0, 0.0), CGSize::new(w as f64, h as f64)),
Some(image),
);
let data = CGBitmapContextGetData(Some(&ctx));
if data.is_null() {
anyhow::bail!("bitmap context returned no data");
}
let pixels = unsafe { std::slice::from_raw_parts(data as *const u8, w * h * 4) };
Capture::from_pixels(budget, "a captured frame", px_w, px_h, pixels)
}
fn find_sc_window(
content: &SCShareableContent,
pid: u32,
title: Option<&str>,
frame: SurfaceGeometry,
) -> Result<Retained<SCWindow>, anyhow::Error> {
let near = |window: &SCWindow| {
let f = unsafe { window.frame() };
(f.origin.x - frame.x).abs() < 2.0
&& (f.origin.y - frame.y).abs() < 2.0
&& (f.size.width - frame.width).abs() < 2.0
&& (f.size.height - frame.height).abs() < 2.0
};
let titled = |windows: &[Retained<SCWindow>], title: &str| {
windows
.iter()
.filter(|window| {
unsafe { window.title() }.map(|s| s.to_string()).as_deref() == Some(title)
})
.cloned()
.collect::<Vec<_>>()
};
let of_app = unsafe { content.windows() }
.iter()
.filter(|window| {
let owner = unsafe { window.owningApplication() };
owner.as_ref().map(|a| unsafe { a.processID() } as u32) == Some(pid)
})
.collect::<Vec<_>>();
let of_frame = of_app
.iter()
.filter(|window| near(window))
.cloned()
.collect::<Vec<_>>();
let matching = match title.filter(|t| !t.is_empty()) {
Some(title) if !of_frame.is_empty() => {
let by_title = titled(&of_frame, title);
if by_title.is_empty() {
of_frame
} else {
by_title
}
}
Some(title) => titled(&of_app, title),
None => of_frame,
};
match matching.as_slice() {
[only] => Ok(only.clone()),
[] => Err(core::taxonomy_error(
core::ERR_SURFACE_GONE,
format!(
"{} this target names is not in the capture list — it closed, or the capture \
facility does not expose it; enumerate windows again to pick a surface",
core::window_description("", title.unwrap_or_default())
),
)),
many => Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"{} windows of the same application match this target's title or frame, so the \
capture cannot tell which one to shoot — move or resize one of them, or \
screenshot the screen instead",
many.len()
),
)),
}
}
fn window_capture_facts(
target: &TargetSpec,
budget: &core::Budget,
) -> Result<(u32, Option<String>, SurfaceGeometry), anyhow::Error> {
let (window, app) = resolve_window(target, budget)?;
budget.check()?;
let pid = app_pid(&app)?;
let title = window_title(&window);
budget.check()?;
let label = resolved_surface_label(target, &app, title.as_deref())?;
refuse_minimized(&window)?;
let frame =
window_frame(&window, budget).map_err(|_| core::unanswered(&label, "for its frame"))?;
Ok((pid, title, frame))
}
fn capture_window_blocking(
target: &TargetSpec,
budget: &CaptureBudget,
) -> Result<Capture, anyhow::Error> {
let lookup = core::Budget::new(core::LIST_BUDGET, "surface lookup", target.describe());
let (pid, title, frame) =
with_target(target, move |target| window_capture_facts(target, &lookup))?;
let content = get_shareable_content()?;
let sc_window = find_sc_window(&content, pid, title.as_deref(), frame)?;
let filter = unsafe {
SCContentFilter::initWithDesktopIndependentWindow(SCContentFilter::alloc(), &sc_window)
};
let scale = unsafe { filter.pointPixelScale() } as f64;
let px_w = (frame.width * scale).round();
let px_h = (frame.height * scale).round();
capture_filter_to_rgba(&filter, px_w as usize, px_h as usize, true, budget)
}
fn capture_screen_blocking(budget: &CaptureBudget) -> Result<Capture, anyhow::Error> {
let content = get_shareable_content()?;
let displays = unsafe { content.displays() };
if displays.is_empty() {
anyhow::bail!("no shareable displays found");
}
let empty: Retained<NSArray<SCWindow>> = NSArray::new();
let mut filters = Vec::with_capacity(displays.len());
let mut ref_scale: f64 = 0.0;
for display in &displays {
let filter = unsafe {
SCContentFilter::initWithDisplay_excludingWindows(
SCContentFilter::alloc(),
&display,
&empty,
)
};
let f = unsafe { display.frame() };
ref_scale = ref_scale.max(unsafe { filter.pointPixelScale() } as f64);
filters.push((
filter,
SurfaceGeometry {
x: f.origin.x,
y: f.origin.y,
width: f.size.width,
height: f.size.height,
},
));
}
if ref_scale <= 0.0 {
anyhow::bail!("no display reported a usable pixel scale");
}
let bbox = screen_surface()?;
let out_w = (bbox.width * ref_scale).round() as usize;
let out_h = (bbox.height * ref_scale).round() as usize;
let (out_px_w, out_px_h) = core::capture_size("the virtual desktop", out_w, out_h)?;
let mut composite = Capture::alloc(budget, "the screen composite", out_px_w, out_px_h)?;
for (filter, frame) in filters {
let px_w = (frame.width * ref_scale).round() as usize;
let px_h = (frame.height * ref_scale).round() as usize;
let cap = capture_filter_to_rgba(&filter, px_w, px_h, false, budget)?;
if (cap.width as usize, cap.height as usize) != (px_w, px_h) {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"a display asked for {px_w}x{px_h} pixels came back at {}x{}, so this \
mixed-density arrangement cannot be composited faithfully — screenshot a \
window instead, or capture with the display arrangement made uniform",
cap.width, cap.height
),
));
}
let dx = ((frame.x - bbox.x) * ref_scale).round();
let dy = ((frame.y - bbox.y) * ref_scale).round();
if dx >= 0.0 && dy >= 0.0 {
core::blit_rgba(
&mut composite.rgba,
out_w,
out_h,
&cap,
dx as usize,
dy as usize,
);
}
}
Ok(composite)
}
fn capture_blocking(target: &TargetSpec, budget: &CaptureBudget) -> Result<Capture, anyhow::Error> {
if !sck_available() {
return Err(screen_capture_needs_14());
}
if !matches!(target, TargetSpec::Screen) {
ensure(Grant::Accessibility)?;
}
let result = match target {
TargetSpec::Screen => capture_screen_blocking(budget),
_ => capture_window_blocking(target, budget),
};
result.map_err(|e| ensure(Grant::ScreenRecording).err().unwrap_or(e))
}
fn capture_available_blocking() -> bool {
sck_available() && Grant::ScreenRecording.held()
}
fn press_element(
element: &AXUIElement,
window: &AXUIElement,
center: Option<(f64, f64)>,
surface: &str,
budget: &core::Budget,
) -> Result<Impact, anyhow::Error> {
let names = raw_action_names(element).map_err(|e| ax_error(e, "read element actions"))?;
let Some(names) = names else {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
"the accessibility API answered without usable action names for this element — act \
on it by coordinates instead of by ref",
));
};
let open_capable = names.iter().any(|a| a == AX_OPEN);
let press_capable = names.iter().any(|a| a == AX_PRESS);
if !open_capable && !press_capable {
let available = action_list(&names);
return Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
format!(
"element has no press/open action; available actions: {available} — click it by \
coordinates instead"
),
));
}
for (advertised, action, label) in [
(press_capable, AX_PRESS, "press"),
(open_capable, AX_OPEN, "open"),
] {
if !advertised {
continue;
}
let err = unsafe { element.perform_action(&axc(action)) };
if !unsupported_action(err) {
return map_action_err(err, label);
}
}
if open_capable {
if open_pointed_at(element) {
return Ok(Impact::DoneWith(
"the element refused its own action, so the item it points at was opened"
.to_string(),
));
}
if let Some((x, y)) = center {
ensure(Grant::Input)?;
bring_window_forward(window, surface, budget)?;
click(CGPoint::new(x, y), MouseButton::Left, true, &[])?;
return Ok(Impact::DoneWith(
"the element refused its own action, so it was double clicked at its position"
.to_string(),
));
}
}
let available = action_list(&names);
Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
format!(
"the element advertises {available} but the accessibility API refused it — click it \
by coordinates instead"
),
))
}
fn action_list(names: &[String]) -> String {
names
.iter()
.map(|a| normalize_action_name(a))
.collect::<Vec<_>>()
.join(", ")
}
fn unsupported_action(err: AXError) -> bool {
matches!(
err,
AXError::AttributeUnsupported | AXError::ActionUnsupported
)
}
fn open_pointed_at(element: &AXUIElement) -> bool {
let Some(value) = attribute_value(element, &axc(AX_URL)).ok() else {
return false;
};
let text = if let Some(url) = value.downcast_ref::<CFURL>() {
url.string().to_string()
} else if let Some(text) = value.downcast_ref::<CFString>() {
text.to_string()
} else {
return false;
};
NSURL::URLWithString(&NSString::from_str(&text))
.is_some_and(|url| NSWorkspace::sharedWorkspace().openURL(&url))
}
fn set_element_value(element: &AXUIElement, text: &str) -> Result<Impact, anyhow::Error> {
let role = attribute_name(element, &axc(AX_ROLE)).unwrap_or_default();
let subrole = attribute_name(element, &axc(AX_SUBROLE));
if is_secure_field(&role, subrole.as_deref()) {
return Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
"element is a secure text field — SetValue refused (use click by ref then type)",
));
}
let cf: CFRetained<CFType> = CFRetained::from(CFString::from_str(text));
let err = unsafe { element.set_attribute_value(&axc(AX_VALUE), &cf) };
if unsupported_action(err) {
return Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
"set value — element does not support SetValue; use click by ref then type without ref",
));
}
map_action_err(err, "set value")
}
fn act_on_element_blocking(
target: &TargetSpec,
locator: &Locator,
act: &ElementAct,
) -> Result<Impact, anyhow::Error> {
let mut budget = core::Budget::new(core::ACT_BUDGET, "element action", target.describe());
let (window, app) = resolve_window(target, &budget)?;
budget.check()?;
let app_name = resolved_app_name(target, &app)?;
let title = attribute_text(&window, &axc(AX_TITLE), core::MAX_NAME_CHARS);
let label = core::name_surface(&mut budget, &app_name, title.as_deref());
let origin =
window_origin(&window).map_err(|_| core::unanswered(&label, "for its position"))?;
let (root, handles, _truncation) = build_tree(&window, origin, &budget)?;
let matched = core::resolve_locator_checked(&root, locator)?;
let element = &handles[core::pre_order_index(&root, matched)];
let center = matched
.frame
.map(|(x, y, w, h)| (origin.0 + x + w / 2.0, origin.1 + y + h / 2.0));
match act {
ElementAct::Press => press_element(element, &window, center, &label, &budget),
ElementAct::SetValue { text } => set_element_value(element, text),
}
}
fn map_action_err(err: AXError, ctx: &str) -> Result<Impact, anyhow::Error> {
match err {
AXError::Success => Ok(Impact::Done),
AXError::InvalidUIElement => Err(core::taxonomy_error(
core::ERR_STALE_ELEMENT,
format!("{ctx} — window closed or changed — re-enumerate"),
)),
AXError::APIDisabled => Err(permission_error(Grant::Accessibility)),
AXError::CannotComplete => Ok(Impact::Unconfirmed(format!(
"{ctx} — the application did not confirm the accessibility action, which may already \
have been performed"
))),
_ => Err(anyhow!("AX error {err:?} while {ctx}")),
}
}
fn session_flag(dict: &CFDictionary<CFString, CFType>, key: &str) -> Option<bool> {
let key = CFString::from_str(key);
let value = dict.get(&key)?;
Some(value.downcast_ref::<CFBoolean>()?.value())
}
fn desktop_unavailable_blocking() -> Option<String> {
let Some(dict) = objc2_core_graphics::CGSessionCopyCurrentDictionary() else {
return Some(
"this process has no window-server session — there is no desktop to read or drive"
.to_string(),
);
};
let dict: CFRetained<CFDictionary<CFString, CFType>> =
unsafe { CFRetained::cast_unchecked(dict) };
if session_flag(&dict, "kCGSSessionOnConsoleKey") == Some(false) {
return Some(
"another logged-in user is on this Mac's screen — this session is not on the console"
.to_string(),
);
}
if session_flag(&dict, "CGSSessionScreenIsLocked") == Some(true) {
return Some("the screen is locked".to_string());
}
None
}
#[async_trait::async_trait]
impl Backend for MacOsBackend {
fn is_supported(&self) -> bool {
true
}
async fn desktop_unavailable(&self) -> Option<String> {
with_platform("read the session state", || {
Ok(desktop_unavailable_blocking())
})
.ok()
.flatten()
}
async fn capture_available(&self) -> bool {
with_platform("probe capture", || Ok(capture_available_blocking())).unwrap_or_else(|e| {
tracing::warn!("computer use: {e:#}");
false
})
}
fn capture_unavailable_error(&self) -> anyhow::Error {
catching_foreign("diagnose capture", || {
Ok(if sck_available() {
permission_error(Grant::ScreenRecording)
} else {
screen_capture_needs_14()
})
})
.unwrap_or_else(std::convert::identity)
}
async fn list_apps(&self) -> Result<Vec<AppInfo>, anyhow::Error> {
with_platform("list applications", || {
let mut out = Vec::new();
for app in NSWorkspace::sharedWorkspace().runningApplications() {
if app.activationPolicy() != NSApplicationActivationPolicy::Regular {
continue;
}
let name = app.localizedName().map_or(String::new(), |s| s.to_string());
if !name.is_empty() {
out.push(AppInfo {
pid: Some(app.processIdentifier() as u32),
name,
});
}
}
out.sort_by(|a, b| a.name.cmp(&b.name));
out.dedup_by(|a, b| a.pid == b.pid);
Ok(out)
})
}
async fn list_windows(&self, app: Option<&AppInfo>) -> Result<WindowListing, anyhow::Error> {
with_platform("list windows", || {
ensure(Grant::Accessibility)?;
let app_infos: Vec<AppInfo> = if let Some(a) = app {
vec![a.clone()]
} else {
let mut v = Vec::new();
for run in NSWorkspace::sharedWorkspace().runningApplications() {
if run.activationPolicy() != NSApplicationActivationPolicy::Regular {
continue;
}
let name = run.localizedName().map_or(String::new(), |s| s.to_string());
if !name.is_empty() {
v.push(AppInfo {
pid: Some(run.processIdentifier() as u32),
name,
});
}
}
v
};
let mut listing = WindowListing::default();
let owner = core::agent_key();
let budget = core::Budget::new(
core::LIST_BUDGET,
"window listing",
app.map_or_else(
|| "the running applications".to_string(),
|a| core::windows_label(&a.name),
),
);
for ai in app_infos {
let Some(pid) = ai.pid else { continue };
let name = ai.name;
if let Err(e) = budget.check() {
listing.unread.push(UnreadWindows {
app_name: name,
reason: format!("{e:#}"),
});
continue;
}
let owner = owner.clone();
let app_name = name.clone();
let app_budget = budget.slice(core::APP_SLICE, core::windows_label(&name));
let share = app_budget.clone();
match on_ax_thread(pid, move || windows_of_app(pid, &name, &owner, &app_budget)) {
Ok(rows) => listing.windows.extend(rows),
Err(e) => listing.unread.push(UnreadWindows {
app_name,
reason: core::unread_reason(&e, &share),
}),
}
}
Ok(listing)
})
.map_err(degraded_read)
}
async fn focused_window(&self) -> Result<WindowInfo, anyhow::Error> {
with_platform("read the focused window", || {
ensure(Grant::Accessibility)?;
let owner = core::agent_key();
let budget = core::Budget::new(
core::LIST_BUDGET,
"focused-window search",
"the running applications",
);
with_target(&TargetSpec::Focused, move |target| {
let (window, app) = resolve_window(target, &budget)?;
budget.check()?;
let pid = app_pid(&app)?;
let app_name = resolved_app_name(target, &app)?;
let title = window_title(&window);
let label = resolved_surface_label(target, &app, title.as_deref())?;
Ok(WindowInfo {
app_pid: Some(pid),
app_name,
title: title.unwrap_or_default(),
key: window_key(&window, &owner),
surface: window_frame(&window, &budget)
.map_err(|_| core::unanswered(&label, "for its frame"))?,
})
})
})
.map_err(degraded_read)
}
async fn surface_geometry(
&self,
target: &TargetSpec,
) -> Result<SurfaceGeometry, anyhow::Error> {
with_platform("read the target's geometry", || {
let budget = core::Budget::new(core::LIST_BUDGET, "surface lookup", target.describe());
surface_geometry_with(target, budget)
})
.map_err(degraded_read)
}
async fn observe(&self, target: &TargetSpec) -> Result<Observation, anyhow::Error> {
with_platform("observe", || {
ensure(Grant::Accessibility)?;
with_target(target, observe_blocking)
})
.map_err(degraded_read)
}
async fn act_on_element(
&self,
target: &TargetSpec,
locator: &Locator,
act: ElementAct,
) -> Result<Impact, anyhow::Error> {
with_platform("act on an element", || {
if matches!(target, TargetSpec::Screen) {
return Err(core::screen_act_error());
}
ensure(Grant::Accessibility)?;
let locator = locator.clone();
match with_target(target, move |target| {
act_on_element_blocking(target, &locator, &act)
}) {
Err(e) if e.downcast_ref::<AxBusy>().is_some() => Ok(Impact::Unconfirmed(format!(
"{e}; the request may still be applied once that thread catches up"
))),
Err(e) if core::is_framework_failed(&e) => Ok(Impact::Unconfirmed(format!(
"the system raised an exception while the element action was running, so it \
may already have been performed — {e}"
))),
other => other,
}
})
}
async fn raw_input(
&self,
target: &TargetSpec,
input: RawInput,
) -> Result<Impact, anyhow::Error> {
match with_platform("synthesize input", || {
check_input_arguments(&input)?;
ensure(Grant::Input)?;
raw_input_blocking(target, &input)
}) {
Err(e) if e.downcast_ref::<AxBusy>().is_some() => Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!("{e} — nothing was sent to the window; retry the call"),
)),
Err(e) if core::is_framework_failed(&e) => Ok(Impact::Unconfirmed(format!(
"the system raised an exception while the input was being synthesized, so part of \
it may already have been delivered — {e}"
))),
other => other,
}
}
async fn cursor_position(&self) -> Result<(f64, f64), anyhow::Error> {
with_platform("read the pointer position", || {
let event =
CGEvent::new(None).ok_or_else(|| anyhow!("failed to create dummy cursor event"))?;
let p = CGEvent::location(Some(&event));
Ok((p.x, p.y))
})
}
async fn capture(
&self,
target: &TargetSpec,
budget: &CaptureBudget,
) -> Result<Capture, anyhow::Error> {
with_platform("capture", || capture_blocking(target, budget)).map_err(degraded_read)
}
}
fn window_title(window: &AXUIElement) -> Option<String> {
attribute_name(window, &axc(AX_TITLE))
}
#[cfg(test)]
mod tests {
use super::*;
use objc2_core_graphics::{CGBitmapInfo, CGColorRenderingIntent, CGDataProvider};
#[test]
fn a_framework_exception_becomes_a_failed_call() {
let err = with_platform("probe", || -> Result<(), anyhow::Error> {
let array = NSArray::<objc2::runtime::NSObject>::new();
let _: *mut objc2::runtime::AnyObject =
unsafe { objc2::msg_send![&array, objectAtIndex: 0usize] };
Ok(())
})
.expect_err("the exception must not escape the firewall");
let text = format!("{err:#}");
assert!(
text.starts_with(core::ERR_FRAMEWORK_FAILED),
"expected a framework-failed call, got: {text}"
);
assert!(text.contains("exception"), "{text}");
}
#[test]
fn typed_members_skip_a_member_of_another_type() {
let name = CFString::from_str("AXPress");
let number = objc2_core_foundation::CFNumber::new_i64(7);
let array = CFArray::<CFType>::from_objects(&[&name, &number]);
let names: Vec<String> = typed_members::<CFString>(array.as_ref())
.iter()
.map(ToString::to_string)
.collect();
assert_eq!(names, vec!["AXPress".to_string()]);
}
#[test]
fn a_capture_keeps_the_source_row_and_column_order() {
let corners: [u8; 16] = [
255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, 255, 255, 255, 255, ];
let provider = unsafe {
CGDataProvider::with_data(
std::ptr::null_mut(),
corners.as_ptr().cast(),
corners.len(),
None,
)
}
.expect("a data provider over the fixture pixels");
let space = CGColorSpace::new_device_rgb().expect("a device RGB space");
let info = CGBitmapInfo(
CGImageAlphaInfo::PremultipliedLast.0 | CGImageByteOrderInfo::Order32Big.0,
);
let image = unsafe {
CGImage::new(
2,
2,
8,
32,
8,
Some(&space),
info,
Some(&provider),
std::ptr::null(),
false,
CGColorRenderingIntent::RenderingIntentDefault,
)
}
.expect("a 2x2 fixture image");
let capture = capture_image_to_rgba(&image, &core::CaptureBudget::default())
.expect("the fixture converts");
assert_eq!(
&capture.rgba[..],
&corners[..],
"corners: top-left red, top-right green, bottom-left blue, bottom-right white"
);
}
#[test]
fn key_table_covers_chord_parser_keys() {
for key in [
"a",
"z",
"5",
"0",
"return",
"space",
"backslash",
"f1",
"f12",
"capslock",
] {
let (mods, parsed) = core::parse_key_chord(key).unwrap();
assert!(mods.is_empty());
assert_eq!(parsed, key.to_lowercase(), "chord parser mangles '{key}'");
assert!(key_code(key).is_some(), "kVK table missing '{key}'");
}
let (mods, parsed) = core::parse_key_chord("cmd+shift+t").unwrap();
assert_eq!(mods, vec![Modifier::Cmd, Modifier::Shift]);
assert_eq!(parsed, "t");
}
}