use accessibility_sys::{
AXObserverRef, AXUIElementCreateApplication, AXUIElementCreateSystemWide, AXUIElementRef,
AXUIElementSetMessagingTimeout, kAXErrorSuccess,
};
use bevy::ecs::component::Component;
use core::ptr::NonNull;
use crossbeam_channel::Sender;
use derive_more::{DerefMut, with_trait::Deref};
use mockall::automock;
use objc2_core_foundation::{CFRetained, CFString, kCFRunLoopCommonModes};
use std::ffi::c_void;
use std::pin::Pin;
use std::ptr::null_mut;
use std::sync::{Arc, LazyLock, RwLock};
use std::thread;
use stdext::sync::rw_lock::RwLockExt;
use stdext::function_name;
use tracing::{debug, error};
use super::skylight::_SLPSGetFrontProcess;
use super::{ProcessApi, Window, WindowOS, ax_window_id};
use crate::config::Config;
use crate::errors::{Error, Result};
use crate::events::{DestroySource, Event, EventSender};
use crate::platform::{
AXObserverAddNotification, AXObserverCreate, AXObserverRemoveNotification, CFStringRef, ConnID,
Pid, ProcessSerialNumber, WinID,
};
use crate::util::{AXUIAttributes, AXUIWrapper, MacResult, add_run_loop, remove_run_loop};
const AX_MESSAGING_TIMEOUT_SEC: f32 = 0.25;
pub(crate) fn bound_ax_messaging_timeout() -> Result<()> {
let system_wide = AXUIWrapper::from_retained(unsafe { AXUIElementCreateSystemWide() })?;
unsafe { AXUIElementSetMessagingTimeout(system_wide.as_ptr(), AX_MESSAGING_TIMEOUT_SEC) }
.to_result(function_name!())
}
pub static AX_NOTIFICATIONS: LazyLock<Vec<&str>> = LazyLock::new(|| {
vec![
accessibility_sys::kAXCreatedNotification,
accessibility_sys::kAXFocusedWindowChangedNotification,
accessibility_sys::kAXFocusedUIElementChangedNotification,
accessibility_sys::kAXWindowMovedNotification,
accessibility_sys::kAXWindowResizedNotification,
accessibility_sys::kAXMenuOpenedNotification,
accessibility_sys::kAXMenuClosedNotification,
]
});
pub static AX_WINDOW_NOTIFICATIONS: LazyLock<Vec<&str>> = LazyLock::new(|| {
vec![
accessibility_sys::kAXUIElementDestroyedNotification,
accessibility_sys::kAXWindowMiniaturizedNotification,
accessibility_sys::kAXWindowDeminiaturizedNotification,
accessibility_sys::kAXTitleChangedNotification,
]
});
#[automock]
pub trait ApplicationApi: Send + Sync {
fn pid(&self) -> Pid;
fn psn(&self) -> ProcessSerialNumber;
fn connection(&self) -> Option<ConnID>;
fn focused_window_id(&self) -> Result<WinID>;
fn focused_window(&self, config: &Config) -> Option<Window>;
fn window_list(&self, config: &Config) -> Vec<Window>;
fn ax_window_ids(&self) -> Vec<WinID>;
fn observe(&mut self) -> Result<bool>;
fn observe_window(&mut self, window: &Window) -> Result<bool>;
fn unobserve_window(&mut self, window: &Window);
fn is_frontmost(&self) -> bool;
fn bundle_id(&self) -> Option<String>;
fn name(&self) -> &str;
}
#[derive(Component, Deref, DerefMut)]
pub struct Application(Box<dyn ApplicationApi>);
impl std::fmt::Debug for Application {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "app (pid {})", self.pid())
}
}
impl std::fmt::Display for Application {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "app (pid {})", self.pid())
}
}
impl Application {
pub fn new(app: Box<dyn ApplicationApi>) -> Self {
Application(app)
}
}
pub struct ApplicationOS {
element: CFRetained<AXUIWrapper>,
psn: ProcessSerialNumber,
pid: Pid,
connection: Option<ConnID>,
handler: AxObserverHandler,
bundle_id: Option<String>,
name: String,
}
impl Drop for ApplicationOS {
fn drop(&mut self) {
self.handler
.remove_observer(&ObserverType::Application, &self.element, &AX_NOTIFICATIONS);
}
}
impl std::fmt::Display for ApplicationOS {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "app '{}' (pid {})", self.name, self.pid)
}
}
impl ApplicationOS {
pub fn new(
connection: Option<ConnID>,
process: &dyn ProcessApi,
events: &EventSender,
resolver: Sender<PendingNotification>,
) -> Result<Self> {
let refer = unsafe {
let ptr = AXUIElementCreateApplication(process.pid());
AXUIElementSetMessagingTimeout(ptr, AX_MESSAGING_TIMEOUT_SEC);
AXUIWrapper::from_retained(ptr)?
};
let bundle_id = process
.application()
.as_ref()
.and_then(|app| app.bundleIdentifier())
.map(|id| id.to_string());
Ok(Self {
element: refer,
psn: process.psn(),
pid: process.pid(),
connection,
handler: AxObserverHandler::new(process.pid(), events.clone(), resolver)?,
bundle_id,
name: process.name().to_string(),
})
}
}
impl ApplicationApi for ApplicationOS {
fn pid(&self) -> Pid {
self.pid
}
fn psn(&self) -> ProcessSerialNumber {
self.psn
}
fn connection(&self) -> Option<ConnID> {
self.connection
}
fn focused_window_id(&self) -> Result<WinID> {
self.element.focused_window_id()
}
fn focused_window(&self, config: &Config) -> Option<Window> {
let bundle_id = self.bundle_id.as_deref();
self.element
.focused_window_element()
.ok()
.and_then(|element| {
WindowOS::new_with_config(&element, config, bundle_id)
.ok()
.map(|window| Window::new(Box::new(window)))
})
}
fn window_list(&self, config: &Config) -> Vec<Window> {
let bundle_id = self.bundle_id.as_deref();
self.element
.windows()
.map(|windows| {
windows
.into_iter()
.flat_map(|element| {
WindowOS::new_with_config(&element, config, bundle_id)
.map(|window| Window::new(Box::new(window)))
})
.collect()
})
.unwrap_or_default()
}
fn ax_window_ids(&self) -> Vec<WinID> {
self.element
.windows()
.map(|windows| {
windows
.iter()
.filter_map(|element| ax_window_id(element.as_ptr()).ok())
.collect()
})
.unwrap_or_default()
}
fn observe(&mut self) -> Result<bool> {
self.handler
.add_observer(&self.element, &AX_NOTIFICATIONS, ObserverType::Application)
.map(|retry| retry.is_empty())
}
fn observe_window(&mut self, window: &Window) -> Result<bool> {
if let Some(element) = window.element() {
self.handler
.add_observer(
&element,
&AX_WINDOW_NOTIFICATIONS,
ObserverType::Window(window.id()),
)
.map(|retry| retry.is_empty())
} else {
Err(Error::InvalidWindow)
}
}
fn unobserve_window(&mut self, window: &Window) {
if let Some(element) = window.element() {
self.handler.remove_observer(
&ObserverType::Window(window.id()),
&element,
&AX_WINDOW_NOTIFICATIONS,
);
}
}
fn is_frontmost(&self) -> bool {
let mut psn = ProcessSerialNumber::default();
unsafe { _SLPSGetFrontProcess(&mut psn) }
.to_result(function_name!())
.is_ok()
&& self.psn == psn
}
fn bundle_id(&self) -> Option<String> {
self.bundle_id.clone()
}
fn name(&self) -> &str {
&self.name
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
enum ObserverType {
Application,
Window(WinID),
}
#[derive(Clone, Copy, Debug)]
enum WindowNotification {
Focused,
Moved,
Resized,
MenuOpened,
MenuClosed,
}
impl WindowNotification {
fn from_name(notification: &str) -> Option<Self> {
match notification {
accessibility_sys::kAXFocusedWindowChangedNotification
| accessibility_sys::kAXFocusedUIElementChangedNotification => Some(Self::Focused),
accessibility_sys::kAXWindowMovedNotification => Some(Self::Moved),
accessibility_sys::kAXWindowResizedNotification => Some(Self::Resized),
accessibility_sys::kAXMenuOpenedNotification => Some(Self::MenuOpened),
accessibility_sys::kAXMenuClosedNotification => Some(Self::MenuClosed),
_ => None,
}
}
fn into_event(self, window_id: WinID) -> Event {
match self {
Self::Focused => Event::WindowFocused { window_id },
Self::Moved => Event::WindowMoved { window_id },
Self::Resized => Event::WindowResized { window_id },
Self::MenuOpened => Event::MenuOpened { window_id },
Self::MenuClosed => Event::MenuClosed { window_id },
}
}
}
pub(crate) struct PendingNotification {
kind: WindowNotification,
element: CFRetained<AXUIWrapper>,
}
#[derive(Clone)]
pub(crate) struct WindowResolver {
sender: Sender<PendingNotification>,
}
impl WindowResolver {
pub(crate) fn new(events: EventSender) -> Self {
let (tx, rx) = crossbeam_channel::unbounded::<PendingNotification>();
thread::Builder::new()
.name("ax-window-resolver".into())
.spawn(move || {
for pending in rx {
match ax_window_id(pending.element.as_ptr()) {
Ok(window_id) => {
_ = events.send(pending.kind.into_event(window_id));
}
Err(err) => debug!("naming window for {:?}: {err}", pending.kind),
}
}
})
.expect("spawning the AX window resolver should succeed");
Self { sender: tx }
}
pub(crate) fn sender(&self) -> Sender<PendingNotification> {
self.sender.clone()
}
}
struct ObserverContext {
events: EventSender,
resolver: Sender<PendingNotification>,
which: ObserverType,
}
impl ObserverContext {
fn notify(&self, notification: &str, element: AXUIElementRef) {
match self.which {
ObserverType::Application => self.notify_app(notification, element),
ObserverType::Window(id) => self.notify_window(notification, id),
}
}
fn notify_app(&self, notification: &str, element: AXUIElementRef) {
if notification == accessibility_sys::kAXCreatedNotification {
let Ok(element) = AXUIWrapper::retain(element).inspect_err(|err| {
error!("invalid element {element:?}: {err}");
}) else {
return;
};
_ = self.events.send(Event::WindowCreated { element });
return;
}
let Some(kind) = WindowNotification::from_name(notification) else {
error!("unhandled application notification: {notification:?}");
return;
};
let Ok(element) = AXUIWrapper::retain(element).inspect_err(|err| {
error!("invalid element {element:?}: {err}");
}) else {
return;
};
_ = self.resolver.send(PendingNotification { kind, element });
}
fn notify_window(&self, notification: &str, window_id: WinID) {
let event = match notification {
accessibility_sys::kAXWindowMiniaturizedNotification => {
Event::WindowMinimized { window_id }
}
accessibility_sys::kAXWindowDeminiaturizedNotification => {
Event::WindowDeminimized { window_id }
}
accessibility_sys::kAXUIElementDestroyedNotification => Event::WindowDestroyed {
window_id,
source: DestroySource::Accessibility,
},
accessibility_sys::kAXTitleChangedNotification => {
Event::WindowTitleChanged { window_id }
}
_ => {
error!("unhandled window notification: {notification:?}");
return;
}
};
_ = self.events.send(event);
}
}
struct AxObserverHandler {
observer: CFRetained<AXUIWrapper>,
events: EventSender,
resolver: Sender<PendingNotification>,
contexts: Arc<RwLock<Vec<Pin<Box<ObserverContext>>>>>,
}
impl Drop for AxObserverHandler {
fn drop(&mut self) {
remove_run_loop(&self.observer);
}
}
impl AxObserverHandler {
fn new(pid: Pid, events: EventSender, resolver: Sender<PendingNotification>) -> Result<Self> {
let observer = unsafe {
let mut observer_ref: AXObserverRef = null_mut();
if kAXErrorSuccess == AXObserverCreate(pid, Self::callback, &mut observer_ref) {
AXUIWrapper::from_retained(observer_ref)?
} else {
return Err(Error::PermissionDenied(format!(
"{}: error creating observer.",
function_name!()
)));
}
};
unsafe { add_run_loop(&observer, kCFRunLoopCommonModes)? };
Ok(Self {
observer,
events,
resolver,
contexts: Arc::new(RwLock::new(Vec::new())),
})
}
pub fn add_observer(
&mut self,
element: &AXUIWrapper,
notifications: &[&'static str],
which: ObserverType,
) -> Result<Vec<&str>> {
let observer: AXObserverRef = self.observer.as_ptr();
let context_ptr = self.get_or_insert_context(which).as_ptr();
let mut retry = vec![];
let added = notifications
.iter()
.filter_map(|name| {
debug!("adding {name} {element:x?} {observer:?}");
let notification = CFString::from_static_str(name);
match unsafe {
AXObserverAddNotification(
observer,
element.as_ptr(),
¬ification,
context_ptr.cast(),
)
} {
accessibility_sys::kAXErrorSuccess
| accessibility_sys::kAXErrorNotificationAlreadyRegistered => Some(*name),
accessibility_sys::kAXErrorCannotComplete => {
retry.push(*name);
None
}
result => {
error!("error adding {name} {element:x?} {observer:?}: {result}");
None
}
}
})
.collect::<Vec<_>>();
if added.is_empty() {
Err(Error::PermissionDenied(format!(
"{}: unable to register any observers!",
function_name!()
)))
} else {
Ok(retry)
}
}
pub fn remove_observer(
&mut self,
which: &ObserverType,
element: &AXUIWrapper,
notifications: &[&'static str],
) {
if self.get_context(*which).is_none() {
debug!("{which:?} ({element}) already un-observed, skipping!");
return;
}
for name in notifications {
let observer: AXObserverRef = self.observer.deref().as_ptr();
let notification = CFString::from_static_str(name);
debug!("removing {name} {element:x?} {observer:?}");
let result =
unsafe { AXObserverRemoveNotification(observer, element.as_ptr(), ¬ification) };
if result != kAXErrorSuccess {
debug!("error removing {name} {element:x?} {observer:?}: {result}");
}
}
}
extern "C" fn callback(
_observer: AXObserverRef,
element: AXUIElementRef,
notification: CFStringRef,
context: *mut c_void,
) {
let notification = NonNull::new(notification.cast_mut())
.map(|ptr| unsafe { ptr.as_ref() })
.map(CFString::to_string);
let context =
NonNull::new(context.cast::<ObserverContext>()).map(|ptr| unsafe { ptr.as_ref() });
let Some((notification, context)) = notification.zip(context) else {
return;
};
context.notify(¬ification, element);
}
fn get_context(&self, which: ObserverType) -> Option<NonNull<ObserverContext>> {
let contexts = self.contexts.force_read();
contexts
.iter()
.find(|context| context.which == which)
.map(|context| NonNull::from_ref(context.as_ref().get_ref()))
}
fn get_or_insert_context(
&self,
which: ObserverType,
) -> NonNull<ObserverContext> {
if let Some(context) = self.get_context(which) {
return context;
}
self.contexts.force_write().push(Box::pin(ObserverContext {
events: self.events.clone(),
resolver: self.resolver.clone(),
which,
}));
self.get_context(which)
.expect("inserted observer context must be present")
}
}
#[cfg(test)]
mod tests {
use std::{
mem::ManuallyDrop,
ptr::NonNull,
sync::{Arc, RwLock},
};
use stdext::sync::rw_lock::RwLockExt as _;
use super::ObserverType;
use crate::{events::EventSender, manager::app::AxObserverHandler, util::AXUIWrapper};
#[test]
fn same_observer_type_reuses_stable_context_pointer() {
let fake_ptr = NonNull::<AXUIWrapper>::dangling().as_ptr();
let observer = AXUIWrapper::from_retained(fake_ptr).unwrap();
let (events, _receiver) = EventSender::new();
let (resolver, _rx) = crossbeam_channel::unbounded();
let handler = ManuallyDrop::new(AxObserverHandler {
observer,
events,
resolver,
contexts: Arc::new(RwLock::new(Vec::new())),
});
let first = handler.get_or_insert_context(ObserverType::Window(42));
let reused = handler.get_or_insert_context(ObserverType::Window(42));
assert_eq!(reused, first);
assert_eq!(handler.contexts.as_ref().force_read().len(), 1);
}
}