1use std::ops::Deref;
2use std::sync::atomic::{AtomicU64, Ordering};
3use std::sync::{Arc, Mutex};
4use std::{
5 fs,
6 io::{Read, Write},
7 marker::PhantomData,
8 path::Path,
9};
10
11use crate::proto::context_session_command::Command as ContextCommand;
12use crate::proto::context_session_event::Event as ContextEvent;
13use crate::proto::hook_completed_event::Result as HookCompletionResult;
14use crate::proto::register_hook_command::Hook as RegisterHook;
15use crate::proto::surface_session_command::Command as SurfaceCommand;
16use crate::proto::surface_session_event::Event as SurfaceEvent;
17use crate::proto::{
18 AccessibilitySnapshotCommand, AppLaunchedEvent, CaptureCommand, ClickElementCommand,
19 CloseDesktopWindowCommand, ConnectDesktopCommand, ConnectMobileCommand, ContextSessionCommand,
20 CountElementsCommand, DesktopAppLaunchedEvent, DesktopConnectedEvent,
21 DesktopPlatform as ProtoDesktopPlatform, DesktopWindowInfo, FillElementCommand,
22 FocusElementCommand, GetInnerTextCommand, GetTextContentCommand, LaunchAppCommand,
23 LaunchDesktopAppCommand, ListDesktopWindowsCommand, MobileConnectedEvent,
24 MobilePlatform as ProtoMobilePlatform, NavigatePageCommand, PressKeyCommand,
25 ReadFileChunkCommand, RegisterDesktopWindowHook, RegisterHookCommand,
26 RegisterMobileDownloadHook, RegisterMobileFileChooserHook, SaveMobileDownloadCommand,
27 ScreenshotCommand, SetMobileFileChooserFilesCommand, SurfaceSessionCommand,
28 UploadFileChunkCommand, WaitForHookCommand, WaitForSelectorCommand,
29};
30
31use super::hook::{DownloadHook, FileChooserHook};
32
33static MOBILE_TRANSFER_COUNTER: AtomicU64 = AtomicU64::new(1);
34
35pub trait AndroidHookType: private::Sealed + Clone + Send + Sync + 'static {
36 type Output;
37}
38
39mod private {
40 use super::*;
41 pub trait Sealed {
42 fn name() -> &'static str;
43 fn decode(
44 app: &NativeApp,
45 result: HookCompletionResult,
46 ) -> Result<<Self as AndroidHookType>::Output>
47 where
48 Self: AndroidHookType;
49 }
50}
51
52impl AndroidHookType for FileChooserHook {
53 type Output = AndroidFileChooser;
54}
55impl private::Sealed for FileChooserHook {
56 fn name() -> &'static str {
57 "file_chooser"
58 }
59 fn decode(app: &NativeApp, result: HookCompletionResult) -> Result<AndroidFileChooser> {
60 let HookCompletionResult::MobileFileChooser(result) = result else {
61 return Err(Error::new(
62 "Android file chooser hook returned an invalid result",
63 ));
64 };
65 Ok(AndroidFileChooser {
66 app: app.clone(),
67 id: result.file_chooser_id,
68 is_multiple: result.is_multiple,
69 })
70 }
71}
72impl AndroidHookType for DownloadHook {
73 type Output = AndroidDownload;
74}
75impl private::Sealed for DownloadHook {
76 fn name() -> &'static str {
77 "download"
78 }
79 fn decode(app: &NativeApp, result: HookCompletionResult) -> Result<AndroidDownload> {
80 let HookCompletionResult::MobileDownload(result) = result else {
81 return Err(Error::new(
82 "Android download hook returned an invalid result",
83 ));
84 };
85 Ok(AndroidDownload {
86 app: app.clone(),
87 id: result.download_id,
88 suggested_filename: result.suggested_filename,
89 })
90 }
91}
92
93pub struct AndroidHook<T: AndroidHookType> {
94 app: NativeApp,
95 id: String,
96 _type: PhantomData<T>,
97}
98
99#[derive(Clone)]
100pub struct AndroidFileChooser {
101 app: NativeApp,
102 id: String,
103 is_multiple: bool,
104}
105
106#[derive(Clone)]
107pub struct AndroidDownload {
108 app: NativeApp,
109 id: String,
110 suggested_filename: String,
111}
112use tokio::sync::{Mutex as AsyncMutex, mpsc};
113use tokio_stream::wrappers::ReceiverStream;
114
115use super::command::command_retry_options;
116use super::runtime::get_runtime;
117use super::types::{
118 AccessibilitySnapshotFormat, AccessibilitySnapshotMode, AccessibilitySnapshotOptions,
119 BoundingBox, CommandOptions, Error, PressOptions, Result, RuntimeClient, ScreenshotOptions,
120 TextResult, WaitForSelectorOptions,
121};
122use super::web_locators::{RoleOptions, TextMatcher, TextOptions};
123
124#[derive(Debug, Clone, Default)]
125pub struct MobileAndroidConnectOptions {
126 pub device: Option<String>,
127 pub adb_endpoint: Option<String>,
128 pub preserve_app_state: bool,
129 pub timeout_ms: Option<u32>,
130}
131
132#[derive(Debug, Clone, Default)]
133pub struct MobileAndroidLaunchOptions {
134 pub apk_path: Option<String>,
135 pub app_id: Option<String>,
136 pub launch_activity: Option<String>,
137 pub stop_before_launch: bool,
138 pub timeout_ms: Option<u32>,
139}
140
141#[derive(Debug, Clone, Default)]
142pub struct MobileIosConnectOptions {
143 pub device: Option<String>,
144 pub agent_endpoint: Option<String>,
145 pub preserve_app_state: bool,
146 pub timeout_ms: Option<u32>,
147}
148
149#[derive(Debug, Clone, Default)]
150pub struct MobileIosLaunchOptions {
151 pub app_path: Option<String>,
152 pub app_id: Option<String>,
153 pub stop_before_launch: bool,
154 pub timeout_ms: Option<u32>,
155}
156
157#[derive(Debug, Clone, Default)]
158pub struct DesktopMacConnectOptions {
159 pub agent_endpoint: Option<String>,
160 pub timeout_ms: Option<u32>,
161}
162
163#[derive(Debug, Clone)]
164pub struct DesktopMacLaunchOptions {
165 pub app_id: String,
166 pub terminate_running: bool,
167 pub timeout_ms: Option<u32>,
168}
169
170pub type DesktopWindowsConnectOptions = DesktopMacConnectOptions;
171pub type DesktopWindowsLaunchOptions = DesktopMacLaunchOptions;
172
173#[derive(Clone)]
174pub struct DesktopApp {
175 app: NativeApp,
176}
177
178#[derive(Clone)]
179pub struct DesktopWindow {
180 app: NativeApp,
181 window_id: String,
182 title: String,
183 is_main: bool,
184}
185
186#[derive(Clone, Copy)]
187pub struct WindowHook;
188pub const WINDOW: WindowHook = WindowHook;
189
190pub struct DesktopWindowHook {
191 app: DesktopApp,
192 id: String,
193}
194
195pub type MacApp = DesktopApp;
196pub type MacLocator = NativeLocator;
197pub type WindowsApp = DesktopApp;
198pub type WindowsLocator = NativeLocator;
199
200#[derive(Clone)]
201pub struct MacDesktop {
202 inner: AndroidDevice,
203}
204pub type WindowsDesktop = MacDesktop;
205
206pub type IosApp = NativeApp;
207pub type IosLocator = NativeLocator;
208
209#[derive(Clone)]
210pub struct IosDevice {
211 inner: AndroidDevice,
212}
213
214#[derive(Clone)]
215pub struct NativeLocator {
216 page: NativeApp,
217 selector: String,
218}
219
220#[derive(Clone)]
221pub struct NativeApp {
222 inner: Arc<NativeAppInner>,
223}
224
225pub type AndroidApp = NativeApp;
226pub type AndroidLocator = NativeLocator;
227
228#[derive(Clone)]
229pub struct AndroidDevice {
230 inner: Arc<AndroidDeviceInner>,
231}
232
233struct AndroidDeviceInner {
234 runtime: Arc<RuntimeClient>,
235 state: AsyncMutex<AndroidDeviceState>,
236 session_id: String,
237 initial_app: NativeApp,
238 current_app: Mutex<NativeApp>,
239}
240
241struct AndroidDeviceState {
242 command_tx: mpsc::Sender<SurfaceSessionCommand>,
243 events: tonic::Streaming<crate::proto::SurfaceSessionEvent>,
244 closed: bool,
245}
246
247struct NativeAppInner {
248 runtime: Arc<RuntimeClient>,
249 surface_session_id: String,
250 session_id: String,
251 state: AsyncMutex<NativeAppState>,
252}
253
254#[derive(Default)]
255struct NativeAppState {
256 handle: Option<AndroidTabHandle>,
257}
258
259struct AndroidTabHandle {
260 command_tx: mpsc::Sender<crate::proto::ContextSessionCommand>,
261 events: tonic::Streaming<crate::proto::ContextSessionEvent>,
262 closed: bool,
263}
264
265pub mod android {
266 use super::*;
267
268 pub async fn connect(options: MobileAndroidConnectOptions) -> Result<AndroidDevice> {
269 let runtime = get_runtime().await?;
270 let mut engine = runtime.engine.clone();
271 let (command_tx, command_rx) = mpsc::channel(16);
272 let response = engine
273 .surface_session(tonic::Request::new(ReceiverStream::new(command_rx)))
274 .await?;
275 let mut events = response.into_inner();
276
277 command_tx
278 .send(SurfaceSessionCommand {
279 command: Some(SurfaceCommand::ConnectMobile(ConnectMobileCommand {
280 platform: ProtoMobilePlatform::Android as i32,
281 device: options.device,
282 adb_endpoint: options.adb_endpoint,
283 preserve_app_state: options.preserve_app_state,
284 retry_options: command_retry_options(options.timeout_ms),
285 })),
286 })
287 .await
288 .map_err(|_| Error::new("failed to send ConnectMobileCommand"))?;
289
290 loop {
291 let event = events.message().await?.ok_or_else(|| {
292 Error::new("surface session closed before mobile connect response")
293 })?;
294
295 match event.event {
296 Some(SurfaceEvent::MobileConnected(MobileConnectedEvent {
297 initial_app_session_id,
298 device_session_id,
299 ..
300 })) => {
301 let initial_app = NativeApp {
302 inner: Arc::new(NativeAppInner {
303 runtime: Arc::clone(&runtime),
304 surface_session_id: event.session_id.clone(),
305 session_id: initial_app_session_id,
306 state: AsyncMutex::new(NativeAppState::default()),
307 }),
308 };
309 return Ok(AndroidDevice {
310 inner: Arc::new(AndroidDeviceInner {
311 runtime,
312 state: AsyncMutex::new(AndroidDeviceState {
313 command_tx,
314 events,
315 closed: false,
316 }),
317 session_id: if device_session_id.is_empty() {
318 event.session_id
319 } else {
320 device_session_id
321 },
322 initial_app: initial_app.clone(),
323 current_app: Mutex::new(initial_app),
324 }),
325 });
326 }
327 Some(SurfaceEvent::Error(error)) => {
328 return Err(Error::new(format!(
329 "surface session error during mobile connect: {}",
330 error.message
331 )));
332 }
333 _ => {}
334 }
335 }
336 }
337}
338
339pub mod ios {
340 use super::*;
341
342 pub async fn connect(options: MobileIosConnectOptions) -> Result<IosDevice> {
343 let runtime = get_runtime().await?;
344 let mut engine = runtime.engine.clone();
345 let (command_tx, command_rx) = mpsc::channel(16);
346 let response = engine
347 .surface_session(tonic::Request::new(ReceiverStream::new(command_rx)))
348 .await?;
349 let mut events = response.into_inner();
350
351 command_tx
352 .send(SurfaceSessionCommand {
353 command: Some(SurfaceCommand::ConnectMobile(ConnectMobileCommand {
354 platform: ProtoMobilePlatform::Ios as i32,
355 device: options.device,
356 adb_endpoint: options.agent_endpoint,
359 preserve_app_state: options.preserve_app_state,
360 retry_options: command_retry_options(options.timeout_ms),
361 })),
362 })
363 .await
364 .map_err(|_| Error::new("failed to send iOS ConnectMobileCommand"))?;
365
366 loop {
367 let event = events
368 .message()
369 .await?
370 .ok_or_else(|| Error::new("surface session closed before iOS connect response"))?;
371 match event.event {
372 Some(SurfaceEvent::MobileConnected(MobileConnectedEvent {
373 initial_app_session_id,
374 device_session_id,
375 ..
376 })) => {
377 let initial_app = NativeApp {
378 inner: Arc::new(NativeAppInner {
379 runtime: Arc::clone(&runtime),
380 surface_session_id: event.session_id.clone(),
381 session_id: initial_app_session_id,
382 state: AsyncMutex::new(NativeAppState::default()),
383 }),
384 };
385 return Ok(IosDevice {
386 inner: AndroidDevice {
387 inner: Arc::new(AndroidDeviceInner {
388 runtime,
389 state: AsyncMutex::new(AndroidDeviceState {
390 command_tx,
391 events,
392 closed: false,
393 }),
394 session_id: if device_session_id.is_empty() {
395 event.session_id
396 } else {
397 device_session_id
398 },
399 initial_app: initial_app.clone(),
400 current_app: Mutex::new(initial_app),
401 }),
402 },
403 });
404 }
405 Some(SurfaceEvent::Error(error)) => {
406 return Err(Error::new(format!(
407 "surface session error during iOS connect: {}",
408 error.message
409 )));
410 }
411 _ => {}
412 }
413 }
414 }
415}
416
417pub mod desktop {
418 use super::*;
419 pub use super::{
420 DesktopApp, DesktopMacConnectOptions, DesktopMacLaunchOptions, DesktopWindow,
421 DesktopWindowHook, DesktopWindowsConnectOptions, DesktopWindowsLaunchOptions, MacApp,
422 MacDesktop, MacLocator, WINDOW, WindowHook, WindowsApp, WindowsDesktop, WindowsLocator,
423 };
424
425 pub mod mac {
426 use super::*;
427
428 pub async fn connect(options: DesktopMacConnectOptions) -> Result<MacDesktop> {
429 let runtime = get_runtime().await?;
430 let mut engine = runtime.engine.clone();
431 let (command_tx, command_rx) = mpsc::channel(16);
432 let response = engine
433 .surface_session(tonic::Request::new(ReceiverStream::new(command_rx)))
434 .await?;
435 let mut events = response.into_inner();
436 command_tx
437 .send(SurfaceSessionCommand {
438 command: Some(SurfaceCommand::ConnectDesktop(ConnectDesktopCommand {
439 platform: ProtoDesktopPlatform::Mac as i32,
440 agent_endpoint: options.agent_endpoint,
441 retry_options: command_retry_options(options.timeout_ms),
442 })),
443 })
444 .await
445 .map_err(|_| Error::new("failed to send ConnectDesktopCommand"))?;
446 loop {
447 let event = events.message().await?.ok_or_else(|| {
448 Error::new("surface session closed before macOS desktop connect response")
449 })?;
450 match event.event {
451 Some(SurfaceEvent::DesktopConnected(DesktopConnectedEvent {
452 initial_app_session_id,
453 desktop_session_id,
454 ..
455 })) => {
456 let initial_app = NativeApp {
457 inner: Arc::new(NativeAppInner {
458 runtime: Arc::clone(&runtime),
459 surface_session_id: event.session_id.clone(),
460 session_id: initial_app_session_id,
461 state: AsyncMutex::new(NativeAppState::default()),
462 }),
463 };
464 return Ok(MacDesktop {
465 inner: AndroidDevice {
466 inner: Arc::new(AndroidDeviceInner {
467 runtime,
468 state: AsyncMutex::new(AndroidDeviceState {
469 command_tx,
470 events,
471 closed: false,
472 }),
473 session_id: if desktop_session_id.is_empty() {
474 event.session_id
475 } else {
476 desktop_session_id
477 },
478 initial_app: initial_app.clone(),
479 current_app: Mutex::new(initial_app),
480 }),
481 },
482 });
483 }
484 Some(SurfaceEvent::Error(error)) => return Err(Error::new(error.message)),
485 _ => {}
486 }
487 }
488 }
489 }
490
491 pub mod windows {
492 use super::*;
493
494 pub async fn connect(options: DesktopWindowsConnectOptions) -> Result<WindowsDesktop> {
495 let runtime = get_runtime().await?;
496 let mut engine = runtime.engine.clone();
497 let (command_tx, command_rx) = mpsc::channel(16);
498 let response = engine
499 .surface_session(tonic::Request::new(ReceiverStream::new(command_rx)))
500 .await?;
501 let mut events = response.into_inner();
502 command_tx
503 .send(SurfaceSessionCommand {
504 command: Some(SurfaceCommand::ConnectDesktop(ConnectDesktopCommand {
505 platform: ProtoDesktopPlatform::Windows as i32,
506 agent_endpoint: options.agent_endpoint,
507 retry_options: command_retry_options(options.timeout_ms),
508 })),
509 })
510 .await
511 .map_err(|_| Error::new("failed to send ConnectDesktopCommand"))?;
512 loop {
513 let event = events.message().await?.ok_or_else(|| {
514 Error::new("surface session closed before Windows desktop connect response")
515 })?;
516 match event.event {
517 Some(SurfaceEvent::DesktopConnected(DesktopConnectedEvent {
518 initial_app_session_id,
519 desktop_session_id,
520 ..
521 })) => {
522 let initial_app = NativeApp {
523 inner: Arc::new(NativeAppInner {
524 runtime: Arc::clone(&runtime),
525 surface_session_id: event.session_id.clone(),
526 session_id: initial_app_session_id,
527 state: AsyncMutex::new(NativeAppState::default()),
528 }),
529 };
530 return Ok(MacDesktop {
531 inner: AndroidDevice {
532 inner: Arc::new(AndroidDeviceInner {
533 runtime,
534 state: AsyncMutex::new(AndroidDeviceState {
535 command_tx,
536 events,
537 closed: false,
538 }),
539 session_id: if desktop_session_id.is_empty() {
540 event.session_id
541 } else {
542 desktop_session_id
543 },
544 initial_app: initial_app.clone(),
545 current_app: Mutex::new(initial_app),
546 }),
547 },
548 });
549 }
550 Some(SurfaceEvent::Error(error)) => return Err(Error::new(error.message)),
551 _ => {}
552 }
553 }
554 }
555 }
556}
557
558impl MacDesktop {
559 pub fn session_id(&self) -> &str {
560 self.inner.session_id()
561 }
562
563 pub fn app(&self) -> MacApp {
564 DesktopApp {
565 app: self.inner.app(),
566 }
567 }
568
569 pub async fn launch(&self, options: DesktopMacLaunchOptions) -> Result<MacApp> {
570 let mut state = self.inner.inner.state.lock().await;
571 ensure_android_device_open(&state, &self.inner.inner.session_id)?;
572 state
573 .command_tx
574 .send(SurfaceSessionCommand {
575 command: Some(SurfaceCommand::LaunchDesktopApp(LaunchDesktopAppCommand {
576 app_id: options.app_id,
577 terminate_running: options.terminate_running,
578 retry_options: command_retry_options(options.timeout_ms),
579 })),
580 })
581 .await
582 .map_err(|_| Error::new("failed to send LaunchDesktopAppCommand"))?;
583 loop {
584 let event = state.events.message().await?.ok_or_else(|| {
585 Error::new("surface session closed before macOS app launch response")
586 })?;
587 match event.event {
588 Some(SurfaceEvent::DesktopAppLaunched(DesktopAppLaunchedEvent {
589 app_session_id,
590 ..
591 })) => {
592 let app = NativeApp {
593 inner: Arc::new(NativeAppInner {
594 runtime: Arc::clone(&self.inner.inner.runtime),
595 surface_session_id: event.session_id,
596 session_id: app_session_id,
597 state: AsyncMutex::new(NativeAppState::default()),
598 }),
599 };
600 if let Ok(mut current_app) = self.inner.inner.current_app.lock() {
601 *current_app = app.clone();
602 }
603 return Ok(DesktopApp { app });
604 }
605 Some(SurfaceEvent::Error(error)) => return Err(Error::new(error.message)),
606 _ => {}
607 }
608 }
609 }
610}
611
612impl Deref for DesktopApp {
613 type Target = NativeApp;
614 fn deref(&self) -> &Self::Target {
615 &self.app
616 }
617}
618
619impl Deref for DesktopWindow {
620 type Target = NativeApp;
621 fn deref(&self) -> &Self::Target {
622 &self.app
623 }
624}
625
626impl DesktopApp {
627 pub async fn windows(&self) -> Result<Vec<DesktopWindow>> {
628 let mut state = self.app.inner.state.lock().await;
629 let handle = self.app.ensure_handle(&mut state).await?;
630 ensure_android_app_open(handle, &self.app.inner.session_id)?;
631 handle
632 .command_tx
633 .send(ContextSessionCommand {
634 surface_session_id: self.app.inner.surface_session_id.clone(),
635 context_session_id: self.app.inner.session_id.clone(),
636 command: Some(ContextCommand::ListDesktopWindows(
637 ListDesktopWindowsCommand {},
638 )),
639 })
640 .await
641 .map_err(|_| Error::new("failed to send ListDesktopWindowsCommand"))?;
642 loop {
643 let event =
644 handle.events.message().await?.ok_or_else(|| {
645 Error::new("desktop app session closed while listing windows")
646 })?;
647 match event.event {
648 Some(ContextEvent::DesktopWindowsResolved(resolved)) => {
649 return resolved
650 .windows
651 .into_iter()
652 .map(|info| self.window(info))
653 .collect();
654 }
655 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
656 _ => {}
657 }
658 }
659 }
660
661 pub async fn main_window(&self) -> Result<DesktopWindow> {
662 self.windows()
663 .await?
664 .into_iter()
665 .find(|window| window.is_main)
666 .ok_or_else(|| {
667 Error::new("desktop application does not currently expose a main window")
668 })
669 }
670
671 pub async fn register_hook(&self, _hook_type: WindowHook) -> Result<DesktopWindowHook> {
672 let mut state = self.app.inner.state.lock().await;
673 let handle = self.app.ensure_handle(&mut state).await?;
674 ensure_android_app_open(handle, &self.app.inner.session_id)?;
675 handle
676 .command_tx
677 .send(ContextSessionCommand {
678 surface_session_id: self.app.inner.surface_session_id.clone(),
679 context_session_id: self.app.inner.session_id.clone(),
680 command: Some(ContextCommand::RegisterHook(RegisterHookCommand {
681 hook: Some(RegisterHook::DesktopWindow(RegisterDesktopWindowHook {})),
682 })),
683 })
684 .await
685 .map_err(|_| Error::new("failed to send desktop RegisterHookCommand"))?;
686 loop {
687 let event = handle.events.message().await?.ok_or_else(|| {
688 Error::new("desktop app session closed while registering window hook")
689 })?;
690 match event.event {
691 Some(ContextEvent::HookRegistered(event)) => {
692 return Ok(DesktopWindowHook {
693 app: self.clone(),
694 id: event.hook_id,
695 });
696 }
697 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
698 _ => {}
699 }
700 }
701 }
702
703 fn window(&self, info: DesktopWindowInfo) -> Result<DesktopWindow> {
704 if info.context_session_id.is_empty() || info.window_id.is_empty() {
705 return Err(Error::new(
706 "desktop window result is missing its context or window id",
707 ));
708 }
709 Ok(DesktopWindow {
710 app: NativeApp {
711 inner: Arc::new(NativeAppInner {
712 runtime: Arc::clone(&self.app.inner.runtime),
713 surface_session_id: self.app.inner.surface_session_id.clone(),
714 session_id: info.context_session_id,
715 state: AsyncMutex::new(NativeAppState::default()),
716 }),
717 },
718 window_id: info.window_id,
719 title: info.title,
720 is_main: info.is_main,
721 })
722 }
723}
724
725impl DesktopWindow {
726 pub fn window_id(&self) -> &str {
727 &self.window_id
728 }
729 pub fn title(&self) -> &str {
730 &self.title
731 }
732 pub fn is_main(&self) -> bool {
733 self.is_main
734 }
735
736 pub async fn close(&self) -> Result<()> {
737 let mut state = self.app.inner.state.lock().await;
738 let handle = self.app.ensure_handle(&mut state).await?;
739 handle
740 .command_tx
741 .send(ContextSessionCommand {
742 surface_session_id: self.app.inner.surface_session_id.clone(),
743 context_session_id: self.app.inner.session_id.clone(),
744 command: Some(ContextCommand::CloseDesktopWindow(
745 CloseDesktopWindowCommand {},
746 )),
747 })
748 .await
749 .map_err(|_| Error::new("failed to send CloseDesktopWindowCommand"))?;
750 loop {
751 let event =
752 handle.events.message().await?.ok_or_else(|| {
753 Error::new("desktop window session closed while closing window")
754 })?;
755 match event.event {
756 Some(ContextEvent::DesktopWindowClosed(event))
757 if event.window_id == self.window_id =>
758 {
759 return Ok(());
760 }
761 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
762 _ => {}
763 }
764 }
765 }
766}
767
768impl DesktopWindowHook {
769 pub fn id(&self) -> &str {
770 &self.id
771 }
772 pub async fn wait(&self) -> Result<DesktopWindow> {
773 self.wait_with_options(CommandOptions::default()).await
774 }
775 pub async fn wait_with_options(&self, options: CommandOptions) -> Result<DesktopWindow> {
776 let mut state = self.app.app.inner.state.lock().await;
777 let handle = self.app.app.ensure_handle(&mut state).await?;
778 handle
779 .command_tx
780 .send(ContextSessionCommand {
781 surface_session_id: self.app.app.inner.surface_session_id.clone(),
782 context_session_id: self.app.app.inner.session_id.clone(),
783 command: Some(ContextCommand::WaitForHook(WaitForHookCommand {
784 hook_id: self.id.clone(),
785 retry_options: command_retry_options(options.timeout_ms),
786 })),
787 })
788 .await
789 .map_err(|_| Error::new("failed to send desktop WaitForHookCommand"))?;
790 loop {
791 let event = handle.events.message().await?.ok_or_else(|| {
792 Error::new("desktop app session closed while waiting for window hook")
793 })?;
794 match event.event {
795 Some(ContextEvent::HookCompleted(event)) if event.hook_id == self.id => {
796 let Some(HookCompletionResult::DesktopWindow(result)) = event.result else {
797 return Err(Error::new("desktop window hook returned an invalid result"));
798 };
799 return self.app.window(
800 result
801 .window
802 .ok_or_else(|| Error::new("desktop window hook returned no window"))?,
803 );
804 }
805 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
806 _ => {}
807 }
808 }
809 }
810}
811
812impl IosDevice {
813 pub fn session_id(&self) -> &str {
814 self.inner.session_id()
815 }
816
817 pub fn app(&self) -> IosApp {
818 self.inner.app()
819 }
820
821 pub fn initial_app(&self) -> IosApp {
822 self.inner.initial_app()
823 }
824
825 pub async fn launch(&self, options: MobileIosLaunchOptions) -> Result<IosApp> {
826 self.inner
827 .launch(MobileAndroidLaunchOptions {
828 apk_path: options.app_path,
829 app_id: options.app_id,
830 stop_before_launch: options.stop_before_launch,
831 timeout_ms: options.timeout_ms,
832 ..MobileAndroidLaunchOptions::default()
833 })
834 .await
835 }
836}
837
838impl AndroidDevice {
839 pub fn session_id(&self) -> &str {
840 &self.inner.session_id
841 }
842
843 pub fn app(&self) -> NativeApp {
844 self.inner
845 .current_app
846 .lock()
847 .map(|app| app.clone())
848 .unwrap_or_else(|_| self.inner.initial_app.clone())
849 }
850
851 pub fn initial_app(&self) -> NativeApp {
852 self.inner.initial_app.clone()
853 }
854
855 pub async fn launch(&self, options: MobileAndroidLaunchOptions) -> Result<NativeApp> {
856 let mut state = self.inner.state.lock().await;
857 ensure_android_device_open(&state, &self.inner.session_id)?;
858
859 state
860 .command_tx
861 .send(SurfaceSessionCommand {
862 command: Some(SurfaceCommand::LaunchApp(LaunchAppCommand {
863 apk_path: options.apk_path,
864 app_id: options.app_id,
865 launch_activity: options.launch_activity,
866 stop_before_launch: options.stop_before_launch,
867 retry_options: command_retry_options(options.timeout_ms),
868 })),
869 })
870 .await
871 .map_err(|_| Error::new("failed to send LaunchAppCommand"))?;
872
873 loop {
874 let event =
875 state.events.message().await?.ok_or_else(|| {
876 Error::new("surface session closed before app launch response")
877 })?;
878
879 match event.event {
880 Some(SurfaceEvent::AppLaunched(AppLaunchedEvent { app_session_id, .. })) => {
881 let app = NativeApp {
882 inner: Arc::new(NativeAppInner {
883 runtime: Arc::clone(&self.inner.runtime),
884 surface_session_id: event.session_id,
885 session_id: app_session_id,
886 state: AsyncMutex::new(NativeAppState::default()),
887 }),
888 };
889 if let Ok(mut current_app) = self.inner.current_app.lock() {
890 *current_app = app.clone();
891 }
892 return Ok(app);
893 }
894 Some(SurfaceEvent::Error(error)) => {
895 return Err(Error::new(format!(
896 "surface session error while launching Android app: {}",
897 error.message
898 )));
899 }
900 Some(SurfaceEvent::Closed(_)) => {
901 state.closed = true;
902 return Err(Error::new(
903 "surface session closed while waiting for Android app launch",
904 ));
905 }
906 _ => {}
907 }
908 }
909 }
910}
911
912impl NativeApp {
913 pub fn session_id(&self) -> &str {
914 &self.inner.session_id
915 }
916
917 pub async fn register_hook<T: AndroidHookType>(&self, _hook_type: T) -> Result<AndroidHook<T>> {
918 let mut state = self.inner.state.lock().await;
919 let handle = self.ensure_handle(&mut state).await?;
920 ensure_android_app_open(handle, &self.inner.session_id)?;
921 let hook = match T::name() {
922 "file_chooser" => Some(RegisterHook::MobileFileChooser(
923 RegisterMobileFileChooserHook {},
924 )),
925 "download" => Some(RegisterHook::MobileDownload(RegisterMobileDownloadHook {})),
926 _ => return Err(Error::new("hook type is not supported by Android apps")),
927 };
928 handle
929 .command_tx
930 .send(ContextSessionCommand {
931 surface_session_id: self.inner.surface_session_id.clone(),
932 context_session_id: self.inner.session_id.clone(),
933 command: Some(ContextCommand::RegisterHook(RegisterHookCommand { hook })),
934 })
935 .await
936 .map_err(|_| Error::new("failed to send Android RegisterHookCommand"))?;
937 loop {
938 let event =
939 handle.events.message().await?.ok_or_else(|| {
940 Error::new("Android app session closed while registering hook")
941 })?;
942 match event.event {
943 Some(ContextEvent::HookRegistered(event)) => {
944 return Ok(AndroidHook {
945 app: self.clone(),
946 id: event.hook_id,
947 _type: PhantomData,
948 });
949 }
950 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
951 _ => {}
952 }
953 }
954 }
955
956 pub fn locator(&self, selector: impl Into<String>) -> NativeLocator {
957 NativeLocator {
958 page: self.clone(),
959 selector: normalize_mobile_selector_for_transport(&selector.into()),
960 }
961 }
962
963 pub fn get_by_role(&self, role: impl Into<String>) -> NativeLocator {
964 self.get_by_role_with_options(role, RoleOptions::default())
965 }
966
967 pub fn get_by_role_with_options(
968 &self,
969 role: impl Into<String>,
970 options: RoleOptions,
971 ) -> NativeLocator {
972 let mut spec = serde_json::to_value(options).expect("role options");
973 spec["kind"] = serde_json::json!("role");
974 spec["role"] = serde_json::json!(role.into());
975 self.locator(mobile_semantic_selector(spec))
976 }
977
978 pub fn get_by_text(&self, text: impl Into<TextMatcher>) -> NativeLocator {
979 self.get_by_text_with_options(text, TextOptions::default())
980 }
981
982 pub fn get_by_text_with_options(
983 &self,
984 text: impl Into<TextMatcher>,
985 options: TextOptions,
986 ) -> NativeLocator {
987 self.locator(mobile_text_selector("text", text.into(), options))
988 }
989
990 pub fn get_by_label(&self, text: impl Into<TextMatcher>) -> NativeLocator {
991 self.get_by_label_with_options(text, TextOptions::default())
992 }
993
994 pub fn get_by_label_with_options(
995 &self,
996 text: impl Into<TextMatcher>,
997 options: TextOptions,
998 ) -> NativeLocator {
999 self.locator(mobile_text_selector("label", text.into(), options))
1000 }
1001
1002 pub fn get_by_test_id(&self, text: impl Into<TextMatcher>) -> NativeLocator {
1003 self.locator(mobile_semantic_selector(
1004 serde_json::json!({"kind":"testId", "text":text.into()}),
1005 ))
1006 }
1007
1008 pub async fn goto(&self, url: &str, options: CommandOptions) -> Result<()> {
1009 self.navigate(url, options).await
1010 }
1011
1012 pub async fn navigate(&self, url: &str, options: CommandOptions) -> Result<()> {
1013 let mut state = self.inner.state.lock().await;
1014 let handle = self.ensure_handle(&mut state).await?;
1015 ensure_android_app_open(handle, &self.inner.session_id)?;
1016 handle
1017 .command_tx
1018 .send(ContextSessionCommand {
1019 surface_session_id: self.inner.surface_session_id.clone(),
1020 context_session_id: self.inner.session_id.clone(),
1021 command: Some(ContextCommand::Navigate(NavigatePageCommand {
1022 url: url.to_string(),
1023 retry_options: command_retry_options(options.timeout_ms),
1024 })),
1025 })
1026 .await
1027 .map_err(|_| Error::new("failed to send mobile NavigatePageCommand"))?;
1028 loop {
1029 let event =
1030 handle.events.message().await?.ok_or_else(|| {
1031 Error::new("native app session closed while opening deep link")
1032 })?;
1033 match event.event {
1034 Some(ContextEvent::Navigated(_)) => return Ok(()),
1035 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1036 Some(ContextEvent::Closed(_)) => {
1037 handle.closed = true;
1038 return Err(Error::new(
1039 "native app session closed while opening deep link",
1040 ));
1041 }
1042 _ => {}
1043 }
1044 }
1045 }
1046
1047 pub async fn click(&self, selector: &str, options: CommandOptions) -> Result<()> {
1048 let selector = normalize_mobile_selector_for_transport(selector);
1049 let mut state = self.inner.state.lock().await;
1050 let handle = self.ensure_handle(&mut state).await?;
1051 ensure_android_app_open(handle, &self.inner.session_id)?;
1052
1053 handle
1054 .command_tx
1055 .send(ContextSessionCommand {
1056 surface_session_id: self.inner.surface_session_id.clone(),
1057 context_session_id: self.inner.session_id.clone(),
1058 command: Some(ContextCommand::ClickElement(ClickElementCommand {
1059 css_selector: selector.clone(),
1060 retry_options: command_retry_options(options.timeout_ms),
1061 button: None,
1062 click_count: None,
1063 })),
1064 })
1065 .await
1066 .map_err(|_| Error::new("failed to send ClickElementCommand"))?;
1067
1068 loop {
1069 let event =
1070 handle.events.message().await?.ok_or_else(|| {
1071 Error::new("app session closed while waiting for click result")
1072 })?;
1073
1074 match event.event {
1075 Some(ContextEvent::Attached(_)) => {}
1076 Some(ContextEvent::ElementClicked(_)) => return Ok(()),
1077 Some(ContextEvent::Error(error)) => {
1078 return Err(Error::new(format!(
1079 "app session error while clicking Android locator {:?}: {}",
1080 selector, error.message,
1081 )));
1082 }
1083 Some(ContextEvent::Closed(_)) => {
1084 handle.closed = true;
1085 return Err(Error::new(format!(
1086 "app session {} closed while waiting for click result",
1087 self.inner.session_id
1088 )));
1089 }
1090 _ => {}
1091 }
1092 }
1093 }
1094
1095 pub async fn fill(&self, selector: &str, value: &str, options: CommandOptions) -> Result<()> {
1096 let selector = normalize_mobile_selector_for_transport(selector);
1097 let mut state = self.inner.state.lock().await;
1098 let handle = self.ensure_handle(&mut state).await?;
1099 ensure_android_app_open(handle, &self.inner.session_id)?;
1100
1101 handle
1102 .command_tx
1103 .send(ContextSessionCommand {
1104 surface_session_id: self.inner.surface_session_id.clone(),
1105 context_session_id: self.inner.session_id.clone(),
1106 command: Some(ContextCommand::FillElement(FillElementCommand {
1107 css_selector: selector.clone(),
1108 value: value.to_string(),
1109 retry_options: command_retry_options(options.timeout_ms),
1110 })),
1111 })
1112 .await
1113 .map_err(|_| Error::new("failed to send FillElementCommand"))?;
1114
1115 loop {
1116 let event =
1117 handle.events.message().await?.ok_or_else(|| {
1118 Error::new("app session closed while waiting for fill result")
1119 })?;
1120
1121 match event.event {
1122 Some(ContextEvent::Attached(_)) => {}
1123 Some(ContextEvent::ElementFilled(_)) => return Ok(()),
1124 Some(ContextEvent::Error(error)) => {
1125 return Err(Error::new(format!(
1126 "app session error while filling Android locator {:?}: {}",
1127 selector, error.message,
1128 )));
1129 }
1130 Some(ContextEvent::Closed(_)) => {
1131 handle.closed = true;
1132 return Err(Error::new(format!(
1133 "app session {} closed while waiting for fill result",
1134 self.inner.session_id
1135 )));
1136 }
1137 _ => {}
1138 }
1139 }
1140 }
1141
1142 pub async fn count(&self, selector: &str, options: CommandOptions) -> Result<u32> {
1143 let selector = normalize_mobile_selector_for_transport(selector);
1144 let mut state = self.inner.state.lock().await;
1145 let handle = self.ensure_handle(&mut state).await?;
1146 ensure_android_app_open(handle, &self.inner.session_id)?;
1147
1148 handle
1149 .command_tx
1150 .send(ContextSessionCommand {
1151 surface_session_id: self.inner.surface_session_id.clone(),
1152 context_session_id: self.inner.session_id.clone(),
1153 command: Some(ContextCommand::CountElements(CountElementsCommand {
1154 css_selector: selector.clone(),
1155 retry_options: command_retry_options(options.timeout_ms),
1156 })),
1157 })
1158 .await
1159 .map_err(|_| Error::new("failed to send CountElementsCommand"))?;
1160
1161 loop {
1162 let event =
1163 handle.events.message().await?.ok_or_else(|| {
1164 Error::new("app session closed while waiting for count result")
1165 })?;
1166
1167 match event.event {
1168 Some(ContextEvent::Attached(_)) => {}
1169 Some(ContextEvent::ElementCounted(counted)) => return Ok(counted.count),
1170 Some(ContextEvent::Error(error)) => {
1171 return Err(Error::new(format!(
1172 "app session error while counting Android locator {:?}: {}",
1173 selector, error.message,
1174 )));
1175 }
1176 Some(ContextEvent::Closed(_)) => {
1177 handle.closed = true;
1178 return Err(Error::new(format!(
1179 "app session {} closed while waiting for count result",
1180 self.inner.session_id
1181 )));
1182 }
1183 _ => {}
1184 }
1185 }
1186 }
1187
1188 pub async fn focus(&self, selector: &str, options: CommandOptions) -> Result<()> {
1189 let selector = normalize_mobile_selector_for_transport(selector);
1190 let mut state = self.inner.state.lock().await;
1191 let handle = self.ensure_handle(&mut state).await?;
1192 ensure_android_app_open(handle, &self.inner.session_id)?;
1193
1194 handle
1195 .command_tx
1196 .send(ContextSessionCommand {
1197 surface_session_id: self.inner.surface_session_id.clone(),
1198 context_session_id: self.inner.session_id.clone(),
1199 command: Some(ContextCommand::FocusElement(FocusElementCommand {
1200 css_selector: selector.clone(),
1201 retry_options: command_retry_options(options.timeout_ms),
1202 })),
1203 })
1204 .await
1205 .map_err(|_| Error::new("failed to send FocusElementCommand"))?;
1206
1207 loop {
1208 let event =
1209 handle.events.message().await?.ok_or_else(|| {
1210 Error::new("app session closed while waiting for focus result")
1211 })?;
1212
1213 match event.event {
1214 Some(ContextEvent::Attached(_)) => {}
1215 Some(ContextEvent::ElementFocused(_)) => return Ok(()),
1216 Some(ContextEvent::Error(error)) => {
1217 return Err(Error::new(format!(
1218 "app session error while focusing Android locator {:?}: {}",
1219 selector, error.message,
1220 )));
1221 }
1222 Some(ContextEvent::Closed(_)) => {
1223 handle.closed = true;
1224 return Err(Error::new(format!(
1225 "app session {} closed while waiting for focus result",
1226 self.inner.session_id
1227 )));
1228 }
1229 _ => {}
1230 }
1231 }
1232 }
1233
1234 pub async fn press(&self, selector: &str, key: &str, options: PressOptions) -> Result<()> {
1235 let selector = normalize_mobile_selector_for_transport(selector);
1236 let mut state = self.inner.state.lock().await;
1237 let handle = self.ensure_handle(&mut state).await?;
1238 ensure_android_app_open(handle, &self.inner.session_id)?;
1239
1240 handle
1241 .command_tx
1242 .send(ContextSessionCommand {
1243 surface_session_id: self.inner.surface_session_id.clone(),
1244 context_session_id: self.inner.session_id.clone(),
1245 command: Some(ContextCommand::PressKey(PressKeyCommand {
1246 css_selector: selector.clone(),
1247 key: key.to_string(),
1248 text: options.text,
1249 retry_options: command_retry_options(options.timeout_ms),
1250 })),
1251 })
1252 .await
1253 .map_err(|_| Error::new("failed to send PressKeyCommand"))?;
1254
1255 loop {
1256 let event =
1257 handle.events.message().await?.ok_or_else(|| {
1258 Error::new("app session closed while waiting for press result")
1259 })?;
1260
1261 match event.event {
1262 Some(ContextEvent::Attached(_)) => {}
1263 Some(ContextEvent::KeyPressed(_)) => return Ok(()),
1264 Some(ContextEvent::Error(error)) => {
1265 return Err(Error::new(format!(
1266 "app session error while pressing Android key on {:?}: {}",
1267 selector, error.message,
1268 )));
1269 }
1270 Some(ContextEvent::Closed(_)) => {
1271 handle.closed = true;
1272 return Err(Error::new(format!(
1273 "app session {} closed while waiting for press result",
1274 self.inner.session_id
1275 )));
1276 }
1277 _ => {}
1278 }
1279 }
1280 }
1281
1282 pub async fn text_content(
1283 &self,
1284 selector: &str,
1285 options: CommandOptions,
1286 ) -> Result<Option<String>> {
1287 self.read_text(selector, options, true)
1288 .await
1289 .map(|result| Some(result.text))
1290 }
1291
1292 pub async fn inner_text(&self, selector: &str, options: CommandOptions) -> Result<String> {
1293 self.read_text(selector, options, false)
1294 .await
1295 .map(|result| result.text)
1296 }
1297
1298 async fn capture(
1299 &self,
1300 kind: crate::proto::CaptureKind,
1301 selector: &str,
1302 attribute_name: &str,
1303 options: CommandOptions,
1304 ) -> Result<crate::proto::CaptureResolvedEvent> {
1305 let mut state = self.inner.state.lock().await;
1306 let handle = self.ensure_handle(&mut state).await?;
1307 ensure_android_app_open(handle, &self.inner.session_id)?;
1308 handle
1309 .command_tx
1310 .send(ContextSessionCommand {
1311 surface_session_id: self.inner.surface_session_id.clone(),
1312 context_session_id: self.inner.session_id.clone(),
1313 command: Some(ContextCommand::Capture(CaptureCommand {
1314 kind: kind as i32,
1315 css_selector: normalize_mobile_selector_for_transport(selector),
1316 attribute_name: attribute_name.to_string(),
1317 retry_options: command_retry_options(options.timeout_ms),
1318 })),
1319 })
1320 .await
1321 .map_err(|_| Error::new("failed to send native capture command"))?;
1322 loop {
1323 let event = handle
1324 .events
1325 .message()
1326 .await?
1327 .ok_or_else(|| Error::new("native app session closed while capturing"))?;
1328 match event.event {
1329 Some(ContextEvent::CaptureResolved(result)) => return Ok(result),
1330 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1331 Some(ContextEvent::Closed(_)) => {
1332 handle.closed = true;
1333 return Err(Error::new("native app session closed while capturing"));
1334 }
1335 _ => {}
1336 }
1337 }
1338 }
1339
1340 pub async fn input_value(&self, selector: &str, options: CommandOptions) -> Result<String> {
1341 Ok(self
1342 .capture(crate::proto::CaptureKind::InputValue, selector, "", options)
1343 .await?
1344 .value
1345 .unwrap_or_default())
1346 }
1347
1348 pub async fn is_checked(&self, selector: &str, options: CommandOptions) -> Result<bool> {
1349 Ok(self
1350 .capture(crate::proto::CaptureKind::Checked, selector, "", options)
1351 .await?
1352 .checked
1353 .unwrap_or(false))
1354 }
1355
1356 pub async fn get_attribute(
1357 &self,
1358 selector: &str,
1359 name: &str,
1360 options: CommandOptions,
1361 ) -> Result<Option<String>> {
1362 Ok(self
1363 .capture(
1364 crate::proto::CaptureKind::Attribute,
1365 selector,
1366 name,
1367 options,
1368 )
1369 .await?
1370 .value)
1371 }
1372
1373 pub async fn bounding_box(
1374 &self,
1375 selector: &str,
1376 options: CommandOptions,
1377 ) -> Result<Option<BoundingBox>> {
1378 Ok(self
1379 .capture(
1380 crate::proto::CaptureKind::BoundingBox,
1381 selector,
1382 "",
1383 options,
1384 )
1385 .await?
1386 .bounding_box
1387 .map(|value| BoundingBox {
1388 x: value.x,
1389 y: value.y,
1390 width: value.width,
1391 height: value.height,
1392 }))
1393 }
1394
1395 pub async fn wait_for_selector(
1396 &self,
1397 selector: &str,
1398 options: WaitForSelectorOptions,
1399 ) -> Result<()> {
1400 let selector = normalize_mobile_selector_for_transport(selector);
1401 let mut state = self.inner.state.lock().await;
1402 let handle = self.ensure_handle(&mut state).await?;
1403 ensure_android_app_open(handle, &self.inner.session_id)?;
1404
1405 handle
1406 .command_tx
1407 .send(ContextSessionCommand {
1408 surface_session_id: self.inner.surface_session_id.clone(),
1409 context_session_id: self.inner.session_id.clone(),
1410 command: Some(ContextCommand::WaitForSelector(WaitForSelectorCommand {
1411 css_selector: selector.clone(),
1412 visible: options.visible,
1413 retry_options: command_retry_options(options.timeout_ms),
1414 })),
1415 })
1416 .await
1417 .map_err(|_| Error::new("failed to send WaitForSelectorCommand"))?;
1418
1419 loop {
1420 let event = handle.events.message().await?.ok_or_else(|| {
1421 Error::new("app session closed while waiting for selector result")
1422 })?;
1423
1424 match event.event {
1425 Some(ContextEvent::Attached(_)) => {}
1426 Some(ContextEvent::SelectorWaitSatisfied(_)) => return Ok(()),
1427 Some(ContextEvent::Error(error)) => {
1428 return Err(Error::new(format!(
1429 "app session error while waiting for Android locator {:?}: {}",
1430 selector, error.message,
1431 )));
1432 }
1433 Some(ContextEvent::Closed(_)) => {
1434 handle.closed = true;
1435 return Err(Error::new(format!(
1436 "app session {} closed while waiting for selector result",
1437 self.inner.session_id
1438 )));
1439 }
1440 _ => {}
1441 }
1442 }
1443 }
1444
1445 pub async fn screenshot(&self) -> Result<Vec<u8>> {
1446 self.screenshot_with_options(ScreenshotOptions::default())
1447 .await
1448 }
1449
1450 pub async fn screenshot_with_options(&self, options: ScreenshotOptions) -> Result<Vec<u8>> {
1451 let mut state = self.inner.state.lock().await;
1452 let handle = self.ensure_handle(&mut state).await?;
1453 ensure_android_app_open(handle, &self.inner.session_id)?;
1454
1455 handle
1456 .command_tx
1457 .send(ContextSessionCommand {
1458 surface_session_id: self.inner.surface_session_id.clone(),
1459 context_session_id: self.inner.session_id.clone(),
1460 command: Some(ContextCommand::Screenshot(ScreenshotCommand {
1461 retry_options: command_retry_options(options.timeout_ms),
1462 full_page: Some(options.full_page),
1463 })),
1464 })
1465 .await
1466 .map_err(|_| Error::new("failed to send ScreenshotCommand"))?;
1467
1468 loop {
1469 let event = handle.events.message().await?.ok_or_else(|| {
1470 Error::new("app session closed while waiting for screenshot result")
1471 })?;
1472
1473 match event.event {
1474 Some(ContextEvent::Attached(_)) => {}
1475 Some(ContextEvent::ScreenshotCaptured(screenshot)) => {
1476 if let Some(path) = options.path.as_ref() {
1477 std::fs::write(path, &screenshot.png_data).map_err(|error| {
1478 Error::new(format!("write screenshot to {}: {error}", path.display()))
1479 })?;
1480 }
1481 return Ok(screenshot.png_data);
1482 }
1483 Some(ContextEvent::Error(error)) => {
1484 return Err(Error::new(format!(
1485 "app session error while capturing Android screenshot: {}",
1486 error.message
1487 )));
1488 }
1489 Some(ContextEvent::Closed(_)) => {
1490 handle.closed = true;
1491 return Err(Error::new(format!(
1492 "app session {} closed while waiting for screenshot result",
1493 self.inner.session_id
1494 )));
1495 }
1496 _ => {}
1497 }
1498 }
1499 }
1500
1501 async fn ensure_handle<'a>(
1502 &self,
1503 state: &'a mut NativeAppState,
1504 ) -> Result<&'a mut AndroidTabHandle> {
1505 if state.handle.is_none() {
1506 let mut engine = self.inner.runtime.engine.clone();
1507 let (command_tx, command_rx) = mpsc::channel(16);
1508 let response = engine
1509 .context_session(tonic::Request::new(ReceiverStream::new(command_rx)))
1510 .await?;
1511 state.handle = Some(AndroidTabHandle {
1512 command_tx,
1513 events: response.into_inner(),
1514 closed: false,
1515 });
1516 }
1517
1518 state
1519 .handle
1520 .as_mut()
1521 .ok_or_else(|| Error::new("android app session handle was not initialized"))
1522 }
1523
1524 async fn read_text(
1525 &self,
1526 selector: &str,
1527 options: CommandOptions,
1528 text_content: bool,
1529 ) -> Result<TextResult> {
1530 let selector = normalize_mobile_selector_for_transport(selector);
1531 let mut state = self.inner.state.lock().await;
1532 let handle = self.ensure_handle(&mut state).await?;
1533 ensure_android_app_open(handle, &self.inner.session_id)?;
1534
1535 let command = if text_content {
1536 ContextCommand::GetTextContent(GetTextContentCommand {
1537 css_selector: selector.clone(),
1538 retry_options: command_retry_options(options.timeout_ms),
1539 })
1540 } else {
1541 ContextCommand::GetInnerText(GetInnerTextCommand {
1542 css_selector: selector.clone(),
1543 retry_options: command_retry_options(options.timeout_ms),
1544 })
1545 };
1546
1547 handle
1548 .command_tx
1549 .send(ContextSessionCommand {
1550 surface_session_id: self.inner.surface_session_id.clone(),
1551 context_session_id: self.inner.session_id.clone(),
1552 command: Some(command),
1553 })
1554 .await
1555 .map_err(|_| Error::new("failed to send text read command"))?;
1556
1557 loop {
1558 let event =
1559 handle.events.message().await?.ok_or_else(|| {
1560 Error::new("app session closed while waiting for text result")
1561 })?;
1562
1563 match event.event {
1564 Some(ContextEvent::Attached(_)) => {}
1565 Some(ContextEvent::TextContentResolved(text)) => {
1566 return Ok(TextResult {
1567 selector: text.css_selector,
1568 text: text.text,
1569 note: text.note,
1570 });
1571 }
1572 Some(ContextEvent::InnerTextResolved(text)) => {
1573 return Ok(TextResult {
1574 selector: text.css_selector,
1575 text: text.text,
1576 note: text.note,
1577 });
1578 }
1579 Some(ContextEvent::Error(error)) => {
1580 return Err(Error::new(format!(
1581 "app session error while reading Android text for {:?}: {}",
1582 selector, error.message,
1583 )));
1584 }
1585 Some(ContextEvent::Closed(_)) => {
1586 handle.closed = true;
1587 return Err(Error::new(format!(
1588 "app session {} closed while waiting for text result",
1589 self.inner.session_id
1590 )));
1591 }
1592 _ => {}
1593 }
1594 }
1595 }
1596}
1597
1598impl<T: AndroidHookType> AndroidHook<T> {
1599 pub fn id(&self) -> &str {
1600 &self.id
1601 }
1602
1603 pub async fn wait(&self) -> Result<T::Output> {
1604 self.wait_with_options(CommandOptions::default()).await
1605 }
1606
1607 pub async fn wait_with_options(&self, options: CommandOptions) -> Result<T::Output> {
1608 let mut state = self.app.inner.state.lock().await;
1609 let handle = self.app.ensure_handle(&mut state).await?;
1610 handle
1611 .command_tx
1612 .send(ContextSessionCommand {
1613 surface_session_id: self.app.inner.surface_session_id.clone(),
1614 context_session_id: self.app.inner.session_id.clone(),
1615 command: Some(ContextCommand::WaitForHook(WaitForHookCommand {
1616 hook_id: self.id.clone(),
1617 retry_options: command_retry_options(options.timeout_ms),
1618 })),
1619 })
1620 .await
1621 .map_err(|_| Error::new("failed to send Android WaitForHookCommand"))?;
1622 loop {
1623 let event =
1624 handle.events.message().await?.ok_or_else(|| {
1625 Error::new("Android app session closed while waiting for hook")
1626 })?;
1627 match event.event {
1628 Some(ContextEvent::HookCompleted(event)) if event.hook_id == self.id => {
1629 return T::decode(
1630 &self.app,
1631 event
1632 .result
1633 .ok_or_else(|| Error::new("Android hook completed without a result"))?,
1634 );
1635 }
1636 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1637 _ => {}
1638 }
1639 }
1640 }
1641}
1642
1643impl AndroidFileChooser {
1644 pub fn id(&self) -> &str {
1645 &self.id
1646 }
1647 pub fn app(&self) -> &NativeApp {
1648 &self.app
1649 }
1650 pub fn is_multiple(&self) -> bool {
1651 self.is_multiple
1652 }
1653 pub async fn set_file(&self, path: impl AsRef<Path>) -> Result<()> {
1654 self.set_files([path]).await
1655 }
1656 pub async fn set_files<I, P>(&self, paths: I) -> Result<()>
1657 where
1658 I: IntoIterator<Item = P>,
1659 P: AsRef<Path>,
1660 {
1661 let paths = paths
1662 .into_iter()
1663 .map(|path| path.as_ref().to_path_buf())
1664 .collect::<Vec<_>>();
1665 if !self.is_multiple && paths.len() > 1 {
1666 return Err(Error::new(
1667 "Android file chooser does not accept multiple files",
1668 ));
1669 }
1670 let mut state = self.app.inner.state.lock().await;
1671 let handle = self.app.ensure_handle(&mut state).await?;
1672 let mut file_ids = Vec::with_capacity(paths.len());
1673 for path in paths {
1674 file_ids.push(upload_android_client_file(&self.app, handle, &path).await?);
1675 }
1676 handle
1677 .command_tx
1678 .send(ContextSessionCommand {
1679 surface_session_id: self.app.inner.surface_session_id.clone(),
1680 context_session_id: self.app.inner.session_id.clone(),
1681 command: Some(ContextCommand::SetMobileFileChooserFiles(
1682 SetMobileFileChooserFilesCommand {
1683 file_chooser_id: self.id.clone(),
1684 file_ids,
1685 retry_options: None,
1686 },
1687 )),
1688 })
1689 .await
1690 .map_err(|_| Error::new("failed to send SetMobileFileChooserFilesCommand"))?;
1691 loop {
1692 let event = handle.events.message().await?.ok_or_else(|| {
1693 Error::new("Android app session closed while setting chooser files")
1694 })?;
1695 match event.event {
1696 Some(ContextEvent::MobileFileChooserFilesSet(result))
1697 if result.file_chooser_id == self.id =>
1698 {
1699 return Ok(());
1700 }
1701 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1702 _ => {}
1703 }
1704 }
1705 }
1706}
1707
1708impl AndroidDownload {
1709 pub fn id(&self) -> &str {
1710 &self.id
1711 }
1712 pub fn app(&self) -> &NativeApp {
1713 &self.app
1714 }
1715 pub fn suggested_filename(&self) -> &str {
1716 &self.suggested_filename
1717 }
1718 pub async fn save_as(&self, path: impl AsRef<Path>) -> Result<()> {
1719 let mut state = self.app.inner.state.lock().await;
1720 let handle = self.app.ensure_handle(&mut state).await?;
1721 handle
1722 .command_tx
1723 .send(ContextSessionCommand {
1724 surface_session_id: self.app.inner.surface_session_id.clone(),
1725 context_session_id: self.app.inner.session_id.clone(),
1726 command: Some(ContextCommand::SaveMobileDownload(
1727 SaveMobileDownloadCommand {
1728 download_id: self.id.clone(),
1729 retry_options: None,
1730 },
1731 )),
1732 })
1733 .await
1734 .map_err(|_| Error::new("failed to send SaveMobileDownloadCommand"))?;
1735 loop {
1736 let event =
1737 handle.events.message().await?.ok_or_else(|| {
1738 Error::new("Android app session closed while saving download")
1739 })?;
1740 match event.event {
1741 Some(ContextEvent::MobileDownloadSaved(result))
1742 if result.download_id == self.id =>
1743 {
1744 return download_android_client_file(
1745 &self.app,
1746 handle,
1747 &result.file_id,
1748 path.as_ref(),
1749 )
1750 .await;
1751 }
1752 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1753 _ => {}
1754 }
1755 }
1756 }
1757}
1758
1759async fn upload_android_client_file(
1760 app: &NativeApp,
1761 handle: &mut AndroidTabHandle,
1762 path: &Path,
1763) -> Result<String> {
1764 let mut source = fs::File::open(path)
1765 .map_err(|error| Error::new(format!("open upload {}: {error}", path.display())))?;
1766 let size = source
1767 .metadata()
1768 .map_err(|error| Error::new(error.to_string()))?
1769 .len();
1770 let transfer_id = format!(
1771 "rust-mobile-upload-{}-{}",
1772 std::process::id(),
1773 MOBILE_TRANSFER_COUNTER.fetch_add(1, Ordering::Relaxed)
1774 );
1775 let name = path
1776 .file_name()
1777 .and_then(|name| name.to_str())
1778 .ok_or_else(|| Error::new("upload requires a valid file name"))?
1779 .to_string();
1780 let mut offset = 0_u64;
1781 loop {
1782 let mut data = vec![0; 256 * 1024];
1783 let count = source
1784 .read(&mut data)
1785 .map_err(|error| Error::new(error.to_string()))?;
1786 data.truncate(count);
1787 let last = offset + count as u64 >= size;
1788 handle
1789 .command_tx
1790 .send(ContextSessionCommand {
1791 surface_session_id: app.inner.surface_session_id.clone(),
1792 context_session_id: app.inner.session_id.clone(),
1793 command: Some(ContextCommand::UploadFileChunk(UploadFileChunkCommand {
1794 transfer_id: transfer_id.clone(),
1795 name: name.clone(),
1796 offset,
1797 data,
1798 last,
1799 })),
1800 })
1801 .await
1802 .map_err(|_| Error::new("failed to send UploadFileChunkCommand"))?;
1803 offset += count as u64;
1804 if last {
1805 break;
1806 }
1807 }
1808 loop {
1809 let event = handle
1810 .events
1811 .message()
1812 .await?
1813 .ok_or_else(|| Error::new("Android app session closed while uploading file"))?;
1814 match event.event {
1815 Some(ContextEvent::FileUploaded(result)) if result.transfer_id == transfer_id => {
1816 return Ok(result.file_id);
1817 }
1818 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1819 _ => {}
1820 }
1821 }
1822}
1823
1824async fn download_android_client_file(
1825 app: &NativeApp,
1826 handle: &mut AndroidTabHandle,
1827 file_id: &str,
1828 path: &Path,
1829) -> Result<()> {
1830 let temporary = path.with_extension(format!(
1831 "allwright-{}.tmp",
1832 MOBILE_TRANSFER_COUNTER.fetch_add(1, Ordering::Relaxed)
1833 ));
1834 let mut output = fs::OpenOptions::new()
1835 .write(true)
1836 .create_new(true)
1837 .open(&temporary)
1838 .map_err(|error| Error::new(error.to_string()))?;
1839 let mut offset = 0_u64;
1840 loop {
1841 handle
1842 .command_tx
1843 .send(ContextSessionCommand {
1844 surface_session_id: app.inner.surface_session_id.clone(),
1845 context_session_id: app.inner.session_id.clone(),
1846 command: Some(ContextCommand::ReadFileChunk(ReadFileChunkCommand {
1847 file_id: file_id.to_string(),
1848 offset,
1849 max_bytes: 256 * 1024,
1850 })),
1851 })
1852 .await
1853 .map_err(|_| Error::new("failed to send ReadFileChunkCommand"))?;
1854 let event = handle
1855 .events
1856 .message()
1857 .await?
1858 .ok_or_else(|| Error::new("Android app session closed while downloading file"))?;
1859 match event.event {
1860 Some(ContextEvent::FileChunk(chunk))
1861 if chunk.file_id == file_id && chunk.offset == offset =>
1862 {
1863 output
1864 .write_all(&chunk.data)
1865 .map_err(|error| Error::new(error.to_string()))?;
1866 offset += chunk.data.len() as u64;
1867 if chunk.last {
1868 drop(output);
1869 fs::rename(&temporary, path).map_err(|error| Error::new(error.to_string()))?;
1870 return Ok(());
1871 }
1872 }
1873 Some(ContextEvent::Error(error)) => {
1874 let _ = fs::remove_file(&temporary);
1875 return Err(Error::new(error.message));
1876 }
1877 _ => {}
1878 }
1879 }
1880}
1881
1882impl NativeLocator {
1883 pub fn app(&self) -> &NativeApp {
1884 &self.page
1885 }
1886
1887 pub fn selector(&self) -> &str {
1888 &self.selector
1889 }
1890
1891 pub fn locator(&self, selector: impl Into<String>) -> NativeLocator {
1892 NativeLocator {
1893 page: self.page.clone(),
1894 selector: chain_mobile_selector_for_transport(&self.selector, &selector.into()),
1895 }
1896 }
1897
1898 pub fn get_by_role(&self, role: impl Into<String>) -> NativeLocator {
1899 self.get_by_role_with_options(role, RoleOptions::default())
1900 }
1901
1902 pub fn get_by_role_with_options(
1903 &self,
1904 role: impl Into<String>,
1905 options: RoleOptions,
1906 ) -> NativeLocator {
1907 let mut spec = serde_json::to_value(options).expect("role options");
1908 spec["kind"] = serde_json::json!("role");
1909 spec["role"] = serde_json::json!(role.into());
1910 self.locator(mobile_semantic_selector(spec))
1911 }
1912
1913 pub fn get_by_text(&self, text: impl Into<TextMatcher>) -> NativeLocator {
1914 self.get_by_text_with_options(text, TextOptions::default())
1915 }
1916
1917 pub fn get_by_text_with_options(
1918 &self,
1919 text: impl Into<TextMatcher>,
1920 options: TextOptions,
1921 ) -> NativeLocator {
1922 self.locator(mobile_text_selector("text", text.into(), options))
1923 }
1924
1925 pub fn get_by_label(&self, text: impl Into<TextMatcher>) -> NativeLocator {
1926 self.get_by_label_with_options(text, TextOptions::default())
1927 }
1928
1929 pub fn get_by_label_with_options(
1930 &self,
1931 text: impl Into<TextMatcher>,
1932 options: TextOptions,
1933 ) -> NativeLocator {
1934 self.locator(mobile_text_selector("label", text.into(), options))
1935 }
1936
1937 pub fn get_by_test_id(&self, text: impl Into<TextMatcher>) -> NativeLocator {
1938 self.locator(mobile_semantic_selector(
1939 serde_json::json!({"kind":"testId", "text":text.into()}),
1940 ))
1941 }
1942
1943 pub async fn click(&self, options: CommandOptions) -> Result<()> {
1944 self.page.click(&self.selector, options).await
1945 }
1946
1947 pub async fn count(&self, options: CommandOptions) -> Result<u32> {
1948 self.page.count(&self.selector, options).await
1949 }
1950
1951 pub async fn focus(&self, options: CommandOptions) -> Result<()> {
1952 self.page.focus(&self.selector, options).await
1953 }
1954
1955 pub async fn fill(&self, value: &str, options: CommandOptions) -> Result<()> {
1956 self.page.fill(&self.selector, value, options).await
1957 }
1958
1959 pub async fn press(&self, key: &str, options: PressOptions) -> Result<()> {
1960 self.page.press(&self.selector, key, options).await
1961 }
1962
1963 pub async fn text_content(&self, options: CommandOptions) -> Result<Option<String>> {
1964 self.page.text_content(&self.selector, options).await
1965 }
1966
1967 pub async fn inner_text(&self, options: CommandOptions) -> Result<String> {
1968 self.page.inner_text(&self.selector, options).await
1969 }
1970
1971 pub async fn input_value(&self, options: CommandOptions) -> Result<String> {
1972 self.page.input_value(&self.selector, options).await
1973 }
1974
1975 pub async fn is_checked(&self, options: CommandOptions) -> Result<bool> {
1976 self.page.is_checked(&self.selector, options).await
1977 }
1978
1979 pub async fn get_attribute(
1980 &self,
1981 name: &str,
1982 options: CommandOptions,
1983 ) -> Result<Option<String>> {
1984 self.page.get_attribute(&self.selector, name, options).await
1985 }
1986
1987 pub async fn bounding_box(&self, options: CommandOptions) -> Result<Option<BoundingBox>> {
1988 self.page.bounding_box(&self.selector, options).await
1989 }
1990
1991 pub async fn wait_for(&self, options: WaitForSelectorOptions) -> Result<()> {
1992 self.page.wait_for_selector(&self.selector, options).await
1993 }
1994}
1995
1996fn ensure_android_device_open(state: &AndroidDeviceState, session_id: &str) -> Result<()> {
1997 if state.closed {
1998 return Err(Error::new(format!(
1999 "android device session {} is closed",
2000 session_id
2001 )));
2002 }
2003 Ok(())
2004}
2005
2006fn ensure_android_app_open(handle: &AndroidTabHandle, session_id: &str) -> Result<()> {
2007 if handle.closed {
2008 return Err(Error::new(format!(
2009 "android app session {} is closed",
2010 session_id
2011 )));
2012 }
2013 Ok(())
2014}
2015
2016fn mobile_semantic_selector(spec: serde_json::Value) -> String {
2017 format!(
2018 "aw={}",
2019 serde_json::to_string(&spec.to_string()).expect("JSON string")
2020 )
2021}
2022
2023fn mobile_text_selector(kind: &str, text: TextMatcher, options: TextOptions) -> String {
2024 mobile_semantic_selector(serde_json::json!({"kind":kind, "text":text, "exact":options.exact}))
2025}
2026
2027#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2028enum MobileSelectorFlavor {
2029 Css,
2030 XPath,
2031 UiAutomator,
2032 Semantic,
2033}
2034
2035impl MobileSelectorFlavor {
2036 fn as_str(self) -> &'static str {
2037 match self {
2038 Self::Css => "css",
2039 Self::XPath => "xpath",
2040 Self::UiAutomator => "uia",
2041 Self::Semantic => "aw",
2042 }
2043 }
2044}
2045
2046const UIAUTOMATOR_SELECTOR_KEYS: &[&str] = &[
2047 "text",
2048 "textcontains",
2049 "textmatches",
2050 "textstartswith",
2051 "classname",
2052 "classnamematches",
2053 "description",
2054 "desc",
2055 "descriptioncontains",
2056 "desccontains",
2057 "descriptionmatches",
2058 "descmatches",
2059 "descriptionstartswith",
2060 "descstartswith",
2061 "checkable",
2062 "checked",
2063 "clickable",
2064 "longclickable",
2065 "scrollable",
2066 "enabled",
2067 "focusable",
2068 "focused",
2069 "selected",
2070 "packagename",
2071 "package",
2072 "packagenamematches",
2073 "resourceid",
2074 "resourceidmatches",
2075 "index",
2076 "instance",
2077];
2078
2079fn parse_explicit_mobile_selector_prefix(selector: &str) -> Option<(MobileSelectorFlavor, usize)> {
2080 let lowered = selector.to_ascii_lowercase();
2081 if lowered.starts_with("xpath=") || lowered.starts_with("xpath:") {
2082 return Some((MobileSelectorFlavor::XPath, 6));
2083 }
2084 if lowered.starts_with("uia=") || lowered.starts_with("uia:") {
2085 return Some((MobileSelectorFlavor::UiAutomator, 4));
2086 }
2087 if lowered.starts_with("aw=") || lowered.starts_with("aw:") {
2088 return Some((MobileSelectorFlavor::Semantic, 3));
2089 }
2090 if let Some(prefix_len) = parse_ui_automator_selector_prefix(&lowered) {
2091 return Some((MobileSelectorFlavor::UiAutomator, prefix_len));
2092 }
2093 if lowered.starts_with("text=") || lowered.starts_with("text:") {
2094 return Some((MobileSelectorFlavor::UiAutomator, 5));
2095 }
2096 if lowered.starts_with("id=") || lowered.starts_with("id:") {
2097 return Some((MobileSelectorFlavor::Css, 3));
2098 }
2099 if lowered.starts_with("css=") || lowered.starts_with("css:") {
2100 return Some((MobileSelectorFlavor::Css, 4));
2101 }
2102 None
2103}
2104
2105fn parse_ui_automator_selector_prefix(selector: &str) -> Option<usize> {
2106 UIAUTOMATOR_SELECTOR_KEYS.iter().find_map(|key| {
2107 if selector.starts_with(key) {
2108 let separator = selector.as_bytes().get(key.len()).copied()?;
2109 if separator == b'=' || separator == b':' {
2110 return Some(key.len() + 1);
2111 }
2112 }
2113 None
2114 })
2115}
2116
2117fn find_json_string_end(value: &str) -> Option<usize> {
2118 let bytes = value.as_bytes();
2119 if bytes.first().copied()? != b'"' {
2120 return None;
2121 }
2122
2123 let mut index = 1usize;
2124 let mut escaped = false;
2125 while index < bytes.len() {
2126 let byte = bytes[index];
2127 if escaped {
2128 escaped = false;
2129 index += 1;
2130 continue;
2131 }
2132 match byte {
2133 b'\\' => escaped = true,
2134 b'"' => return Some(index + 1),
2135 _ => {}
2136 }
2137 index += 1;
2138 }
2139 None
2140}
2141
2142fn is_normalized_mobile_transport_selector(selector: &str) -> bool {
2143 let trimmed = selector.trim();
2144 if trimmed.is_empty() {
2145 return false;
2146 }
2147
2148 let mut index = 0usize;
2149 while index < trimmed.len() {
2150 let Some((_, prefix_len)) = parse_explicit_mobile_selector_prefix(&trimmed[index..]) else {
2151 return false;
2152 };
2153 index += prefix_len;
2154
2155 let remainder = &trimmed[index..];
2156 let Some(json_end) = find_json_string_end(remainder) else {
2157 return false;
2158 };
2159 index += json_end;
2160
2161 if index == trimmed.len() {
2162 return true;
2163 }
2164
2165 let whitespace_len = trimmed[index..]
2166 .chars()
2167 .take_while(|char| char.is_ascii_whitespace())
2168 .count();
2169 if whitespace_len == 0 {
2170 return false;
2171 }
2172 index += whitespace_len;
2173
2174 if parse_explicit_mobile_selector_prefix(&trimmed[index..]).is_none() {
2175 return false;
2176 }
2177 }
2178
2179 true
2180}
2181
2182fn decode_selector_body(body: &str) -> String {
2183 let candidate = body.trim();
2184 if candidate.len() >= 2 && candidate.starts_with('"') && candidate.ends_with('"') {
2185 if let Ok(decoded) = serde_json::from_str::<String>(candidate) {
2186 return unescape_shell_escaped_selector(&decoded);
2187 }
2188 }
2189 unescape_shell_escaped_selector(candidate)
2190}
2191
2192fn unescape_shell_escaped_selector(value: &str) -> String {
2193 let mut result = String::with_capacity(value.len());
2194 let mut chars = value.chars().peekable();
2195 while let Some(ch) = chars.next() {
2196 if ch == '\\' {
2197 match chars.peek().copied() {
2198 Some('_' | ' ' | '#' | ':' | '[' | ']' | '(' | ')' | '"' | '\'') => {
2199 result.push(chars.next().expect("peeked char should exist"));
2200 continue;
2201 }
2202 _ => {}
2203 }
2204 }
2205 result.push(ch);
2206 }
2207 result
2208}
2209
2210fn parse_mobile_selector_for_transport(selector: &str) -> (MobileSelectorFlavor, String) {
2211 let trimmed = selector.trim();
2212 if let Some((flavor, prefix_len)) = parse_explicit_mobile_selector_prefix(trimmed) {
2213 let body = decode_selector_body(&trimmed[prefix_len..]);
2214 return match flavor {
2215 MobileSelectorFlavor::Css if prefix_len == 3 => {
2216 let normalized = if body.starts_with('#') {
2217 body
2218 } else {
2219 format!("#{body}")
2220 };
2221 (MobileSelectorFlavor::Css, normalized)
2222 }
2223 MobileSelectorFlavor::UiAutomator
2224 if prefix_len != 4 && !trimmed[..prefix_len].eq_ignore_ascii_case("text=") =>
2225 {
2226 (
2227 MobileSelectorFlavor::UiAutomator,
2228 format!("{}={body}", &trimmed[..prefix_len - 1]),
2229 )
2230 }
2231 MobileSelectorFlavor::UiAutomator if prefix_len == 5 => {
2232 (MobileSelectorFlavor::UiAutomator, format!("text={body}"))
2233 }
2234 _ => (flavor, body),
2235 };
2236 }
2237
2238 if trimmed.starts_with("//")
2239 || trimmed.starts_with(".//")
2240 || trimmed.starts_with("../")
2241 || trimmed.starts_with('/')
2242 || trimmed.starts_with('(')
2243 {
2244 return (MobileSelectorFlavor::XPath, trimmed.to_string());
2245 }
2246
2247 (MobileSelectorFlavor::Css, trimmed.to_string())
2248}
2249
2250fn normalize_mobile_selector_for_transport(selector: &str) -> String {
2251 let trimmed = selector.trim();
2252 if trimmed.is_empty() {
2253 return String::new();
2254 }
2255 let lowered = trimmed.to_ascii_lowercase();
2256 if lowered.starts_with("ref=") || lowered.starts_with("ref:") {
2257 return format!("ref={}", decode_selector_body(&trimmed[4..]));
2258 }
2259 if is_normalized_mobile_transport_selector(trimmed) {
2260 return trimmed.to_string();
2261 }
2262 let (flavor, body) = parse_mobile_selector_for_transport(selector);
2263 format!(
2264 "{}={}",
2265 flavor.as_str(),
2266 serde_json::to_string(&body).unwrap_or_else(|_| format!("{body:?}"))
2267 )
2268}
2269
2270fn chain_mobile_selector_for_transport(parent: &str, child: &str) -> String {
2271 let parent = if parent.trim().is_empty() {
2272 String::new()
2273 } else {
2274 normalize_mobile_selector_for_transport(parent)
2275 };
2276 let child = if child.trim().is_empty() {
2277 String::new()
2278 } else {
2279 normalize_mobile_selector_for_transport(child)
2280 };
2281 if parent.is_empty() {
2282 return child;
2283 }
2284 if child.is_empty() {
2285 return parent;
2286 }
2287 format!("{parent} {child}")
2288}
2289
2290impl NativeApp {
2291 pub async fn accessibility_snapshot(&self) -> Result<String> {
2292 self.accessibility_snapshot_with_options(AccessibilitySnapshotOptions::default())
2293 .await
2294 }
2295
2296 pub async fn accessibility_snapshot_with_options(
2297 &self,
2298 options: AccessibilitySnapshotOptions,
2299 ) -> Result<String> {
2300 let mut state = self.inner.state.lock().await;
2301 let handle = self.ensure_handle(&mut state).await?;
2302 ensure_android_app_open(handle, &self.inner.session_id)?;
2303 handle
2304 .command_tx
2305 .send(ContextSessionCommand {
2306 surface_session_id: self.inner.surface_session_id.clone(),
2307 context_session_id: self.inner.session_id.clone(),
2308 command: Some(ContextCommand::AccessibilitySnapshot(
2309 AccessibilitySnapshotCommand {
2310 format: match options.format {
2311 AccessibilitySnapshotFormat::Json => "json",
2312 AccessibilitySnapshotFormat::Yaml => "yaml",
2313 }
2314 .into(),
2315 mode: match options.mode {
2316 AccessibilitySnapshotMode::Default => "default",
2317 AccessibilitySnapshotMode::Ai => "ai",
2318 AccessibilitySnapshotMode::Autoexpect => "autoexpect",
2319 AccessibilitySnapshotMode::Codegen => "codegen",
2320 }
2321 .into(),
2322 retry_options: command_retry_options(options.timeout_ms),
2323 },
2324 )),
2325 })
2326 .await
2327 .map_err(|_| Error::new("failed to send AccessibilitySnapshotCommand"))?;
2328 loop {
2329 let event = handle.events.message().await?.ok_or_else(|| {
2330 Error::new("android app session closed while capturing accessibility snapshot")
2331 })?;
2332 match event.event {
2333 Some(ContextEvent::AccessibilitySnapshotCaptured(result)) => {
2334 return Ok(result.snapshot);
2335 }
2336 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
2337 Some(ContextEvent::Closed(_)) => {
2338 handle.closed = true;
2339 return Err(Error::new(
2340 "android app session closed while capturing accessibility snapshot",
2341 ));
2342 }
2343 _ => {}
2344 }
2345 }
2346 }
2347}