1use std::sync::atomic::{AtomicU64, Ordering};
2use std::sync::{Arc, Mutex};
3use std::{
4 fs,
5 io::{Read, Write},
6 marker::PhantomData,
7 path::Path,
8};
9
10use crate::proto::context_session_command::Command as ContextCommand;
11use crate::proto::context_session_event::Event as ContextEvent;
12use crate::proto::hook_completed_event::Result as HookCompletionResult;
13use crate::proto::register_hook_command::Hook as RegisterHook;
14use crate::proto::surface_session_command::Command as SurfaceCommand;
15use crate::proto::surface_session_event::Event as SurfaceEvent;
16use crate::proto::{
17 AccessibilitySnapshotCommand, AppLaunchedEvent, CaptureCommand, ClickElementCommand,
18 ConnectDesktopCommand, ConnectMobileCommand, ContextSessionCommand, CountElementsCommand,
19 DesktopAppLaunchedEvent, DesktopConnectedEvent, DesktopPlatform as ProtoDesktopPlatform,
20 FillElementCommand, FocusElementCommand, GetInnerTextCommand, GetTextContentCommand,
21 LaunchAppCommand, LaunchDesktopAppCommand, MobileConnectedEvent,
22 MobilePlatform as ProtoMobilePlatform, NavigatePageCommand, PressKeyCommand,
23 ReadFileChunkCommand, RegisterHookCommand, RegisterMobileDownloadHook,
24 RegisterMobileFileChooserHook, SaveMobileDownloadCommand, ScreenshotCommand,
25 SetMobileFileChooserFilesCommand, SurfaceSessionCommand, UploadFileChunkCommand,
26 WaitForHookCommand, WaitForSelectorCommand,
27};
28
29use super::hook::{DownloadHook, FileChooserHook};
30
31static MOBILE_TRANSFER_COUNTER: AtomicU64 = AtomicU64::new(1);
32
33pub trait AndroidHookType: private::Sealed + Clone + Send + Sync + 'static {
34 type Output;
35}
36
37mod private {
38 use super::*;
39 pub trait Sealed {
40 fn name() -> &'static str;
41 fn decode(
42 app: &NativeApp,
43 result: HookCompletionResult,
44 ) -> Result<<Self as AndroidHookType>::Output>
45 where
46 Self: AndroidHookType;
47 }
48}
49
50impl AndroidHookType for FileChooserHook {
51 type Output = AndroidFileChooser;
52}
53impl private::Sealed for FileChooserHook {
54 fn name() -> &'static str {
55 "file_chooser"
56 }
57 fn decode(app: &NativeApp, result: HookCompletionResult) -> Result<AndroidFileChooser> {
58 let HookCompletionResult::MobileFileChooser(result) = result else {
59 return Err(Error::new(
60 "Android file chooser hook returned an invalid result",
61 ));
62 };
63 Ok(AndroidFileChooser {
64 app: app.clone(),
65 id: result.file_chooser_id,
66 is_multiple: result.is_multiple,
67 })
68 }
69}
70impl AndroidHookType for DownloadHook {
71 type Output = AndroidDownload;
72}
73impl private::Sealed for DownloadHook {
74 fn name() -> &'static str {
75 "download"
76 }
77 fn decode(app: &NativeApp, result: HookCompletionResult) -> Result<AndroidDownload> {
78 let HookCompletionResult::MobileDownload(result) = result else {
79 return Err(Error::new(
80 "Android download hook returned an invalid result",
81 ));
82 };
83 Ok(AndroidDownload {
84 app: app.clone(),
85 id: result.download_id,
86 suggested_filename: result.suggested_filename,
87 })
88 }
89}
90
91pub struct AndroidHook<T: AndroidHookType> {
92 app: NativeApp,
93 id: String,
94 _type: PhantomData<T>,
95}
96
97#[derive(Clone)]
98pub struct AndroidFileChooser {
99 app: NativeApp,
100 id: String,
101 is_multiple: bool,
102}
103
104#[derive(Clone)]
105pub struct AndroidDownload {
106 app: NativeApp,
107 id: String,
108 suggested_filename: String,
109}
110use tokio::sync::{Mutex as AsyncMutex, mpsc};
111use tokio_stream::wrappers::ReceiverStream;
112
113use super::command::command_retry_options;
114use super::runtime::get_runtime;
115use super::types::{
116 AccessibilitySnapshotFormat, AccessibilitySnapshotMode, AccessibilitySnapshotOptions,
117 BoundingBox, CommandOptions, Error, PressOptions, Result, RuntimeClient, ScreenshotOptions,
118 TextResult, WaitForSelectorOptions,
119};
120use super::web_locators::{RoleOptions, TextMatcher, TextOptions};
121
122#[derive(Debug, Clone, Default)]
123pub struct MobileAndroidConnectOptions {
124 pub device: Option<String>,
125 pub adb_endpoint: Option<String>,
126 pub preserve_app_state: bool,
127 pub timeout_ms: Option<u32>,
128}
129
130#[derive(Debug, Clone, Default)]
131pub struct MobileAndroidLaunchOptions {
132 pub apk_path: Option<String>,
133 pub app_id: Option<String>,
134 pub launch_activity: Option<String>,
135 pub stop_before_launch: bool,
136 pub timeout_ms: Option<u32>,
137}
138
139#[derive(Debug, Clone, Default)]
140pub struct MobileIosConnectOptions {
141 pub device: Option<String>,
142 pub agent_endpoint: Option<String>,
143 pub preserve_app_state: bool,
144 pub timeout_ms: Option<u32>,
145}
146
147#[derive(Debug, Clone, Default)]
148pub struct MobileIosLaunchOptions {
149 pub app_path: Option<String>,
150 pub app_id: Option<String>,
151 pub stop_before_launch: bool,
152 pub timeout_ms: Option<u32>,
153}
154
155#[derive(Debug, Clone, Default)]
156pub struct DesktopMacConnectOptions {
157 pub agent_endpoint: Option<String>,
158 pub timeout_ms: Option<u32>,
159}
160
161#[derive(Debug, Clone)]
162pub struct DesktopMacLaunchOptions {
163 pub app_id: String,
164 pub terminate_running: bool,
165 pub timeout_ms: Option<u32>,
166}
167
168pub type DesktopWindowsConnectOptions = DesktopMacConnectOptions;
169pub type DesktopWindowsLaunchOptions = DesktopMacLaunchOptions;
170
171pub type MacApp = NativeApp;
172pub type MacLocator = NativeLocator;
173pub type WindowsApp = NativeApp;
174pub type WindowsLocator = NativeLocator;
175
176#[derive(Clone)]
177pub struct MacDesktop {
178 inner: AndroidDevice,
179}
180pub type WindowsDesktop = MacDesktop;
181
182pub type IosApp = NativeApp;
183pub type IosLocator = NativeLocator;
184
185#[derive(Clone)]
186pub struct IosDevice {
187 inner: AndroidDevice,
188}
189
190#[derive(Clone)]
191pub struct NativeLocator {
192 page: NativeApp,
193 selector: String,
194}
195
196#[derive(Clone)]
197pub struct NativeApp {
198 inner: Arc<NativeAppInner>,
199}
200
201pub type AndroidApp = NativeApp;
202pub type AndroidLocator = NativeLocator;
203
204#[derive(Clone)]
205pub struct AndroidDevice {
206 inner: Arc<AndroidDeviceInner>,
207}
208
209struct AndroidDeviceInner {
210 runtime: Arc<RuntimeClient>,
211 state: AsyncMutex<AndroidDeviceState>,
212 session_id: String,
213 initial_app: NativeApp,
214 current_app: Mutex<NativeApp>,
215}
216
217struct AndroidDeviceState {
218 command_tx: mpsc::Sender<SurfaceSessionCommand>,
219 events: tonic::Streaming<crate::proto::SurfaceSessionEvent>,
220 closed: bool,
221}
222
223struct NativeAppInner {
224 runtime: Arc<RuntimeClient>,
225 surface_session_id: String,
226 session_id: String,
227 state: AsyncMutex<NativeAppState>,
228}
229
230#[derive(Default)]
231struct NativeAppState {
232 handle: Option<AndroidTabHandle>,
233}
234
235struct AndroidTabHandle {
236 command_tx: mpsc::Sender<crate::proto::ContextSessionCommand>,
237 events: tonic::Streaming<crate::proto::ContextSessionEvent>,
238 closed: bool,
239}
240
241pub mod android {
242 use super::*;
243
244 pub async fn connect(options: MobileAndroidConnectOptions) -> Result<AndroidDevice> {
245 let runtime = get_runtime().await?;
246 let mut engine = runtime.engine.clone();
247 let (command_tx, command_rx) = mpsc::channel(16);
248 let response = engine
249 .surface_session(tonic::Request::new(ReceiverStream::new(command_rx)))
250 .await?;
251 let mut events = response.into_inner();
252
253 command_tx
254 .send(SurfaceSessionCommand {
255 command: Some(SurfaceCommand::ConnectMobile(ConnectMobileCommand {
256 platform: ProtoMobilePlatform::Android as i32,
257 device: options.device,
258 adb_endpoint: options.adb_endpoint,
259 preserve_app_state: options.preserve_app_state,
260 retry_options: command_retry_options(options.timeout_ms),
261 })),
262 })
263 .await
264 .map_err(|_| Error::new("failed to send ConnectMobileCommand"))?;
265
266 loop {
267 let event = events.message().await?.ok_or_else(|| {
268 Error::new("surface session closed before mobile connect response")
269 })?;
270
271 match event.event {
272 Some(SurfaceEvent::MobileConnected(MobileConnectedEvent {
273 initial_app_session_id,
274 device_session_id,
275 ..
276 })) => {
277 let initial_app = NativeApp {
278 inner: Arc::new(NativeAppInner {
279 runtime: Arc::clone(&runtime),
280 surface_session_id: event.session_id.clone(),
281 session_id: initial_app_session_id,
282 state: AsyncMutex::new(NativeAppState::default()),
283 }),
284 };
285 return Ok(AndroidDevice {
286 inner: Arc::new(AndroidDeviceInner {
287 runtime,
288 state: AsyncMutex::new(AndroidDeviceState {
289 command_tx,
290 events,
291 closed: false,
292 }),
293 session_id: if device_session_id.is_empty() {
294 event.session_id
295 } else {
296 device_session_id
297 },
298 initial_app: initial_app.clone(),
299 current_app: Mutex::new(initial_app),
300 }),
301 });
302 }
303 Some(SurfaceEvent::Error(error)) => {
304 return Err(Error::new(format!(
305 "surface session error during mobile connect: {}",
306 error.message
307 )));
308 }
309 _ => {}
310 }
311 }
312 }
313}
314
315pub mod ios {
316 use super::*;
317
318 pub async fn connect(options: MobileIosConnectOptions) -> Result<IosDevice> {
319 let runtime = get_runtime().await?;
320 let mut engine = runtime.engine.clone();
321 let (command_tx, command_rx) = mpsc::channel(16);
322 let response = engine
323 .surface_session(tonic::Request::new(ReceiverStream::new(command_rx)))
324 .await?;
325 let mut events = response.into_inner();
326
327 command_tx
328 .send(SurfaceSessionCommand {
329 command: Some(SurfaceCommand::ConnectMobile(ConnectMobileCommand {
330 platform: ProtoMobilePlatform::Ios as i32,
331 device: options.device,
332 adb_endpoint: options.agent_endpoint,
335 preserve_app_state: options.preserve_app_state,
336 retry_options: command_retry_options(options.timeout_ms),
337 })),
338 })
339 .await
340 .map_err(|_| Error::new("failed to send iOS ConnectMobileCommand"))?;
341
342 loop {
343 let event = events
344 .message()
345 .await?
346 .ok_or_else(|| Error::new("surface session closed before iOS connect response"))?;
347 match event.event {
348 Some(SurfaceEvent::MobileConnected(MobileConnectedEvent {
349 initial_app_session_id,
350 device_session_id,
351 ..
352 })) => {
353 let initial_app = NativeApp {
354 inner: Arc::new(NativeAppInner {
355 runtime: Arc::clone(&runtime),
356 surface_session_id: event.session_id.clone(),
357 session_id: initial_app_session_id,
358 state: AsyncMutex::new(NativeAppState::default()),
359 }),
360 };
361 return Ok(IosDevice {
362 inner: AndroidDevice {
363 inner: Arc::new(AndroidDeviceInner {
364 runtime,
365 state: AsyncMutex::new(AndroidDeviceState {
366 command_tx,
367 events,
368 closed: false,
369 }),
370 session_id: if device_session_id.is_empty() {
371 event.session_id
372 } else {
373 device_session_id
374 },
375 initial_app: initial_app.clone(),
376 current_app: Mutex::new(initial_app),
377 }),
378 },
379 });
380 }
381 Some(SurfaceEvent::Error(error)) => {
382 return Err(Error::new(format!(
383 "surface session error during iOS connect: {}",
384 error.message
385 )));
386 }
387 _ => {}
388 }
389 }
390 }
391}
392
393pub mod desktop {
394 use super::*;
395 pub use super::{
396 DesktopMacConnectOptions, DesktopMacLaunchOptions, DesktopWindowsConnectOptions,
397 DesktopWindowsLaunchOptions, MacApp, MacDesktop, MacLocator, WindowsApp, WindowsDesktop,
398 WindowsLocator,
399 };
400
401 pub mod mac {
402 use super::*;
403
404 pub async fn connect(options: DesktopMacConnectOptions) -> Result<MacDesktop> {
405 let runtime = get_runtime().await?;
406 let mut engine = runtime.engine.clone();
407 let (command_tx, command_rx) = mpsc::channel(16);
408 let response = engine
409 .surface_session(tonic::Request::new(ReceiverStream::new(command_rx)))
410 .await?;
411 let mut events = response.into_inner();
412 command_tx
413 .send(SurfaceSessionCommand {
414 command: Some(SurfaceCommand::ConnectDesktop(ConnectDesktopCommand {
415 platform: ProtoDesktopPlatform::Mac as i32,
416 agent_endpoint: options.agent_endpoint,
417 retry_options: command_retry_options(options.timeout_ms),
418 })),
419 })
420 .await
421 .map_err(|_| Error::new("failed to send ConnectDesktopCommand"))?;
422 loop {
423 let event = events.message().await?.ok_or_else(|| {
424 Error::new("surface session closed before macOS desktop connect response")
425 })?;
426 match event.event {
427 Some(SurfaceEvent::DesktopConnected(DesktopConnectedEvent {
428 initial_app_session_id,
429 desktop_session_id,
430 ..
431 })) => {
432 let initial_app = NativeApp {
433 inner: Arc::new(NativeAppInner {
434 runtime: Arc::clone(&runtime),
435 surface_session_id: event.session_id.clone(),
436 session_id: initial_app_session_id,
437 state: AsyncMutex::new(NativeAppState::default()),
438 }),
439 };
440 return Ok(MacDesktop {
441 inner: AndroidDevice {
442 inner: Arc::new(AndroidDeviceInner {
443 runtime,
444 state: AsyncMutex::new(AndroidDeviceState {
445 command_tx,
446 events,
447 closed: false,
448 }),
449 session_id: if desktop_session_id.is_empty() {
450 event.session_id
451 } else {
452 desktop_session_id
453 },
454 initial_app: initial_app.clone(),
455 current_app: Mutex::new(initial_app),
456 }),
457 },
458 });
459 }
460 Some(SurfaceEvent::Error(error)) => return Err(Error::new(error.message)),
461 _ => {}
462 }
463 }
464 }
465 }
466
467 pub mod windows {
468 use super::*;
469
470 pub async fn connect(options: DesktopWindowsConnectOptions) -> Result<WindowsDesktop> {
471 let runtime = get_runtime().await?;
472 let mut engine = runtime.engine.clone();
473 let (command_tx, command_rx) = mpsc::channel(16);
474 let response = engine
475 .surface_session(tonic::Request::new(ReceiverStream::new(command_rx)))
476 .await?;
477 let mut events = response.into_inner();
478 command_tx
479 .send(SurfaceSessionCommand {
480 command: Some(SurfaceCommand::ConnectDesktop(ConnectDesktopCommand {
481 platform: ProtoDesktopPlatform::Windows as i32,
482 agent_endpoint: options.agent_endpoint,
483 retry_options: command_retry_options(options.timeout_ms),
484 })),
485 })
486 .await
487 .map_err(|_| Error::new("failed to send ConnectDesktopCommand"))?;
488 loop {
489 let event = events.message().await?.ok_or_else(|| {
490 Error::new("surface session closed before Windows desktop connect response")
491 })?;
492 match event.event {
493 Some(SurfaceEvent::DesktopConnected(DesktopConnectedEvent {
494 initial_app_session_id,
495 desktop_session_id,
496 ..
497 })) => {
498 let initial_app = NativeApp {
499 inner: Arc::new(NativeAppInner {
500 runtime: Arc::clone(&runtime),
501 surface_session_id: event.session_id.clone(),
502 session_id: initial_app_session_id,
503 state: AsyncMutex::new(NativeAppState::default()),
504 }),
505 };
506 return Ok(MacDesktop {
507 inner: AndroidDevice {
508 inner: Arc::new(AndroidDeviceInner {
509 runtime,
510 state: AsyncMutex::new(AndroidDeviceState {
511 command_tx,
512 events,
513 closed: false,
514 }),
515 session_id: if desktop_session_id.is_empty() {
516 event.session_id
517 } else {
518 desktop_session_id
519 },
520 initial_app: initial_app.clone(),
521 current_app: Mutex::new(initial_app),
522 }),
523 },
524 });
525 }
526 Some(SurfaceEvent::Error(error)) => return Err(Error::new(error.message)),
527 _ => {}
528 }
529 }
530 }
531 }
532}
533
534impl MacDesktop {
535 pub fn session_id(&self) -> &str {
536 self.inner.session_id()
537 }
538
539 pub fn app(&self) -> MacApp {
540 self.inner.app()
541 }
542
543 pub async fn launch(&self, options: DesktopMacLaunchOptions) -> Result<MacApp> {
544 let mut state = self.inner.inner.state.lock().await;
545 ensure_android_device_open(&state, &self.inner.inner.session_id)?;
546 state
547 .command_tx
548 .send(SurfaceSessionCommand {
549 command: Some(SurfaceCommand::LaunchDesktopApp(LaunchDesktopAppCommand {
550 app_id: options.app_id,
551 terminate_running: options.terminate_running,
552 retry_options: command_retry_options(options.timeout_ms),
553 })),
554 })
555 .await
556 .map_err(|_| Error::new("failed to send LaunchDesktopAppCommand"))?;
557 loop {
558 let event = state.events.message().await?.ok_or_else(|| {
559 Error::new("surface session closed before macOS app launch response")
560 })?;
561 match event.event {
562 Some(SurfaceEvent::DesktopAppLaunched(DesktopAppLaunchedEvent {
563 app_session_id,
564 ..
565 })) => {
566 let app = NativeApp {
567 inner: Arc::new(NativeAppInner {
568 runtime: Arc::clone(&self.inner.inner.runtime),
569 surface_session_id: event.session_id,
570 session_id: app_session_id,
571 state: AsyncMutex::new(NativeAppState::default()),
572 }),
573 };
574 if let Ok(mut current_app) = self.inner.inner.current_app.lock() {
575 *current_app = app.clone();
576 }
577 return Ok(app);
578 }
579 Some(SurfaceEvent::Error(error)) => return Err(Error::new(error.message)),
580 _ => {}
581 }
582 }
583 }
584}
585
586impl IosDevice {
587 pub fn session_id(&self) -> &str {
588 self.inner.session_id()
589 }
590
591 pub fn app(&self) -> IosApp {
592 self.inner.app()
593 }
594
595 pub fn initial_app(&self) -> IosApp {
596 self.inner.initial_app()
597 }
598
599 pub async fn launch(&self, options: MobileIosLaunchOptions) -> Result<IosApp> {
600 self.inner
601 .launch(MobileAndroidLaunchOptions {
602 apk_path: options.app_path,
603 app_id: options.app_id,
604 stop_before_launch: options.stop_before_launch,
605 timeout_ms: options.timeout_ms,
606 ..MobileAndroidLaunchOptions::default()
607 })
608 .await
609 }
610}
611
612impl AndroidDevice {
613 pub fn session_id(&self) -> &str {
614 &self.inner.session_id
615 }
616
617 pub fn app(&self) -> NativeApp {
618 self.inner
619 .current_app
620 .lock()
621 .map(|app| app.clone())
622 .unwrap_or_else(|_| self.inner.initial_app.clone())
623 }
624
625 pub fn initial_app(&self) -> NativeApp {
626 self.inner.initial_app.clone()
627 }
628
629 pub async fn launch(&self, options: MobileAndroidLaunchOptions) -> Result<NativeApp> {
630 let mut state = self.inner.state.lock().await;
631 ensure_android_device_open(&state, &self.inner.session_id)?;
632
633 state
634 .command_tx
635 .send(SurfaceSessionCommand {
636 command: Some(SurfaceCommand::LaunchApp(LaunchAppCommand {
637 apk_path: options.apk_path,
638 app_id: options.app_id,
639 launch_activity: options.launch_activity,
640 stop_before_launch: options.stop_before_launch,
641 retry_options: command_retry_options(options.timeout_ms),
642 })),
643 })
644 .await
645 .map_err(|_| Error::new("failed to send LaunchAppCommand"))?;
646
647 loop {
648 let event =
649 state.events.message().await?.ok_or_else(|| {
650 Error::new("surface session closed before app launch response")
651 })?;
652
653 match event.event {
654 Some(SurfaceEvent::AppLaunched(AppLaunchedEvent { app_session_id, .. })) => {
655 let app = NativeApp {
656 inner: Arc::new(NativeAppInner {
657 runtime: Arc::clone(&self.inner.runtime),
658 surface_session_id: event.session_id,
659 session_id: app_session_id,
660 state: AsyncMutex::new(NativeAppState::default()),
661 }),
662 };
663 if let Ok(mut current_app) = self.inner.current_app.lock() {
664 *current_app = app.clone();
665 }
666 return Ok(app);
667 }
668 Some(SurfaceEvent::Error(error)) => {
669 return Err(Error::new(format!(
670 "surface session error while launching Android app: {}",
671 error.message
672 )));
673 }
674 Some(SurfaceEvent::Closed(_)) => {
675 state.closed = true;
676 return Err(Error::new(
677 "surface session closed while waiting for Android app launch",
678 ));
679 }
680 _ => {}
681 }
682 }
683 }
684}
685
686impl NativeApp {
687 pub fn session_id(&self) -> &str {
688 &self.inner.session_id
689 }
690
691 pub async fn register_hook<T: AndroidHookType>(&self, _hook_type: T) -> Result<AndroidHook<T>> {
692 let mut state = self.inner.state.lock().await;
693 let handle = self.ensure_handle(&mut state).await?;
694 ensure_android_app_open(handle, &self.inner.session_id)?;
695 let hook = match T::name() {
696 "file_chooser" => Some(RegisterHook::MobileFileChooser(
697 RegisterMobileFileChooserHook {},
698 )),
699 "download" => Some(RegisterHook::MobileDownload(RegisterMobileDownloadHook {})),
700 _ => return Err(Error::new("hook type is not supported by Android apps")),
701 };
702 handle
703 .command_tx
704 .send(ContextSessionCommand {
705 surface_session_id: self.inner.surface_session_id.clone(),
706 context_session_id: self.inner.session_id.clone(),
707 command: Some(ContextCommand::RegisterHook(RegisterHookCommand { hook })),
708 })
709 .await
710 .map_err(|_| Error::new("failed to send Android RegisterHookCommand"))?;
711 loop {
712 let event =
713 handle.events.message().await?.ok_or_else(|| {
714 Error::new("Android app session closed while registering hook")
715 })?;
716 match event.event {
717 Some(ContextEvent::HookRegistered(event)) => {
718 return Ok(AndroidHook {
719 app: self.clone(),
720 id: event.hook_id,
721 _type: PhantomData,
722 });
723 }
724 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
725 _ => {}
726 }
727 }
728 }
729
730 pub fn locator(&self, selector: impl Into<String>) -> NativeLocator {
731 NativeLocator {
732 page: self.clone(),
733 selector: normalize_mobile_selector_for_transport(&selector.into()),
734 }
735 }
736
737 pub fn get_by_role(&self, role: impl Into<String>) -> NativeLocator {
738 self.get_by_role_with_options(role, RoleOptions::default())
739 }
740
741 pub fn get_by_role_with_options(
742 &self,
743 role: impl Into<String>,
744 options: RoleOptions,
745 ) -> NativeLocator {
746 let mut spec = serde_json::to_value(options).expect("role options");
747 spec["kind"] = serde_json::json!("role");
748 spec["role"] = serde_json::json!(role.into());
749 self.locator(mobile_semantic_selector(spec))
750 }
751
752 pub fn get_by_text(&self, text: impl Into<TextMatcher>) -> NativeLocator {
753 self.get_by_text_with_options(text, TextOptions::default())
754 }
755
756 pub fn get_by_text_with_options(
757 &self,
758 text: impl Into<TextMatcher>,
759 options: TextOptions,
760 ) -> NativeLocator {
761 self.locator(mobile_text_selector("text", text.into(), options))
762 }
763
764 pub fn get_by_label(&self, text: impl Into<TextMatcher>) -> NativeLocator {
765 self.get_by_label_with_options(text, TextOptions::default())
766 }
767
768 pub fn get_by_label_with_options(
769 &self,
770 text: impl Into<TextMatcher>,
771 options: TextOptions,
772 ) -> NativeLocator {
773 self.locator(mobile_text_selector("label", text.into(), options))
774 }
775
776 pub fn get_by_test_id(&self, text: impl Into<TextMatcher>) -> NativeLocator {
777 self.locator(mobile_semantic_selector(
778 serde_json::json!({"kind":"testId", "text":text.into()}),
779 ))
780 }
781
782 pub async fn goto(&self, url: &str, options: CommandOptions) -> Result<()> {
783 self.navigate(url, options).await
784 }
785
786 pub async fn navigate(&self, url: &str, options: CommandOptions) -> Result<()> {
787 let mut state = self.inner.state.lock().await;
788 let handle = self.ensure_handle(&mut state).await?;
789 ensure_android_app_open(handle, &self.inner.session_id)?;
790 handle
791 .command_tx
792 .send(ContextSessionCommand {
793 surface_session_id: self.inner.surface_session_id.clone(),
794 context_session_id: self.inner.session_id.clone(),
795 command: Some(ContextCommand::Navigate(NavigatePageCommand {
796 url: url.to_string(),
797 retry_options: command_retry_options(options.timeout_ms),
798 })),
799 })
800 .await
801 .map_err(|_| Error::new("failed to send mobile NavigatePageCommand"))?;
802 loop {
803 let event =
804 handle.events.message().await?.ok_or_else(|| {
805 Error::new("native app session closed while opening deep link")
806 })?;
807 match event.event {
808 Some(ContextEvent::Navigated(_)) => return Ok(()),
809 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
810 Some(ContextEvent::Closed(_)) => {
811 handle.closed = true;
812 return Err(Error::new(
813 "native app session closed while opening deep link",
814 ));
815 }
816 _ => {}
817 }
818 }
819 }
820
821 pub async fn click(&self, selector: &str, options: CommandOptions) -> Result<()> {
822 let selector = normalize_mobile_selector_for_transport(selector);
823 let mut state = self.inner.state.lock().await;
824 let handle = self.ensure_handle(&mut state).await?;
825 ensure_android_app_open(handle, &self.inner.session_id)?;
826
827 handle
828 .command_tx
829 .send(ContextSessionCommand {
830 surface_session_id: self.inner.surface_session_id.clone(),
831 context_session_id: self.inner.session_id.clone(),
832 command: Some(ContextCommand::ClickElement(ClickElementCommand {
833 css_selector: selector.clone(),
834 retry_options: command_retry_options(options.timeout_ms),
835 button: None,
836 click_count: None,
837 })),
838 })
839 .await
840 .map_err(|_| Error::new("failed to send ClickElementCommand"))?;
841
842 loop {
843 let event =
844 handle.events.message().await?.ok_or_else(|| {
845 Error::new("app session closed while waiting for click result")
846 })?;
847
848 match event.event {
849 Some(ContextEvent::Attached(_)) => {}
850 Some(ContextEvent::ElementClicked(_)) => return Ok(()),
851 Some(ContextEvent::Error(error)) => {
852 return Err(Error::new(format!(
853 "app session error while clicking Android locator {:?}: {}",
854 selector, error.message,
855 )));
856 }
857 Some(ContextEvent::Closed(_)) => {
858 handle.closed = true;
859 return Err(Error::new(format!(
860 "app session {} closed while waiting for click result",
861 self.inner.session_id
862 )));
863 }
864 _ => {}
865 }
866 }
867 }
868
869 pub async fn fill(&self, selector: &str, value: &str, options: CommandOptions) -> Result<()> {
870 let selector = normalize_mobile_selector_for_transport(selector);
871 let mut state = self.inner.state.lock().await;
872 let handle = self.ensure_handle(&mut state).await?;
873 ensure_android_app_open(handle, &self.inner.session_id)?;
874
875 handle
876 .command_tx
877 .send(ContextSessionCommand {
878 surface_session_id: self.inner.surface_session_id.clone(),
879 context_session_id: self.inner.session_id.clone(),
880 command: Some(ContextCommand::FillElement(FillElementCommand {
881 css_selector: selector.clone(),
882 value: value.to_string(),
883 retry_options: command_retry_options(options.timeout_ms),
884 })),
885 })
886 .await
887 .map_err(|_| Error::new("failed to send FillElementCommand"))?;
888
889 loop {
890 let event =
891 handle.events.message().await?.ok_or_else(|| {
892 Error::new("app session closed while waiting for fill result")
893 })?;
894
895 match event.event {
896 Some(ContextEvent::Attached(_)) => {}
897 Some(ContextEvent::ElementFilled(_)) => return Ok(()),
898 Some(ContextEvent::Error(error)) => {
899 return Err(Error::new(format!(
900 "app session error while filling Android locator {:?}: {}",
901 selector, error.message,
902 )));
903 }
904 Some(ContextEvent::Closed(_)) => {
905 handle.closed = true;
906 return Err(Error::new(format!(
907 "app session {} closed while waiting for fill result",
908 self.inner.session_id
909 )));
910 }
911 _ => {}
912 }
913 }
914 }
915
916 pub async fn count(&self, selector: &str, options: CommandOptions) -> Result<u32> {
917 let selector = normalize_mobile_selector_for_transport(selector);
918 let mut state = self.inner.state.lock().await;
919 let handle = self.ensure_handle(&mut state).await?;
920 ensure_android_app_open(handle, &self.inner.session_id)?;
921
922 handle
923 .command_tx
924 .send(ContextSessionCommand {
925 surface_session_id: self.inner.surface_session_id.clone(),
926 context_session_id: self.inner.session_id.clone(),
927 command: Some(ContextCommand::CountElements(CountElementsCommand {
928 css_selector: selector.clone(),
929 retry_options: command_retry_options(options.timeout_ms),
930 })),
931 })
932 .await
933 .map_err(|_| Error::new("failed to send CountElementsCommand"))?;
934
935 loop {
936 let event =
937 handle.events.message().await?.ok_or_else(|| {
938 Error::new("app session closed while waiting for count result")
939 })?;
940
941 match event.event {
942 Some(ContextEvent::Attached(_)) => {}
943 Some(ContextEvent::ElementCounted(counted)) => return Ok(counted.count),
944 Some(ContextEvent::Error(error)) => {
945 return Err(Error::new(format!(
946 "app session error while counting Android locator {:?}: {}",
947 selector, error.message,
948 )));
949 }
950 Some(ContextEvent::Closed(_)) => {
951 handle.closed = true;
952 return Err(Error::new(format!(
953 "app session {} closed while waiting for count result",
954 self.inner.session_id
955 )));
956 }
957 _ => {}
958 }
959 }
960 }
961
962 pub async fn focus(&self, selector: &str, options: CommandOptions) -> Result<()> {
963 let selector = normalize_mobile_selector_for_transport(selector);
964 let mut state = self.inner.state.lock().await;
965 let handle = self.ensure_handle(&mut state).await?;
966 ensure_android_app_open(handle, &self.inner.session_id)?;
967
968 handle
969 .command_tx
970 .send(ContextSessionCommand {
971 surface_session_id: self.inner.surface_session_id.clone(),
972 context_session_id: self.inner.session_id.clone(),
973 command: Some(ContextCommand::FocusElement(FocusElementCommand {
974 css_selector: selector.clone(),
975 retry_options: command_retry_options(options.timeout_ms),
976 })),
977 })
978 .await
979 .map_err(|_| Error::new("failed to send FocusElementCommand"))?;
980
981 loop {
982 let event =
983 handle.events.message().await?.ok_or_else(|| {
984 Error::new("app session closed while waiting for focus result")
985 })?;
986
987 match event.event {
988 Some(ContextEvent::Attached(_)) => {}
989 Some(ContextEvent::ElementFocused(_)) => return Ok(()),
990 Some(ContextEvent::Error(error)) => {
991 return Err(Error::new(format!(
992 "app session error while focusing Android locator {:?}: {}",
993 selector, error.message,
994 )));
995 }
996 Some(ContextEvent::Closed(_)) => {
997 handle.closed = true;
998 return Err(Error::new(format!(
999 "app session {} closed while waiting for focus result",
1000 self.inner.session_id
1001 )));
1002 }
1003 _ => {}
1004 }
1005 }
1006 }
1007
1008 pub async fn press(&self, selector: &str, key: &str, options: PressOptions) -> Result<()> {
1009 let selector = normalize_mobile_selector_for_transport(selector);
1010 let mut state = self.inner.state.lock().await;
1011 let handle = self.ensure_handle(&mut state).await?;
1012 ensure_android_app_open(handle, &self.inner.session_id)?;
1013
1014 handle
1015 .command_tx
1016 .send(ContextSessionCommand {
1017 surface_session_id: self.inner.surface_session_id.clone(),
1018 context_session_id: self.inner.session_id.clone(),
1019 command: Some(ContextCommand::PressKey(PressKeyCommand {
1020 css_selector: selector.clone(),
1021 key: key.to_string(),
1022 text: options.text,
1023 retry_options: command_retry_options(options.timeout_ms),
1024 })),
1025 })
1026 .await
1027 .map_err(|_| Error::new("failed to send PressKeyCommand"))?;
1028
1029 loop {
1030 let event =
1031 handle.events.message().await?.ok_or_else(|| {
1032 Error::new("app session closed while waiting for press result")
1033 })?;
1034
1035 match event.event {
1036 Some(ContextEvent::Attached(_)) => {}
1037 Some(ContextEvent::KeyPressed(_)) => return Ok(()),
1038 Some(ContextEvent::Error(error)) => {
1039 return Err(Error::new(format!(
1040 "app session error while pressing Android key on {:?}: {}",
1041 selector, error.message,
1042 )));
1043 }
1044 Some(ContextEvent::Closed(_)) => {
1045 handle.closed = true;
1046 return Err(Error::new(format!(
1047 "app session {} closed while waiting for press result",
1048 self.inner.session_id
1049 )));
1050 }
1051 _ => {}
1052 }
1053 }
1054 }
1055
1056 pub async fn text_content(
1057 &self,
1058 selector: &str,
1059 options: CommandOptions,
1060 ) -> Result<Option<String>> {
1061 self.read_text(selector, options, true)
1062 .await
1063 .map(|result| Some(result.text))
1064 }
1065
1066 pub async fn inner_text(&self, selector: &str, options: CommandOptions) -> Result<String> {
1067 self.read_text(selector, options, false)
1068 .await
1069 .map(|result| result.text)
1070 }
1071
1072 async fn capture(
1073 &self,
1074 kind: crate::proto::CaptureKind,
1075 selector: &str,
1076 attribute_name: &str,
1077 options: CommandOptions,
1078 ) -> Result<crate::proto::CaptureResolvedEvent> {
1079 let mut state = self.inner.state.lock().await;
1080 let handle = self.ensure_handle(&mut state).await?;
1081 ensure_android_app_open(handle, &self.inner.session_id)?;
1082 handle
1083 .command_tx
1084 .send(ContextSessionCommand {
1085 surface_session_id: self.inner.surface_session_id.clone(),
1086 context_session_id: self.inner.session_id.clone(),
1087 command: Some(ContextCommand::Capture(CaptureCommand {
1088 kind: kind as i32,
1089 css_selector: normalize_mobile_selector_for_transport(selector),
1090 attribute_name: attribute_name.to_string(),
1091 retry_options: command_retry_options(options.timeout_ms),
1092 })),
1093 })
1094 .await
1095 .map_err(|_| Error::new("failed to send native capture command"))?;
1096 loop {
1097 let event = handle
1098 .events
1099 .message()
1100 .await?
1101 .ok_or_else(|| Error::new("native app session closed while capturing"))?;
1102 match event.event {
1103 Some(ContextEvent::CaptureResolved(result)) => return Ok(result),
1104 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1105 Some(ContextEvent::Closed(_)) => {
1106 handle.closed = true;
1107 return Err(Error::new("native app session closed while capturing"));
1108 }
1109 _ => {}
1110 }
1111 }
1112 }
1113
1114 pub async fn input_value(&self, selector: &str, options: CommandOptions) -> Result<String> {
1115 Ok(self
1116 .capture(crate::proto::CaptureKind::InputValue, selector, "", options)
1117 .await?
1118 .value
1119 .unwrap_or_default())
1120 }
1121
1122 pub async fn is_checked(&self, selector: &str, options: CommandOptions) -> Result<bool> {
1123 Ok(self
1124 .capture(crate::proto::CaptureKind::Checked, selector, "", options)
1125 .await?
1126 .checked
1127 .unwrap_or(false))
1128 }
1129
1130 pub async fn get_attribute(
1131 &self,
1132 selector: &str,
1133 name: &str,
1134 options: CommandOptions,
1135 ) -> Result<Option<String>> {
1136 Ok(self
1137 .capture(
1138 crate::proto::CaptureKind::Attribute,
1139 selector,
1140 name,
1141 options,
1142 )
1143 .await?
1144 .value)
1145 }
1146
1147 pub async fn bounding_box(
1148 &self,
1149 selector: &str,
1150 options: CommandOptions,
1151 ) -> Result<Option<BoundingBox>> {
1152 Ok(self
1153 .capture(
1154 crate::proto::CaptureKind::BoundingBox,
1155 selector,
1156 "",
1157 options,
1158 )
1159 .await?
1160 .bounding_box
1161 .map(|value| BoundingBox {
1162 x: value.x,
1163 y: value.y,
1164 width: value.width,
1165 height: value.height,
1166 }))
1167 }
1168
1169 pub async fn wait_for_selector(
1170 &self,
1171 selector: &str,
1172 options: WaitForSelectorOptions,
1173 ) -> Result<()> {
1174 let selector = normalize_mobile_selector_for_transport(selector);
1175 let mut state = self.inner.state.lock().await;
1176 let handle = self.ensure_handle(&mut state).await?;
1177 ensure_android_app_open(handle, &self.inner.session_id)?;
1178
1179 handle
1180 .command_tx
1181 .send(ContextSessionCommand {
1182 surface_session_id: self.inner.surface_session_id.clone(),
1183 context_session_id: self.inner.session_id.clone(),
1184 command: Some(ContextCommand::WaitForSelector(WaitForSelectorCommand {
1185 css_selector: selector.clone(),
1186 visible: options.visible,
1187 retry_options: command_retry_options(options.timeout_ms),
1188 })),
1189 })
1190 .await
1191 .map_err(|_| Error::new("failed to send WaitForSelectorCommand"))?;
1192
1193 loop {
1194 let event = handle.events.message().await?.ok_or_else(|| {
1195 Error::new("app session closed while waiting for selector result")
1196 })?;
1197
1198 match event.event {
1199 Some(ContextEvent::Attached(_)) => {}
1200 Some(ContextEvent::SelectorWaitSatisfied(_)) => return Ok(()),
1201 Some(ContextEvent::Error(error)) => {
1202 return Err(Error::new(format!(
1203 "app session error while waiting for Android locator {:?}: {}",
1204 selector, error.message,
1205 )));
1206 }
1207 Some(ContextEvent::Closed(_)) => {
1208 handle.closed = true;
1209 return Err(Error::new(format!(
1210 "app session {} closed while waiting for selector result",
1211 self.inner.session_id
1212 )));
1213 }
1214 _ => {}
1215 }
1216 }
1217 }
1218
1219 pub async fn screenshot(&self) -> Result<Vec<u8>> {
1220 self.screenshot_with_options(ScreenshotOptions::default())
1221 .await
1222 }
1223
1224 pub async fn screenshot_with_options(&self, options: ScreenshotOptions) -> Result<Vec<u8>> {
1225 let mut state = self.inner.state.lock().await;
1226 let handle = self.ensure_handle(&mut state).await?;
1227 ensure_android_app_open(handle, &self.inner.session_id)?;
1228
1229 handle
1230 .command_tx
1231 .send(ContextSessionCommand {
1232 surface_session_id: self.inner.surface_session_id.clone(),
1233 context_session_id: self.inner.session_id.clone(),
1234 command: Some(ContextCommand::Screenshot(ScreenshotCommand {
1235 retry_options: command_retry_options(options.timeout_ms),
1236 full_page: Some(options.full_page),
1237 })),
1238 })
1239 .await
1240 .map_err(|_| Error::new("failed to send ScreenshotCommand"))?;
1241
1242 loop {
1243 let event = handle.events.message().await?.ok_or_else(|| {
1244 Error::new("app session closed while waiting for screenshot result")
1245 })?;
1246
1247 match event.event {
1248 Some(ContextEvent::Attached(_)) => {}
1249 Some(ContextEvent::ScreenshotCaptured(screenshot)) => {
1250 if let Some(path) = options.path.as_ref() {
1251 std::fs::write(path, &screenshot.png_data).map_err(|error| {
1252 Error::new(format!("write screenshot to {}: {error}", path.display()))
1253 })?;
1254 }
1255 return Ok(screenshot.png_data);
1256 }
1257 Some(ContextEvent::Error(error)) => {
1258 return Err(Error::new(format!(
1259 "app session error while capturing Android screenshot: {}",
1260 error.message
1261 )));
1262 }
1263 Some(ContextEvent::Closed(_)) => {
1264 handle.closed = true;
1265 return Err(Error::new(format!(
1266 "app session {} closed while waiting for screenshot result",
1267 self.inner.session_id
1268 )));
1269 }
1270 _ => {}
1271 }
1272 }
1273 }
1274
1275 async fn ensure_handle<'a>(
1276 &self,
1277 state: &'a mut NativeAppState,
1278 ) -> Result<&'a mut AndroidTabHandle> {
1279 if state.handle.is_none() {
1280 let mut engine = self.inner.runtime.engine.clone();
1281 let (command_tx, command_rx) = mpsc::channel(16);
1282 let response = engine
1283 .context_session(tonic::Request::new(ReceiverStream::new(command_rx)))
1284 .await?;
1285 state.handle = Some(AndroidTabHandle {
1286 command_tx,
1287 events: response.into_inner(),
1288 closed: false,
1289 });
1290 }
1291
1292 state
1293 .handle
1294 .as_mut()
1295 .ok_or_else(|| Error::new("android app session handle was not initialized"))
1296 }
1297
1298 async fn read_text(
1299 &self,
1300 selector: &str,
1301 options: CommandOptions,
1302 text_content: bool,
1303 ) -> Result<TextResult> {
1304 let selector = normalize_mobile_selector_for_transport(selector);
1305 let mut state = self.inner.state.lock().await;
1306 let handle = self.ensure_handle(&mut state).await?;
1307 ensure_android_app_open(handle, &self.inner.session_id)?;
1308
1309 let command = if text_content {
1310 ContextCommand::GetTextContent(GetTextContentCommand {
1311 css_selector: selector.clone(),
1312 retry_options: command_retry_options(options.timeout_ms),
1313 })
1314 } else {
1315 ContextCommand::GetInnerText(GetInnerTextCommand {
1316 css_selector: selector.clone(),
1317 retry_options: command_retry_options(options.timeout_ms),
1318 })
1319 };
1320
1321 handle
1322 .command_tx
1323 .send(ContextSessionCommand {
1324 surface_session_id: self.inner.surface_session_id.clone(),
1325 context_session_id: self.inner.session_id.clone(),
1326 command: Some(command),
1327 })
1328 .await
1329 .map_err(|_| Error::new("failed to send text read command"))?;
1330
1331 loop {
1332 let event =
1333 handle.events.message().await?.ok_or_else(|| {
1334 Error::new("app session closed while waiting for text result")
1335 })?;
1336
1337 match event.event {
1338 Some(ContextEvent::Attached(_)) => {}
1339 Some(ContextEvent::TextContentResolved(text)) => {
1340 return Ok(TextResult {
1341 selector: text.css_selector,
1342 text: text.text,
1343 note: text.note,
1344 });
1345 }
1346 Some(ContextEvent::InnerTextResolved(text)) => {
1347 return Ok(TextResult {
1348 selector: text.css_selector,
1349 text: text.text,
1350 note: text.note,
1351 });
1352 }
1353 Some(ContextEvent::Error(error)) => {
1354 return Err(Error::new(format!(
1355 "app session error while reading Android text for {:?}: {}",
1356 selector, error.message,
1357 )));
1358 }
1359 Some(ContextEvent::Closed(_)) => {
1360 handle.closed = true;
1361 return Err(Error::new(format!(
1362 "app session {} closed while waiting for text result",
1363 self.inner.session_id
1364 )));
1365 }
1366 _ => {}
1367 }
1368 }
1369 }
1370}
1371
1372impl<T: AndroidHookType> AndroidHook<T> {
1373 pub fn id(&self) -> &str {
1374 &self.id
1375 }
1376
1377 pub async fn wait(&self) -> Result<T::Output> {
1378 self.wait_with_options(CommandOptions::default()).await
1379 }
1380
1381 pub async fn wait_with_options(&self, options: CommandOptions) -> Result<T::Output> {
1382 let mut state = self.app.inner.state.lock().await;
1383 let handle = self.app.ensure_handle(&mut state).await?;
1384 handle
1385 .command_tx
1386 .send(ContextSessionCommand {
1387 surface_session_id: self.app.inner.surface_session_id.clone(),
1388 context_session_id: self.app.inner.session_id.clone(),
1389 command: Some(ContextCommand::WaitForHook(WaitForHookCommand {
1390 hook_id: self.id.clone(),
1391 retry_options: command_retry_options(options.timeout_ms),
1392 })),
1393 })
1394 .await
1395 .map_err(|_| Error::new("failed to send Android WaitForHookCommand"))?;
1396 loop {
1397 let event =
1398 handle.events.message().await?.ok_or_else(|| {
1399 Error::new("Android app session closed while waiting for hook")
1400 })?;
1401 match event.event {
1402 Some(ContextEvent::HookCompleted(event)) if event.hook_id == self.id => {
1403 return T::decode(
1404 &self.app,
1405 event
1406 .result
1407 .ok_or_else(|| Error::new("Android hook completed without a result"))?,
1408 );
1409 }
1410 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1411 _ => {}
1412 }
1413 }
1414 }
1415}
1416
1417impl AndroidFileChooser {
1418 pub fn id(&self) -> &str {
1419 &self.id
1420 }
1421 pub fn app(&self) -> &NativeApp {
1422 &self.app
1423 }
1424 pub fn is_multiple(&self) -> bool {
1425 self.is_multiple
1426 }
1427 pub async fn set_file(&self, path: impl AsRef<Path>) -> Result<()> {
1428 self.set_files([path]).await
1429 }
1430 pub async fn set_files<I, P>(&self, paths: I) -> Result<()>
1431 where
1432 I: IntoIterator<Item = P>,
1433 P: AsRef<Path>,
1434 {
1435 let paths = paths
1436 .into_iter()
1437 .map(|path| path.as_ref().to_path_buf())
1438 .collect::<Vec<_>>();
1439 if !self.is_multiple && paths.len() > 1 {
1440 return Err(Error::new(
1441 "Android file chooser does not accept multiple files",
1442 ));
1443 }
1444 let mut state = self.app.inner.state.lock().await;
1445 let handle = self.app.ensure_handle(&mut state).await?;
1446 let mut file_ids = Vec::with_capacity(paths.len());
1447 for path in paths {
1448 file_ids.push(upload_android_client_file(&self.app, handle, &path).await?);
1449 }
1450 handle
1451 .command_tx
1452 .send(ContextSessionCommand {
1453 surface_session_id: self.app.inner.surface_session_id.clone(),
1454 context_session_id: self.app.inner.session_id.clone(),
1455 command: Some(ContextCommand::SetMobileFileChooserFiles(
1456 SetMobileFileChooserFilesCommand {
1457 file_chooser_id: self.id.clone(),
1458 file_ids,
1459 retry_options: None,
1460 },
1461 )),
1462 })
1463 .await
1464 .map_err(|_| Error::new("failed to send SetMobileFileChooserFilesCommand"))?;
1465 loop {
1466 let event = handle.events.message().await?.ok_or_else(|| {
1467 Error::new("Android app session closed while setting chooser files")
1468 })?;
1469 match event.event {
1470 Some(ContextEvent::MobileFileChooserFilesSet(result))
1471 if result.file_chooser_id == self.id =>
1472 {
1473 return Ok(());
1474 }
1475 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1476 _ => {}
1477 }
1478 }
1479 }
1480}
1481
1482impl AndroidDownload {
1483 pub fn id(&self) -> &str {
1484 &self.id
1485 }
1486 pub fn app(&self) -> &NativeApp {
1487 &self.app
1488 }
1489 pub fn suggested_filename(&self) -> &str {
1490 &self.suggested_filename
1491 }
1492 pub async fn save_as(&self, path: impl AsRef<Path>) -> Result<()> {
1493 let mut state = self.app.inner.state.lock().await;
1494 let handle = self.app.ensure_handle(&mut state).await?;
1495 handle
1496 .command_tx
1497 .send(ContextSessionCommand {
1498 surface_session_id: self.app.inner.surface_session_id.clone(),
1499 context_session_id: self.app.inner.session_id.clone(),
1500 command: Some(ContextCommand::SaveMobileDownload(
1501 SaveMobileDownloadCommand {
1502 download_id: self.id.clone(),
1503 retry_options: None,
1504 },
1505 )),
1506 })
1507 .await
1508 .map_err(|_| Error::new("failed to send SaveMobileDownloadCommand"))?;
1509 loop {
1510 let event =
1511 handle.events.message().await?.ok_or_else(|| {
1512 Error::new("Android app session closed while saving download")
1513 })?;
1514 match event.event {
1515 Some(ContextEvent::MobileDownloadSaved(result))
1516 if result.download_id == self.id =>
1517 {
1518 return download_android_client_file(
1519 &self.app,
1520 handle,
1521 &result.file_id,
1522 path.as_ref(),
1523 )
1524 .await;
1525 }
1526 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1527 _ => {}
1528 }
1529 }
1530 }
1531}
1532
1533async fn upload_android_client_file(
1534 app: &NativeApp,
1535 handle: &mut AndroidTabHandle,
1536 path: &Path,
1537) -> Result<String> {
1538 let mut source = fs::File::open(path)
1539 .map_err(|error| Error::new(format!("open upload {}: {error}", path.display())))?;
1540 let size = source
1541 .metadata()
1542 .map_err(|error| Error::new(error.to_string()))?
1543 .len();
1544 let transfer_id = format!(
1545 "rust-mobile-upload-{}-{}",
1546 std::process::id(),
1547 MOBILE_TRANSFER_COUNTER.fetch_add(1, Ordering::Relaxed)
1548 );
1549 let name = path
1550 .file_name()
1551 .and_then(|name| name.to_str())
1552 .ok_or_else(|| Error::new("upload requires a valid file name"))?
1553 .to_string();
1554 let mut offset = 0_u64;
1555 loop {
1556 let mut data = vec![0; 256 * 1024];
1557 let count = source
1558 .read(&mut data)
1559 .map_err(|error| Error::new(error.to_string()))?;
1560 data.truncate(count);
1561 let last = offset + count as u64 >= size;
1562 handle
1563 .command_tx
1564 .send(ContextSessionCommand {
1565 surface_session_id: app.inner.surface_session_id.clone(),
1566 context_session_id: app.inner.session_id.clone(),
1567 command: Some(ContextCommand::UploadFileChunk(UploadFileChunkCommand {
1568 transfer_id: transfer_id.clone(),
1569 name: name.clone(),
1570 offset,
1571 data,
1572 last,
1573 })),
1574 })
1575 .await
1576 .map_err(|_| Error::new("failed to send UploadFileChunkCommand"))?;
1577 offset += count as u64;
1578 if last {
1579 break;
1580 }
1581 }
1582 loop {
1583 let event = handle
1584 .events
1585 .message()
1586 .await?
1587 .ok_or_else(|| Error::new("Android app session closed while uploading file"))?;
1588 match event.event {
1589 Some(ContextEvent::FileUploaded(result)) if result.transfer_id == transfer_id => {
1590 return Ok(result.file_id);
1591 }
1592 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
1593 _ => {}
1594 }
1595 }
1596}
1597
1598async fn download_android_client_file(
1599 app: &NativeApp,
1600 handle: &mut AndroidTabHandle,
1601 file_id: &str,
1602 path: &Path,
1603) -> Result<()> {
1604 let temporary = path.with_extension(format!(
1605 "allwright-{}.tmp",
1606 MOBILE_TRANSFER_COUNTER.fetch_add(1, Ordering::Relaxed)
1607 ));
1608 let mut output = fs::OpenOptions::new()
1609 .write(true)
1610 .create_new(true)
1611 .open(&temporary)
1612 .map_err(|error| Error::new(error.to_string()))?;
1613 let mut offset = 0_u64;
1614 loop {
1615 handle
1616 .command_tx
1617 .send(ContextSessionCommand {
1618 surface_session_id: app.inner.surface_session_id.clone(),
1619 context_session_id: app.inner.session_id.clone(),
1620 command: Some(ContextCommand::ReadFileChunk(ReadFileChunkCommand {
1621 file_id: file_id.to_string(),
1622 offset,
1623 max_bytes: 256 * 1024,
1624 })),
1625 })
1626 .await
1627 .map_err(|_| Error::new("failed to send ReadFileChunkCommand"))?;
1628 let event = handle
1629 .events
1630 .message()
1631 .await?
1632 .ok_or_else(|| Error::new("Android app session closed while downloading file"))?;
1633 match event.event {
1634 Some(ContextEvent::FileChunk(chunk))
1635 if chunk.file_id == file_id && chunk.offset == offset =>
1636 {
1637 output
1638 .write_all(&chunk.data)
1639 .map_err(|error| Error::new(error.to_string()))?;
1640 offset += chunk.data.len() as u64;
1641 if chunk.last {
1642 drop(output);
1643 fs::rename(&temporary, path).map_err(|error| Error::new(error.to_string()))?;
1644 return Ok(());
1645 }
1646 }
1647 Some(ContextEvent::Error(error)) => {
1648 let _ = fs::remove_file(&temporary);
1649 return Err(Error::new(error.message));
1650 }
1651 _ => {}
1652 }
1653 }
1654}
1655
1656impl NativeLocator {
1657 pub fn app(&self) -> &NativeApp {
1658 &self.page
1659 }
1660
1661 pub fn selector(&self) -> &str {
1662 &self.selector
1663 }
1664
1665 pub fn locator(&self, selector: impl Into<String>) -> NativeLocator {
1666 NativeLocator {
1667 page: self.page.clone(),
1668 selector: chain_mobile_selector_for_transport(&self.selector, &selector.into()),
1669 }
1670 }
1671
1672 pub fn get_by_role(&self, role: impl Into<String>) -> NativeLocator {
1673 self.get_by_role_with_options(role, RoleOptions::default())
1674 }
1675
1676 pub fn get_by_role_with_options(
1677 &self,
1678 role: impl Into<String>,
1679 options: RoleOptions,
1680 ) -> NativeLocator {
1681 let mut spec = serde_json::to_value(options).expect("role options");
1682 spec["kind"] = serde_json::json!("role");
1683 spec["role"] = serde_json::json!(role.into());
1684 self.locator(mobile_semantic_selector(spec))
1685 }
1686
1687 pub fn get_by_text(&self, text: impl Into<TextMatcher>) -> NativeLocator {
1688 self.get_by_text_with_options(text, TextOptions::default())
1689 }
1690
1691 pub fn get_by_text_with_options(
1692 &self,
1693 text: impl Into<TextMatcher>,
1694 options: TextOptions,
1695 ) -> NativeLocator {
1696 self.locator(mobile_text_selector("text", text.into(), options))
1697 }
1698
1699 pub fn get_by_label(&self, text: impl Into<TextMatcher>) -> NativeLocator {
1700 self.get_by_label_with_options(text, TextOptions::default())
1701 }
1702
1703 pub fn get_by_label_with_options(
1704 &self,
1705 text: impl Into<TextMatcher>,
1706 options: TextOptions,
1707 ) -> NativeLocator {
1708 self.locator(mobile_text_selector("label", text.into(), options))
1709 }
1710
1711 pub fn get_by_test_id(&self, text: impl Into<TextMatcher>) -> NativeLocator {
1712 self.locator(mobile_semantic_selector(
1713 serde_json::json!({"kind":"testId", "text":text.into()}),
1714 ))
1715 }
1716
1717 pub async fn click(&self, options: CommandOptions) -> Result<()> {
1718 self.page.click(&self.selector, options).await
1719 }
1720
1721 pub async fn count(&self, options: CommandOptions) -> Result<u32> {
1722 self.page.count(&self.selector, options).await
1723 }
1724
1725 pub async fn focus(&self, options: CommandOptions) -> Result<()> {
1726 self.page.focus(&self.selector, options).await
1727 }
1728
1729 pub async fn fill(&self, value: &str, options: CommandOptions) -> Result<()> {
1730 self.page.fill(&self.selector, value, options).await
1731 }
1732
1733 pub async fn press(&self, key: &str, options: PressOptions) -> Result<()> {
1734 self.page.press(&self.selector, key, options).await
1735 }
1736
1737 pub async fn text_content(&self, options: CommandOptions) -> Result<Option<String>> {
1738 self.page.text_content(&self.selector, options).await
1739 }
1740
1741 pub async fn inner_text(&self, options: CommandOptions) -> Result<String> {
1742 self.page.inner_text(&self.selector, options).await
1743 }
1744
1745 pub async fn input_value(&self, options: CommandOptions) -> Result<String> {
1746 self.page.input_value(&self.selector, options).await
1747 }
1748
1749 pub async fn is_checked(&self, options: CommandOptions) -> Result<bool> {
1750 self.page.is_checked(&self.selector, options).await
1751 }
1752
1753 pub async fn get_attribute(
1754 &self,
1755 name: &str,
1756 options: CommandOptions,
1757 ) -> Result<Option<String>> {
1758 self.page.get_attribute(&self.selector, name, options).await
1759 }
1760
1761 pub async fn bounding_box(&self, options: CommandOptions) -> Result<Option<BoundingBox>> {
1762 self.page.bounding_box(&self.selector, options).await
1763 }
1764
1765 pub async fn wait_for(&self, options: WaitForSelectorOptions) -> Result<()> {
1766 self.page.wait_for_selector(&self.selector, options).await
1767 }
1768}
1769
1770fn ensure_android_device_open(state: &AndroidDeviceState, session_id: &str) -> Result<()> {
1771 if state.closed {
1772 return Err(Error::new(format!(
1773 "android device session {} is closed",
1774 session_id
1775 )));
1776 }
1777 Ok(())
1778}
1779
1780fn ensure_android_app_open(handle: &AndroidTabHandle, session_id: &str) -> Result<()> {
1781 if handle.closed {
1782 return Err(Error::new(format!(
1783 "android app session {} is closed",
1784 session_id
1785 )));
1786 }
1787 Ok(())
1788}
1789
1790fn mobile_semantic_selector(spec: serde_json::Value) -> String {
1791 format!(
1792 "aw={}",
1793 serde_json::to_string(&spec.to_string()).expect("JSON string")
1794 )
1795}
1796
1797fn mobile_text_selector(kind: &str, text: TextMatcher, options: TextOptions) -> String {
1798 mobile_semantic_selector(serde_json::json!({"kind":kind, "text":text, "exact":options.exact}))
1799}
1800
1801#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1802enum MobileSelectorFlavor {
1803 Css,
1804 XPath,
1805 UiAutomator,
1806 Semantic,
1807}
1808
1809impl MobileSelectorFlavor {
1810 fn as_str(self) -> &'static str {
1811 match self {
1812 Self::Css => "css",
1813 Self::XPath => "xpath",
1814 Self::UiAutomator => "uia",
1815 Self::Semantic => "aw",
1816 }
1817 }
1818}
1819
1820const UIAUTOMATOR_SELECTOR_KEYS: &[&str] = &[
1821 "text",
1822 "textcontains",
1823 "textmatches",
1824 "textstartswith",
1825 "classname",
1826 "classnamematches",
1827 "description",
1828 "desc",
1829 "descriptioncontains",
1830 "desccontains",
1831 "descriptionmatches",
1832 "descmatches",
1833 "descriptionstartswith",
1834 "descstartswith",
1835 "checkable",
1836 "checked",
1837 "clickable",
1838 "longclickable",
1839 "scrollable",
1840 "enabled",
1841 "focusable",
1842 "focused",
1843 "selected",
1844 "packagename",
1845 "package",
1846 "packagenamematches",
1847 "resourceid",
1848 "resourceidmatches",
1849 "index",
1850 "instance",
1851];
1852
1853fn parse_explicit_mobile_selector_prefix(selector: &str) -> Option<(MobileSelectorFlavor, usize)> {
1854 let lowered = selector.to_ascii_lowercase();
1855 if lowered.starts_with("xpath=") || lowered.starts_with("xpath:") {
1856 return Some((MobileSelectorFlavor::XPath, 6));
1857 }
1858 if lowered.starts_with("uia=") || lowered.starts_with("uia:") {
1859 return Some((MobileSelectorFlavor::UiAutomator, 4));
1860 }
1861 if lowered.starts_with("aw=") || lowered.starts_with("aw:") {
1862 return Some((MobileSelectorFlavor::Semantic, 3));
1863 }
1864 if let Some(prefix_len) = parse_ui_automator_selector_prefix(&lowered) {
1865 return Some((MobileSelectorFlavor::UiAutomator, prefix_len));
1866 }
1867 if lowered.starts_with("text=") || lowered.starts_with("text:") {
1868 return Some((MobileSelectorFlavor::UiAutomator, 5));
1869 }
1870 if lowered.starts_with("id=") || lowered.starts_with("id:") {
1871 return Some((MobileSelectorFlavor::Css, 3));
1872 }
1873 if lowered.starts_with("css=") || lowered.starts_with("css:") {
1874 return Some((MobileSelectorFlavor::Css, 4));
1875 }
1876 None
1877}
1878
1879fn parse_ui_automator_selector_prefix(selector: &str) -> Option<usize> {
1880 UIAUTOMATOR_SELECTOR_KEYS.iter().find_map(|key| {
1881 if selector.starts_with(key) {
1882 let separator = selector.as_bytes().get(key.len()).copied()?;
1883 if separator == b'=' || separator == b':' {
1884 return Some(key.len() + 1);
1885 }
1886 }
1887 None
1888 })
1889}
1890
1891fn find_json_string_end(value: &str) -> Option<usize> {
1892 let bytes = value.as_bytes();
1893 if bytes.first().copied()? != b'"' {
1894 return None;
1895 }
1896
1897 let mut index = 1usize;
1898 let mut escaped = false;
1899 while index < bytes.len() {
1900 let byte = bytes[index];
1901 if escaped {
1902 escaped = false;
1903 index += 1;
1904 continue;
1905 }
1906 match byte {
1907 b'\\' => escaped = true,
1908 b'"' => return Some(index + 1),
1909 _ => {}
1910 }
1911 index += 1;
1912 }
1913 None
1914}
1915
1916fn is_normalized_mobile_transport_selector(selector: &str) -> bool {
1917 let trimmed = selector.trim();
1918 if trimmed.is_empty() {
1919 return false;
1920 }
1921
1922 let mut index = 0usize;
1923 while index < trimmed.len() {
1924 let Some((_, prefix_len)) = parse_explicit_mobile_selector_prefix(&trimmed[index..]) else {
1925 return false;
1926 };
1927 index += prefix_len;
1928
1929 let remainder = &trimmed[index..];
1930 let Some(json_end) = find_json_string_end(remainder) else {
1931 return false;
1932 };
1933 index += json_end;
1934
1935 if index == trimmed.len() {
1936 return true;
1937 }
1938
1939 let whitespace_len = trimmed[index..]
1940 .chars()
1941 .take_while(|char| char.is_ascii_whitespace())
1942 .count();
1943 if whitespace_len == 0 {
1944 return false;
1945 }
1946 index += whitespace_len;
1947
1948 if parse_explicit_mobile_selector_prefix(&trimmed[index..]).is_none() {
1949 return false;
1950 }
1951 }
1952
1953 true
1954}
1955
1956fn decode_selector_body(body: &str) -> String {
1957 let candidate = body.trim();
1958 if candidate.len() >= 2 && candidate.starts_with('"') && candidate.ends_with('"') {
1959 if let Ok(decoded) = serde_json::from_str::<String>(candidate) {
1960 return unescape_shell_escaped_selector(&decoded);
1961 }
1962 }
1963 unescape_shell_escaped_selector(candidate)
1964}
1965
1966fn unescape_shell_escaped_selector(value: &str) -> String {
1967 let mut result = String::with_capacity(value.len());
1968 let mut chars = value.chars().peekable();
1969 while let Some(ch) = chars.next() {
1970 if ch == '\\' {
1971 match chars.peek().copied() {
1972 Some('_' | ' ' | '#' | ':' | '[' | ']' | '(' | ')' | '"' | '\'') => {
1973 result.push(chars.next().expect("peeked char should exist"));
1974 continue;
1975 }
1976 _ => {}
1977 }
1978 }
1979 result.push(ch);
1980 }
1981 result
1982}
1983
1984fn parse_mobile_selector_for_transport(selector: &str) -> (MobileSelectorFlavor, String) {
1985 let trimmed = selector.trim();
1986 if let Some((flavor, prefix_len)) = parse_explicit_mobile_selector_prefix(trimmed) {
1987 let body = decode_selector_body(&trimmed[prefix_len..]);
1988 return match flavor {
1989 MobileSelectorFlavor::Css if prefix_len == 3 => {
1990 let normalized = if body.starts_with('#') {
1991 body
1992 } else {
1993 format!("#{body}")
1994 };
1995 (MobileSelectorFlavor::Css, normalized)
1996 }
1997 MobileSelectorFlavor::UiAutomator
1998 if prefix_len != 4 && !trimmed[..prefix_len].eq_ignore_ascii_case("text=") =>
1999 {
2000 (
2001 MobileSelectorFlavor::UiAutomator,
2002 format!("{}={body}", &trimmed[..prefix_len - 1]),
2003 )
2004 }
2005 MobileSelectorFlavor::UiAutomator if prefix_len == 5 => {
2006 (MobileSelectorFlavor::UiAutomator, format!("text={body}"))
2007 }
2008 _ => (flavor, body),
2009 };
2010 }
2011
2012 if trimmed.starts_with("//")
2013 || trimmed.starts_with(".//")
2014 || trimmed.starts_with("../")
2015 || trimmed.starts_with('/')
2016 || trimmed.starts_with('(')
2017 {
2018 return (MobileSelectorFlavor::XPath, trimmed.to_string());
2019 }
2020
2021 (MobileSelectorFlavor::Css, trimmed.to_string())
2022}
2023
2024fn normalize_mobile_selector_for_transport(selector: &str) -> String {
2025 let trimmed = selector.trim();
2026 if trimmed.is_empty() {
2027 return String::new();
2028 }
2029 let lowered = trimmed.to_ascii_lowercase();
2030 if lowered.starts_with("ref=") || lowered.starts_with("ref:") {
2031 return format!("ref={}", decode_selector_body(&trimmed[4..]));
2032 }
2033 if is_normalized_mobile_transport_selector(trimmed) {
2034 return trimmed.to_string();
2035 }
2036 let (flavor, body) = parse_mobile_selector_for_transport(selector);
2037 format!(
2038 "{}={}",
2039 flavor.as_str(),
2040 serde_json::to_string(&body).unwrap_or_else(|_| format!("{body:?}"))
2041 )
2042}
2043
2044fn chain_mobile_selector_for_transport(parent: &str, child: &str) -> String {
2045 let parent = if parent.trim().is_empty() {
2046 String::new()
2047 } else {
2048 normalize_mobile_selector_for_transport(parent)
2049 };
2050 let child = if child.trim().is_empty() {
2051 String::new()
2052 } else {
2053 normalize_mobile_selector_for_transport(child)
2054 };
2055 if parent.is_empty() {
2056 return child;
2057 }
2058 if child.is_empty() {
2059 return parent;
2060 }
2061 format!("{parent} {child}")
2062}
2063
2064impl NativeApp {
2065 pub async fn accessibility_snapshot(&self) -> Result<String> {
2066 self.accessibility_snapshot_with_options(AccessibilitySnapshotOptions::default())
2067 .await
2068 }
2069
2070 pub async fn accessibility_snapshot_with_options(
2071 &self,
2072 options: AccessibilitySnapshotOptions,
2073 ) -> Result<String> {
2074 let mut state = self.inner.state.lock().await;
2075 let handle = self.ensure_handle(&mut state).await?;
2076 ensure_android_app_open(handle, &self.inner.session_id)?;
2077 handle
2078 .command_tx
2079 .send(ContextSessionCommand {
2080 surface_session_id: self.inner.surface_session_id.clone(),
2081 context_session_id: self.inner.session_id.clone(),
2082 command: Some(ContextCommand::AccessibilitySnapshot(
2083 AccessibilitySnapshotCommand {
2084 format: match options.format {
2085 AccessibilitySnapshotFormat::Json => "json",
2086 AccessibilitySnapshotFormat::Yaml => "yaml",
2087 }
2088 .into(),
2089 mode: match options.mode {
2090 AccessibilitySnapshotMode::Default => "default",
2091 AccessibilitySnapshotMode::Ai => "ai",
2092 AccessibilitySnapshotMode::Autoexpect => "autoexpect",
2093 AccessibilitySnapshotMode::Codegen => "codegen",
2094 }
2095 .into(),
2096 retry_options: command_retry_options(options.timeout_ms),
2097 },
2098 )),
2099 })
2100 .await
2101 .map_err(|_| Error::new("failed to send AccessibilitySnapshotCommand"))?;
2102 loop {
2103 let event = handle.events.message().await?.ok_or_else(|| {
2104 Error::new("android app session closed while capturing accessibility snapshot")
2105 })?;
2106 match event.event {
2107 Some(ContextEvent::AccessibilitySnapshotCaptured(result)) => {
2108 return Ok(result.snapshot);
2109 }
2110 Some(ContextEvent::Error(error)) => return Err(Error::new(error.message)),
2111 Some(ContextEvent::Closed(_)) => {
2112 handle.closed = true;
2113 return Err(Error::new(
2114 "android app session closed while capturing accessibility snapshot",
2115 ));
2116 }
2117 _ => {}
2118 }
2119 }
2120 }
2121}