use crate::{annotate, observe, snapshot, spa};
pub use crate::spa::{RouterType, SpaRouterInfo};
pub use crate::target::{BBox, LivePattern, Resolved, Target};
use std::collections::HashSet;
use std::fmt;
use crate::eoka::{BoundingBox, MouseButton, Page, Result};
pub use crate::eoka::{Browser, Error, StealthConfig};
#[derive(Debug, Clone)]
pub struct InteractiveElement {
pub index: usize,
pub tag: String,
pub role: Option<String>,
pub text: String,
pub placeholder: Option<String>,
pub input_type: Option<String>,
pub selector: String,
pub checked: bool,
pub disabled: bool,
pub visible: bool,
pub value: Option<String>,
pub bbox: BoundingBox,
pub fingerprint: u64,
}
impl InteractiveElement {
pub fn compute_fingerprint(
tag: &str,
text: &str,
role: Option<&str>,
input_type: Option<&str>,
placeholder: Option<&str>,
selector: &str,
) -> u64 {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
tag.hash(&mut hasher);
text.hash(&mut hasher);
role.hash(&mut hasher);
input_type.hash(&mut hasher);
placeholder.hash(&mut hasher);
selector.hash(&mut hasher);
hasher.finish()
}
}
impl fmt::Display for InteractiveElement {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "[{}] <{}", self.index, self.tag)?;
if let Some(ref t) = self.input_type {
if t != "text" {
write!(f, " type=\"{}\"", t)?;
}
}
f.write_str(">")?;
if self.checked {
f.write_str(" [checked]")?;
}
if !self.text.is_empty() {
write!(f, " \"{}\"", self.text)?;
}
if let Some(ref v) = self.value {
write!(f, " value=\"{}\"", v)?;
}
if let Some(ref p) = self.placeholder {
write!(f, " placeholder=\"{}\"", p)?;
}
if let Some(ref r) = self.role {
let redundant = (r == "button" && self.tag == "button")
|| (r == "link" && self.tag == "a")
|| (r == "menuitem" && self.tag == "a");
if !redundant {
write!(f, " role=\"{}\"", r)?;
}
}
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct ObserveConfig {
pub viewport_only: bool,
}
impl Default for ObserveConfig {
fn default() -> Self {
Self {
viewport_only: true,
}
}
}
#[derive(Debug)]
pub struct ObserveDiff {
pub added: Vec<usize>,
pub removed: usize,
pub total: usize,
}
impl fmt::Display for ObserveDiff {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.added.is_empty() && self.removed == 0 {
write!(f, "no changes ({} elements)", self.total)
} else {
let mut need_sep = false;
if !self.added.is_empty() {
write!(f, "+{} added", self.added.len())?;
need_sep = true;
}
if self.removed > 0 {
if need_sep {
write!(f, ", ")?;
}
write!(f, "-{} removed", self.removed)?;
}
write!(f, " ({} total)", self.total)
}
}
}
pub struct Session {
operations: crate::operations::OperationManager,
action_options: eoka::ActionOptions,
browser: Browser,
page: Page,
elements: Vec<InteractiveElement>,
config: ObserveConfig,
}
impl Session {
pub async fn launch() -> Result<Self> {
let browser = Browser::launch().await?;
let page = browser.new_page("about:blank").await?;
Ok(Self {
operations: Default::default(),
action_options: Default::default(),
browser,
page,
elements: Vec::new(),
config: ObserveConfig::default(),
})
}
pub async fn launch_with_config(stealth: StealthConfig) -> Result<Self> {
let browser = Browser::launch_with_config(stealth).await?;
let page = browser.new_page("about:blank").await?;
Ok(Self {
operations: Default::default(),
action_options: Default::default(),
browser,
page,
elements: Vec::new(),
config: ObserveConfig::default(),
})
}
pub fn set_action_options(&mut self, options: eoka::ActionOptions) {
self.action_options = options;
}
pub fn operations(&self) -> &crate::operations::OperationManager {
&self.operations
}
pub async fn perform(
&self,
request: eoka_protocol::ActionPerformArgs,
) -> anyhow::Result<eoka_protocol::OperationRecord> {
self.operations
.submit(
self.page.clone(),
request.operation_id,
request.action,
request.options,
)
.await
}
async fn perform_and_wait(&self, action: eoka::Action) -> Result<()> {
self.perform_with_options(action, self.action_options.clone())
.await
}
async fn perform_with_options(
&self,
action: eoka::Action,
options: eoka::ActionOptions,
) -> Result<()> {
let id = uuid::Uuid::new_v4().to_string();
let record = self
.operations
.submit(self.page.clone(), id.clone(), action, options)
.await
.map_err(|e| Error::transport(format!("Operation {id}: {e}")))?;
let record = if matches!(record.state, eoka_protocol::OperationState::Running) {
self.operations
.wait(&id, 86_400_000)
.await
.map_err(|e| Error::transport(format!("Operation {id}: {e}")))?
} else {
record
};
if record.outcome == eoka::ActionOutcome::VerifiedComplete {
return Ok(());
}
if let Some(receipt) = record.receipt {
return Err(Error::Action {
code: record.error_code.unwrap_or_else(|| "action_failed".into()),
receipt: Box::new(receipt),
source: Box::new(Error::transport(format!("Inspect operation {id}"))),
});
}
Err(Error::transport(format!(
"Operation {id}: {:?}",
record.outcome
)))
}
pub fn set_observe_config(&mut self, config: ObserveConfig) {
self.config = config;
}
pub fn page(&self) -> &Page {
&self.page
}
pub fn browser(&self) -> &Browser {
&self.browser
}
pub async fn ax_snapshot(&self, include_all: bool) -> anyhow::Result<snapshot::SnapshotResult> {
snapshot::snapshot(&self.page, include_all).await
}
pub async fn observe(&mut self) -> Result<&[InteractiveElement]> {
self.elements = observe::observe(&self.page, self.config.viewport_only).await?;
Ok(&self.elements)
}
pub async fn screenshot(&mut self) -> Result<Vec<u8>> {
if self.elements.is_empty() {
self.observe().await?;
}
annotate::annotated_screenshot(&self.page, &self.elements).await
}
pub fn element_list(&self) -> String {
let mut out = String::with_capacity(self.elements.len() * 40);
for el in &self.elements {
out.push_str(&el.to_string());
out.push('\n');
}
out
}
pub fn get(&self, index: usize) -> Option<&InteractiveElement> {
self.elements.get(index)
}
pub fn elements(&self) -> &[InteractiveElement] {
&self.elements
}
pub fn len(&self) -> usize {
self.elements.len()
}
pub fn is_empty(&self) -> bool {
self.elements.is_empty()
}
pub fn find_by_text(&self, needle: &str) -> Option<usize> {
let needle_lower = needle.to_lowercase();
self.elements
.iter()
.find(|e| e.text.to_lowercase().contains(&needle_lower))
.map(|e| e.index)
}
pub fn find_all_by_text(&self, needle: &str) -> Vec<usize> {
let needle_lower = needle.to_lowercase();
self.elements
.iter()
.filter(|e| e.text.to_lowercase().contains(&needle_lower))
.map(|e| e.index)
.collect()
}
pub async fn observe_diff(&mut self) -> Result<ObserveDiff> {
let old_selectors: HashSet<String> =
self.elements.iter().map(|e| e.selector.clone()).collect();
self.elements = observe::observe(&self.page, self.config.viewport_only).await?;
let new_selectors: HashSet<&str> =
self.elements.iter().map(|e| e.selector.as_str()).collect();
let added: Vec<usize> = self
.elements
.iter()
.filter(|e| !old_selectors.contains(&e.selector))
.map(|e| e.index)
.collect();
let removed = old_selectors
.iter()
.filter(|s| !new_selectors.contains(s.as_str()))
.count();
Ok(ObserveDiff {
added,
removed,
total: self.elements.len(),
})
}
pub fn added_element_list(&self, diff: &ObserveDiff) -> String {
let mut out = String::new();
for &idx in &diff.added {
if let Some(el) = self.elements.get(idx) {
out.push_str(&el.to_string());
out.push('\n');
}
}
out
}
pub async fn screenshot_plain(&self) -> Result<Vec<u8>> {
self.page.screenshot().await
}
async fn require_fresh(&mut self, index: usize) -> Result<&InteractiveElement> {
let stored = self.elements.get(index).cloned();
if let Some(ref el) = stored {
let js = format!(
"!!document.querySelector({})",
serde_json::to_string(&el.selector).unwrap()
);
let exists: bool = self.page.evaluate(&js).await.unwrap_or(false);
if exists {
return self.elements.get(index).ok_or_else(|| {
crate::eoka::Error::ElementNotFound(format!("element [{}] disappeared", index))
});
}
self.observe().await?;
if let Some(new_idx) = self
.elements
.iter()
.position(|e| e.fingerprint == el.fingerprint)
{
return Err(crate::eoka::Error::ElementNotFound(format!(
"element [{}] \"{}\" moved to [{}] - call observe() to refresh",
index, el.text, new_idx
)));
}
return Err(crate::eoka::Error::ElementNotFound(format!(
"element [{}] \"{}\" no longer exists on page",
index, el.text
)));
}
Err(crate::eoka::Error::ElementNotFound(format!(
"element [{}] not found (observed {} elements)",
index,
self.elements.len()
)))
}
pub async fn click(&mut self, index: usize) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::Click {
target: eoka::LocatorSpec::css(selector),
human: false,
double: false,
})
.await
}
pub async fn fill(&mut self, index: usize, text: &str) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::Fill {
target: eoka::LocatorSpec::css(selector),
value: text.into(),
human: false,
})
.await
}
pub async fn select(&mut self, index: usize, value: &str) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::SelectOption {
target: eoka::LocatorSpec::css(selector),
value: value.into(),
})
.await
}
pub async fn hover(&mut self, index: usize) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::Hover {
target: eoka::LocatorSpec::css(selector),
human: false,
})
.await
}
pub async fn scroll_to(&mut self, index: usize) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::ScrollIntoView {
target: eoka::LocatorSpec::css(selector),
})
.await
}
pub async fn try_click(&mut self, index: usize) -> Result<bool> {
let selector = self.require_fresh(index).await?.selector.clone();
let mut options = self.action_options.clone();
options.wait_for_target = false;
match self
.perform_with_options(
eoka::Action::Click {
target: eoka::LocatorSpec::css(selector),
human: false,
double: false,
},
options,
)
.await
{
Ok(()) => Ok(true),
Err(Error::Action { code, receipt, .. })
if code == "target_unavailable"
&& receipt.outcome == eoka::ActionOutcome::NotDispatched =>
{
Ok(false)
}
Err(error) => Err(error),
}
}
pub async fn human_click(&mut self, index: usize) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::Click {
target: eoka::LocatorSpec::css(selector),
human: true,
double: false,
})
.await
}
pub async fn human_fill(&mut self, index: usize, text: &str) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::Fill {
target: eoka::LocatorSpec::css(selector),
value: text.into(),
human: true,
})
.await
}
pub async fn focus(&mut self, index: usize) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::Focus {
target: eoka::LocatorSpec::css(selector),
})
.await
}
pub async fn submit(&mut self, index: usize) -> Result<()> {
let selector = self.require_fresh(index).await?.selector.clone();
self.perform_and_wait(eoka::Action::Submit {
target: eoka::LocatorSpec::css(selector),
})
.await
}
pub async fn options(&mut self, index: usize) -> Result<Vec<(String, String)>> {
let el = self.require_fresh(index).await?;
let selector = el.selector.clone();
let js = format!(
r#"(() => {{
const sel = document.querySelector({});
if (!sel || !sel.options) return '[]';
return JSON.stringify(Array.from(sel.options).map(o => [o.value, o.text]));
}})()"#,
serde_json::to_string(&selector).unwrap()
);
let json_str: String = self.page.evaluate(&js).await?;
let pairs: Vec<(String, String)> = serde_json::from_str(&json_str)
.map_err(|e| crate::eoka::Error::cdp_msg(format!("options parse error: {}", e)))?;
Ok(pairs)
}
pub async fn goto(&mut self, url: &str) -> Result<()> {
self.perform_and_wait(eoka::Action::Navigate {
url: url.into(),
referrer: None,
})
.await
}
pub async fn back(&mut self) -> Result<()> {
self.perform_and_wait(eoka::Action::Back).await
}
pub async fn forward(&mut self) -> Result<()> {
self.perform_and_wait(eoka::Action::Forward).await
}
pub async fn reload(&mut self) -> Result<()> {
self.perform_and_wait(eoka::Action::Reload).await
}
pub async fn url(&self) -> Result<String> {
self.page.url().await
}
pub async fn title(&self) -> Result<String> {
self.page.title().await
}
pub async fn text(&self) -> Result<String> {
self.page.text().await
}
pub async fn scroll_down(&self) -> Result<()> {
self.page
.execute("window.scrollBy(0, window.innerHeight * 0.8)")
.await
}
pub async fn scroll_up(&self) -> Result<()> {
self.page
.execute("window.scrollBy(0, -window.innerHeight * 0.8)")
.await
}
pub async fn scroll_to_top(&self) -> Result<()> {
self.page.execute("window.scrollTo(0, 0)").await
}
pub async fn scroll_to_bottom(&self) -> Result<()> {
self.page
.execute("window.scrollTo(0, document.body.scrollHeight)")
.await
}
pub async fn wait_for_stable(&self) -> Result<()> {
let _ = self.page.wait_for_network_idle(200, 2000).await;
self.page.wait(50).await;
Ok(())
}
pub async fn wait(&self, ms: u64) {
self.page.wait(ms).await;
}
pub async fn wait_for_text(&self, text: &str, timeout_ms: u64) -> Result<()> {
self.page.wait_for_text(text, timeout_ms).await?;
Ok(())
}
pub async fn wait_for_url(&self, pattern: &str, timeout_ms: u64) -> Result<()> {
self.page.wait_for_url_contains(pattern, timeout_ms).await
}
pub async fn wait_for_idle(&self, timeout_ms: u64) -> Result<()> {
self.page.wait_for_network_idle(500, timeout_ms).await
}
pub async fn press_key(&self, key: &str) -> Result<()> {
self.perform_and_wait(eoka::Action::PressKey { key: key.into() })
.await
}
pub async fn mouse_down(&self, x: f64, y: f64, button: MouseButton) -> Result<()> {
self.page.mouse_down(x, y, button).await
}
pub async fn mouse_move(&self, x: f64, y: f64) -> Result<()> {
self.page.mouse_move(x, y).await
}
pub async fn mouse_up(&self, x: f64, y: f64, button: MouseButton) -> Result<()> {
self.page.mouse_up(x, y, button).await
}
pub async fn key_down(&self, key: &str) -> Result<()> {
self.page.key_down(key).await
}
pub async fn key_up(&self, key: &str) -> Result<()> {
self.page.key_up(key).await
}
pub async fn release_all_inputs(&self) -> Result<()> {
self.page.release_all_inputs().await
}
pub async fn eval<T: serde::de::DeserializeOwned>(&self, js: &str) -> Result<T> {
self.page.evaluate(js).await
}
pub async fn exec(&self, js: &str) -> Result<()> {
self.page.execute(js).await
}
pub async fn extract<T: serde::de::DeserializeOwned>(&self, js_expression: &str) -> Result<T> {
let escaped_js = serde_json::to_string(js_expression)
.map_err(|e| crate::eoka::Error::cdp_msg(format!("Failed to escape JS: {}", e)))?;
let js = format!("JSON.stringify(eval({}))", escaped_js);
let json_str: String = self.page.evaluate(&js).await?;
if json_str == "null" || json_str == "undefined" || json_str.is_empty() {
return Err(crate::eoka::Error::cdp_msg(format!(
"extract returned null/undefined for: {}",
if js_expression.len() > 60 {
&js_expression[..60]
} else {
js_expression
}
)));
}
serde_json::from_str(&json_str).map_err(|e| {
crate::eoka::Error::cdp_msg(format!(
"extract parse error: {} (got: {})",
e,
if json_str.len() > 80 {
&json_str[..80]
} else {
&json_str
}
))
})
}
pub async fn spa_info(&self) -> Result<SpaRouterInfo> {
spa::detect_router(&self.page).await
}
pub async fn spa_navigate(&mut self, path: &str) -> Result<String> {
let info = spa::detect_router(&self.page).await?;
let result = spa::spa_navigate(&self.page, &info.router_type, path).await?;
self.elements.clear();
Ok(result)
}
pub async fn history_go(&mut self, delta: i32) -> Result<()> {
self.perform_and_wait(eoka::Action::History { delta })
.await?;
self.elements.clear();
Ok(())
}
pub async fn close(self) -> Result<()> {
self.operations.stop().await;
let release_result = self.page.release_all_inputs().await;
self.browser.close().await?;
release_result
}
}
#[cfg(test)]
mod tests {
use super::*;
#[allow(clippy::too_many_arguments)]
fn make_element(
index: usize,
tag: &str,
text: &str,
role: Option<&str>,
input_type: Option<&str>,
placeholder: Option<&str>,
value: Option<&str>,
checked: bool,
) -> InteractiveElement {
let selector = format!("[data-idx=\"{}\"]", index);
let fingerprint = InteractiveElement::compute_fingerprint(
tag,
text,
role,
input_type,
placeholder,
&selector,
);
InteractiveElement {
index,
tag: tag.to_string(),
text: text.to_string(),
role: role.map(|s| s.to_string()),
input_type: input_type.map(|s| s.to_string()),
placeholder: placeholder.map(|s| s.to_string()),
value: value.map(|s| s.to_string()),
checked,
selector,
disabled: false,
visible: true,
bbox: BoundingBox {
x: 0.0,
y: 0.0,
width: 100.0,
height: 30.0,
},
fingerprint,
}
}
#[test]
fn test_element_display_basic() {
let el = make_element(0, "button", "Submit", None, None, None, None, false);
assert_eq!(el.to_string(), "[0] <button> \"Submit\"");
}
#[test]
fn test_element_display_with_input_type() {
let el = make_element(0, "input", "", None, Some("text"), None, None, false);
assert_eq!(el.to_string(), "[0] <input>");
let el = make_element(0, "input", "", None, Some("password"), None, None, false);
assert_eq!(el.to_string(), "[0] <input type=\"password\">");
}
#[test]
fn test_element_display_with_placeholder() {
let el = make_element(
0,
"input",
"",
None,
Some("text"),
Some("Enter email"),
None,
false,
);
assert_eq!(el.to_string(), "[0] <input> placeholder=\"Enter email\"");
}
#[test]
fn test_element_display_with_value() {
let el = make_element(
0,
"input",
"",
None,
Some("text"),
None,
Some("hello"),
false,
);
assert_eq!(el.to_string(), "[0] <input> value=\"hello\"");
}
#[test]
fn test_element_display_checked() {
let el = make_element(0, "input", "", None, Some("checkbox"), None, None, true);
assert_eq!(el.to_string(), "[0] <input type=\"checkbox\"> [checked]");
}
#[test]
fn test_element_display_redundant_role_suppressed() {
let el = make_element(
0,
"button",
"Click",
Some("button"),
None,
None,
None,
false,
);
assert_eq!(el.to_string(), "[0] <button> \"Click\"");
let el = make_element(0, "a", "Link", Some("link"), None, None, None, false);
assert_eq!(el.to_string(), "[0] <a> \"Link\"");
let el = make_element(0, "a", "Menu", Some("menuitem"), None, None, None, false);
assert_eq!(el.to_string(), "[0] <a> \"Menu\"");
}
#[test]
fn test_element_display_non_redundant_role_shown() {
let el = make_element(0, "button", "Tab 1", Some("tab"), None, None, None, false);
assert_eq!(el.to_string(), "[0] <button> \"Tab 1\" role=\"tab\"");
let el = make_element(0, "div", "Click", Some("button"), None, None, None, false);
assert_eq!(el.to_string(), "[0] <div> \"Click\" role=\"button\"");
}
#[test]
fn test_observe_diff_display_no_changes() {
let diff = ObserveDiff {
added: vec![],
removed: 0,
total: 5,
};
assert_eq!(diff.to_string(), "no changes (5 elements)");
}
#[test]
fn test_observe_diff_display_added_only() {
let diff = ObserveDiff {
added: vec![5, 6],
removed: 0,
total: 7,
};
assert_eq!(diff.to_string(), "+2 added (7 total)");
}
#[test]
fn test_observe_diff_display_removed_only() {
let diff = ObserveDiff {
added: vec![],
removed: 3,
total: 2,
};
assert_eq!(diff.to_string(), "-3 removed (2 total)");
}
#[test]
fn test_observe_diff_display_both() {
let diff = ObserveDiff {
added: vec![3, 4],
removed: 1,
total: 5,
};
assert_eq!(diff.to_string(), "+2 added, -1 removed (5 total)");
}
#[test]
fn test_observe_config_default() {
let config = ObserveConfig::default();
assert!(config.viewport_only);
}
#[test]
fn test_fingerprint_uses_full_selector() {
let base = "a".repeat(50);
let sel_a = format!("{}AAAA", base);
let sel_b = format!("{}BBBB", base);
let fp_a = InteractiveElement::compute_fingerprint("button", "X", None, None, None, &sel_a);
let fp_b = InteractiveElement::compute_fingerprint("button", "X", None, None, None, &sel_b);
assert_ne!(
fp_a, fp_b,
"selectors differing after char 50 should produce different fingerprints"
);
}
}