use blitz_control_protocol::{AgentAction, AgentControlRequest, AgentSnapshot, SemanticNode};
use eyre::{Result, bail};
use std::collections::HashSet;
use crate::inspector::{Client, inspect};
use crate::{cli, reach};
pub(crate) fn resolved_action_target(nodes: &[SemanticNode], want: &str) -> Option<String> {
nodes
.iter()
.find(|node| {
selector_matches_node(node, want)
&& node.enabled
&& node.visible
&& painted_bounds(node).is_some()
})
.map(|node| node.name.clone())
}
pub(crate) fn painted_bounds(node: &SemanticNode) -> Option<[f64; 4]> {
node.bounds
.filter(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
}
fn targetable_bounds(node: &SemanticNode) -> Option<[f64; 4]> {
match painted_bounds(node) {
Some(bounds) => Some(bounds),
None if cli::headless() => node.bounds,
None => None,
}
}
pub(crate) fn painted_named(nodes: &[SemanticNode], want: &str) -> bool {
nodes
.iter()
.any(|node| selector_matches_node(node, want) && painted_bounds(node).is_some())
}
pub(crate) fn name_matches(name: &str, pattern: &str) -> bool {
let (name, pattern) = (name.to_lowercase(), pattern.to_lowercase());
if pattern == "*" {
return true;
}
let Some(stripped) = pattern.strip_prefix('*') else {
return match pattern.strip_suffix('*') {
Some(prefix) => name.starts_with(prefix),
None => name.contains(&pattern),
};
};
match stripped.strip_suffix('*') {
Some(middle) => name.contains(middle),
None => name.ends_with(stripped),
}
}
pub(crate) fn viewport_of(snapshot: &AgentSnapshot) -> (f64, f64) {
if let Some(bounds) = snapshot.viewport {
return (bounds[1], bounds[1] + bounds[3]);
}
viewport_of_nodes(&snapshot.nodes)
}
pub(crate) fn viewport_of_nodes(nodes: &[SemanticNode]) -> (f64, f64) {
let window = nodes
.iter()
.filter(|node| node.parent.is_none())
.filter_map(|node| node.bounds)
.max_by(|a, b| {
(a[2] * a[3])
.partial_cmp(&(b[2] * b[3]))
.unwrap_or(std::cmp::Ordering::Equal)
});
if let Some(bounds) = window {
return (bounds[1], bounds[1] + bounds[3]);
}
let bottom = nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| b[1] + b[3])
.fold(f64::MIN, f64::max);
(0.0, if bottom > f64::MIN { bottom } else { f64::MAX })
}
pub(crate) fn viewport_for_node(snapshot: &AgentSnapshot, node_id: u64) -> (f64, f64) {
if snapshot
.nodes
.iter()
.find(|node| node.id == node_id)
.is_some_and(|node| node.viewport_fixed)
&& let Some(bounds) = snapshot.viewport
{
return (bounds[1], bounds[1] + bounds[3]);
}
let inferred = viewport_for_node_in(&snapshot.nodes, node_id);
if let Some(bounds) = snapshot.viewport {
let window = (bounds[1], bounds[1] + bounds[3]);
return (inferred.0.max(window.0), inferred.1.min(window.1));
}
inferred
}
pub(crate) fn viewport_for_node_in(nodes: &[SemanticNode], node_id: u64) -> (f64, f64) {
let mut cursor = Some(node_id);
let mut root_bounds = None;
let mut main_bounds = None;
for _ in 0..32 {
let Some(id) = cursor else { break };
let Some(node) = nodes.iter().find(|node| node.id == id) else {
break;
};
if node.role == "main"
&& let Some(bounds) = node.bounds
{
main_bounds = Some(bounds);
}
if node.parent.is_none() {
root_bounds = node.bounds;
}
cursor = node.parent;
}
if let Some(bounds) = root_bounds {
let window = (bounds[1], bounds[1] + bounds[3]);
if let Some(main) = main_bounds {
return (window.0.max(main[1]), window.1.min(main[1] + main[3]));
}
return window;
}
viewport_of_nodes(nodes)
}
pub(crate) fn offscreen(bounds: [f64; 4], viewport: (f64, f64)) -> bool {
bounds[1] + bounds[3] < viewport.0 || bounds[0] + bounds[2] < 0.0 || bounds[1] > viewport.1
}
pub(crate) async fn locate_control(
client: &mut Client,
want: &str,
roles: &[&str],
) -> Result<(u64, [f64; 4])> {
let pick = |snapshot: &AgentSnapshot| -> Option<(u64, [f64; 4])> {
let modal_scope: HashSet<u64> = reach::dismissers(&snapshot.nodes)
.first()
.map(|(id, _)| reach::enclosing_dialog(&snapshot.nodes, *id))
.unwrap_or_default()
.into_iter()
.collect();
let surface_scope: HashSet<u64> = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(&snapshot.nodes, surface))
.map(|surface| reach::on_surface_subtree(&snapshot.nodes, surface))
.unwrap_or_default()
.into_iter()
.collect();
let mut candidates: Vec<_> = snapshot
.nodes
.iter()
.filter(|n| {
selector_dom_id(want).is_some()
|| selector_slot(want).is_some()
|| selector_role(want).is_some()
|| roles.contains(&"*")
|| roles.is_empty() && reach::interactive(n)
|| roles.contains(&n.role.as_str())
})
.filter(|n| selector_matches_node(n, want))
.filter(|node| node.visible || cli::headless())
.filter_map(|node| targetable_bounds(node).map(|bounds| (node, bounds)))
.collect();
retain_exact_candidates(&mut candidates, want);
candidates.retain(|(node, _)| node.enabled);
prioritize_actionable_candidates(&mut candidates);
for scope in [&modal_scope, &surface_scope] {
if let Some((node, bounds)) = candidates.iter().find(|(node, bounds)| {
scope.contains(&node.id)
&& !offscreen(*bounds, viewport_for_node(snapshot, node.id))
}) {
return Some((node.id, *bounds));
}
}
if let Some((node, bounds)) = candidates
.iter()
.find(|(node, bounds)| !offscreen(*bounds, viewport_for_node(snapshot, node.id)))
{
return Some((node.id, *bounds));
}
let mut fallback = None;
for (node, bounds) in candidates {
let recoverable = bounds[0] + bounds[2] > 0.0;
if recoverable && !fallback.is_some_and(|(_, b): (u64, [f64; 4])| b[0] + b[2] > 0.0) {
fallback = Some((node.id, bounds));
} else {
fallback.get_or_insert((node.id, bounds));
}
}
fallback
};
let (snapshot, _) = inspect(client).await?;
let Some((id, bounds)) = pick(&snapshot) else {
bail!("no visible, enabled, sized semantic control matching it");
};
let viewport = viewport_for_node(&snapshot, id);
if !offscreen(bounds, viewport) {
return Ok((id, bounds));
}
let mut target = (id, bounds);
let mut latest = snapshot;
for _ in 0..4 {
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
let (settled, _) = inspect(client).await?;
if let Some(found) = pick(&settled) {
target = found;
let viewport = viewport_for_node(&settled, target.0);
if !offscreen(target.1, viewport) {
return Ok(target);
}
}
latest = settled;
}
if cli::trace() {
println!(
" {want:?} is off-screen at {:?}, scrolling it in",
target.1
);
}
for _ in 0..4 {
for node_id in reach::reveal_chain(&latest.nodes, target.0) {
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id,
}))
.await?;
}
let (settled, _) = inspect(client).await?;
let Some(found) = pick(&settled) else {
bail!("no visible, enabled, sized semantic control matching it");
};
target = found;
let viewport = viewport_for_node(&settled, target.0);
if !offscreen(target.1, viewport) {
return Ok(target);
}
let delta_y = if target.1[1] > viewport.1 {
target.1[1] + target.1[3] - viewport.1 + 16.0
} else if target.1[1] + target.1[3] < viewport.0 {
target.1[1] - viewport.0 - 16.0
} else {
0.0
};
if delta_y != 0.0
&& let Some(node_id) = reach::reveal_chain(&settled.nodes, target.0).first()
{
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollBy {
node_id: *node_id,
delta_x: 0.0,
delta_y,
}))
.await?;
}
latest = settled;
}
bail!(
"{want:?} is still off-screen at {:?} after four semantic reveal attempts",
target.1
)
}
fn prioritize_actionable_candidates(candidates: &mut Vec<(&SemanticNode, [f64; 4])>) {
let has_visible_painted = candidates
.iter()
.any(|(node, _)| node.visible && painted_bounds(node).is_some());
if has_visible_painted {
candidates.retain(|(node, _)| node.visible && painted_bounds(node).is_some());
} else {
candidates.sort_by_key(|(node, _)| (!node.visible, painted_bounds(node).is_none()));
}
}
fn selector_slot(selector: &str) -> Option<&str> {
selector.strip_prefix('@')
}
fn selector_dom_id(selector: &str) -> Option<&str> {
selector.strip_prefix('#').filter(|id| !id.is_empty())
}
pub(crate) fn selector_matches_node(node: &SemanticNode, selector: &str) -> bool {
if let Some(dom_id) = selector_dom_id(selector) {
return node.dom_id.as_deref() == Some(dom_id);
}
if let Some(slot) = selector_slot(selector) {
return node.slot.as_deref() == Some(slot);
}
if let Some((role, name)) = selector.split_once(':')
&& role.eq_ignore_ascii_case(&node.role)
{
return name_matches(&node.name, name);
}
name_matches(&node.name, selector)
}
pub(crate) fn selector_role(selector: &str) -> Option<&str> {
if selector.starts_with('#') || selector.starts_with('@') {
return None;
}
let (role, name) = selector.split_once(':')?;
(!role.is_empty() && !name.is_empty() && !role.contains(char::is_whitespace)).then_some(role)
}
pub(crate) fn unmatched_selector_reason(nodes: &[SemanticNode], selector: &str) -> Option<String> {
if selector.is_empty()
|| nodes
.iter()
.any(|node| selector_matches_node(node, selector))
{
return None;
}
if let Some(dom_id) = selector_dom_id(selector) {
return Some(format!(
"no node carries the DOM id {dom_id:?}; drop the leading '#' to match an \
accessible name instead"
));
}
if let Some(slot) = selector_slot(selector) {
return Some(format!(
"no node carries the component slot {slot:?}; drop the leading '@' to match an \
accessible name instead"
));
}
let role = selector_role(selector)?;
if nodes
.iter()
.any(|node| node.role.eq_ignore_ascii_case(role))
{
return None;
}
let mut present: Vec<&str> = nodes.iter().map(|node| node.role.as_str()).collect();
present.sort_unstable();
present.dedup();
present.truncate(12);
Some(format!(
"no node has role {role:?}, so {selector:?} can never match; roles present: {}",
present.join(", ")
))
}
pub(crate) fn exact_selector_matches_node(node: &SemanticNode, selector: &str) -> bool {
if let Some(dom_id) = selector_dom_id(selector) {
return node.dom_id.as_deref() == Some(dom_id);
}
if let Some(slot) = selector_slot(selector) {
return node.slot.as_deref() == Some(slot);
}
if let Some((role, name)) = selector.split_once(':')
&& role.eq_ignore_ascii_case(&node.role)
{
return node.name.eq_ignore_ascii_case(name);
}
node.name.eq_ignore_ascii_case(selector)
}
pub(crate) fn retain_exact_candidates(
candidates: &mut Vec<(&SemanticNode, [f64; 4])>,
selector: &str,
) {
if candidates
.iter()
.any(|(node, _)| exact_selector_matches_node(node, selector))
{
candidates.retain(|(node, _)| exact_selector_matches_node(node, selector));
}
}
#[cfg(test)]
mod tests {
use super::*;
fn node(dom_id: Option<&str>, name: &str) -> SemanticNode {
SemanticNode {
dom_id: dom_id.map(str::to_owned),
id: 1,
parent: None,
role: "button".into(),
name: name.into(),
value: None,
enabled: true,
focusable: false,
viewport_fixed: false,
visible: true,
selected: false,
bounds: Some([0.0, 0.0, 20.0, 20.0]),
slot: Some("button".into()),
}
}
#[test]
fn dom_id_is_an_exact_stable_selector() {
let save = node(Some("settings-save"), "Save settings");
assert!(selector_matches_node(&save, "#settings-save"));
assert!(exact_selector_matches_node(&save, "#settings-save"));
assert!(!selector_matches_node(&save, "#settings"));
assert!(!selector_matches_node(&save, "#"));
}
#[test]
fn dom_id_does_not_fall_back_to_the_accessible_name() {
let anonymous = node(None, "settings-save");
assert!(!selector_matches_node(&anonymous, "#settings-save"));
}
#[test]
fn role_and_name_globs_have_one_case_insensitive_semantics() {
let save = node(None, "Save Settings");
assert!(selector_matches_node(&save, "BUTTON:save*"));
assert!(selector_matches_node(&save, "button:*settings"));
assert!(selector_matches_node(&save, "*VE SET*"));
assert!(painted_named(&[save], "BuTtOn:*SETTINGS"));
}
#[test]
fn a_bare_selector_matches_names_not_roles() {
let save = node(None, "Save settings");
assert!(!selector_matches_node(&save, "button"));
assert!(selector_matches_node(&save, "save"));
}
#[test]
fn a_visible_painted_copy_excludes_a_hidden_headless_fallback() {
let mut hidden = node(None, "Next page");
hidden.visible = false;
hidden.bounds = Some([0.0, 0.0, 0.0, 0.0]);
let mut desktop = node(None, "Next page");
desktop.id = 2;
desktop.bounds = Some([1200.0, 400.0, 48.0, 48.0]);
let mut candidates = vec![
(&hidden, hidden.bounds.expect("hidden bounds")),
(&desktop, desktop.bounds.expect("desktop bounds")),
];
prioritize_actionable_candidates(&mut candidates);
assert_eq!(candidates.len(), 1);
assert_eq!(candidates[0].0.id, desktop.id);
}
fn contains_either(node: &SemanticNode, want: &str) -> bool {
node.name.contains(want) || node.role.contains(want)
}
#[test]
fn a_role_selector_matched_nothing_under_a_substring_predicate() {
let link = SemanticNode {
role: "link".into(),
..node(None, "crates.vip")
};
let heading = SemanticNode {
role: "heading".into(),
..node(None, "A private registry")
};
let page = [link.clone(), heading.clone()];
assert!(
!page
.iter()
.any(|node| contains_either(node, "link:crates.vip")),
"the substring predicate silently selects an empty page"
);
assert!(selector_matches_node(&link, "link:crates.vip"));
assert!(!selector_matches_node(&heading, "link:crates.vip"));
assert!(unmatched_selector_reason(&page, "link:crates.vip").is_none());
}
#[test]
fn a_grammar_selector_the_tree_cannot_answer_is_reported_as_such() {
let page = [node(Some("save"), "Save settings")];
let role = unmatched_selector_reason(&page, "link:crates.vip")
.expect("this document has no links at all");
assert!(role.contains("role \"link\""), "{role}");
assert!(
role.contains("button"),
"the roles present are named: {role}"
);
let dom_id =
unmatched_selector_reason(&page, "#missing").expect("no node carries that DOM id");
assert!(dom_id.contains("DOM id"), "{dom_id}");
let slot = unmatched_selector_reason(&page, "@listbox").expect("no node carries that slot");
assert!(slot.contains("slot"), "{slot}");
assert!(unmatched_selector_reason(&page, "Nothing here").is_none());
assert!(unmatched_selector_reason(&page, "button:Nothing here").is_none());
assert!(unmatched_selector_reason(&page, "Default permission: Auto").is_none());
}
#[test]
fn a_colon_in_a_bare_name_is_not_misread_as_a_role() {
let permission = node(None, "Default permission: Auto");
assert!(selector_matches_node(
&permission,
"Default permission: Auto"
));
assert!(exact_selector_matches_node(
&permission,
"Default permission: Auto"
));
}
}