use blitz_control_protocol::{AgentAction, AgentControlRequest, AgentSnapshot, SemanticNode};
use eyre::{Result, bail};
use std::collections::HashSet;
use crate::inspector::{Client, inspect};
use crate::{cli, reach};
pub(crate) fn resolved_action_target(nodes: &[SemanticNode], want: &str) -> Option<String> {
nodes
.iter()
.find(|node| {
selector_matches_node(node, want)
&& node.enabled
&& node.visible
&& painted_bounds(node).is_some()
})
.map(|node| node.name.clone())
}
pub(crate) fn painted_bounds(node: &SemanticNode) -> Option<[f64; 4]> {
node.bounds
.filter(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
}
pub(crate) fn painted_named(nodes: &[SemanticNode], want: &str) -> bool {
nodes
.iter()
.any(|node| selector_matches_node(node, want) && painted_bounds(node).is_some())
}
pub(crate) fn name_matches(name: &str, pattern: &str) -> bool {
let (name, pattern) = (name.to_lowercase(), pattern.to_lowercase());
if pattern == "*" {
return true;
}
let Some(stripped) = pattern.strip_prefix('*') else {
return match pattern.strip_suffix('*') {
Some(prefix) => name.starts_with(prefix),
None => name.contains(&pattern),
};
};
match stripped.strip_suffix('*') {
Some(middle) => name.contains(middle),
None => name.ends_with(stripped),
}
}
pub(crate) fn viewport_of(snapshot: &AgentSnapshot) -> (f64, f64) {
let bottom = snapshot
.nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| b[1] + b[3])
.fold(f64::MIN, f64::max);
(0.0, if bottom > f64::MIN { bottom } else { f64::MAX })
}
pub(crate) fn viewport_for_node(snapshot: &AgentSnapshot, node_id: u64) -> (f64, f64) {
let mut cursor = Some(node_id);
for _ in 0..32 {
let Some(id) = cursor else { break };
let Some(node) = snapshot.nodes.iter().find(|node| node.id == id) else {
break;
};
if node.role == "main"
&& let Some(bounds) = node.bounds
{
return (bounds[1], bounds[1] + bounds[3]);
}
cursor = node.parent;
}
viewport_of(snapshot)
}
pub(crate) fn offscreen(bounds: [f64; 4], viewport: (f64, f64)) -> bool {
bounds[1] + bounds[3] < viewport.0 || bounds[0] + bounds[2] < 0.0 || bounds[1] > viewport.1
}
pub(crate) async fn locate_control(
client: &mut Client,
want: &str,
roles: &[&str],
) -> Result<(u64, [f64; 4])> {
let pick = |snapshot: &AgentSnapshot| -> Option<(u64, [f64; 4])> {
let modal_scope: HashSet<u64> = reach::dismissers(&snapshot.nodes)
.first()
.map(|(id, _)| reach::enclosing_dialog(&snapshot.nodes, *id))
.unwrap_or_default()
.into_iter()
.collect();
let surface_scope: HashSet<u64> = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(&snapshot.nodes, surface))
.map(|surface| reach::on_surface_subtree(&snapshot.nodes, surface))
.unwrap_or_default()
.into_iter()
.collect();
let mut candidates: Vec<_> = snapshot
.nodes
.iter()
.filter(|n| {
roles.contains(&"*")
|| roles.is_empty() && reach::interactive(n)
|| roles.contains(&n.role.as_str())
})
.filter(|n| selector_matches_node(n, want))
.filter(|node| node.visible)
.filter_map(|node| painted_bounds(node).map(|bounds| (node, bounds)))
.collect();
retain_exact_candidates(&mut candidates, want);
candidates.retain(|(node, _)| node.enabled);
for scope in [&modal_scope, &surface_scope] {
if let Some((node, bounds)) = candidates.iter().find(|(node, bounds)| {
scope.contains(&node.id)
&& !offscreen(*bounds, viewport_for_node(snapshot, node.id))
}) {
return Some((node.id, *bounds));
}
}
if let Some((node, bounds)) = candidates
.iter()
.find(|(node, bounds)| !offscreen(*bounds, viewport_for_node(snapshot, node.id)))
{
return Some((node.id, *bounds));
}
let mut fallback = None;
for (node, bounds) in candidates {
let recoverable = bounds[0] + bounds[2] > 0.0;
if recoverable && !fallback.is_some_and(|(_, b): (u64, [f64; 4])| b[0] + b[2] > 0.0) {
fallback = Some((node.id, bounds));
} else {
fallback.get_or_insert((node.id, bounds));
}
}
fallback
};
let (snapshot, _) = inspect(client).await?;
let Some((id, bounds)) = pick(&snapshot) else {
bail!("no visible, enabled, sized semantic control matching it");
};
let viewport = viewport_for_node(&snapshot, id);
if !offscreen(bounds, viewport) {
return Ok((id, bounds));
}
if cli::trace() {
println!(" {want:?} is off-screen at {bounds:?}, scrolling it in");
}
let mut target = (id, bounds);
let mut latest = snapshot;
for _ in 0..4 {
for node_id in reach::reveal_chain(&latest.nodes, target.0) {
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id,
}))
.await?;
}
let (settled, _) = inspect(client).await?;
let Some(found) = pick(&settled) else {
bail!("no visible, enabled, sized semantic control matching it");
};
target = found;
let viewport = viewport_for_node(&settled, target.0);
if !offscreen(target.1, viewport) {
return Ok(target);
}
let delta_y = if target.1[1] > viewport.1 {
target.1[1] + target.1[3] - viewport.1 + 16.0
} else if target.1[1] + target.1[3] < viewport.0 {
target.1[1] - viewport.0 - 16.0
} else {
0.0
};
if delta_y != 0.0
&& let Some(node_id) = reach::reveal_chain(&settled.nodes, target.0).first()
{
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollBy {
node_id: *node_id,
delta_x: 0.0,
delta_y,
}))
.await?;
}
latest = settled;
}
bail!(
"{want:?} is still off-screen at {:?} after four semantic reveal attempts",
target.1
)
}
fn selector_slot(selector: &str) -> Option<&str> {
selector.strip_prefix('@')
}
fn selector_dom_id(selector: &str) -> Option<&str> {
selector.strip_prefix('#').filter(|id| !id.is_empty())
}
pub(crate) fn selector_matches_node(node: &SemanticNode, selector: &str) -> bool {
if let Some(dom_id) = selector_dom_id(selector) {
return node.dom_id.as_deref() == Some(dom_id);
}
if let Some(slot) = selector_slot(selector) {
return node.slot.as_deref() == Some(slot);
}
if let Some((role, name)) = selector.split_once(':')
&& role.eq_ignore_ascii_case(&node.role)
{
return name_matches(&node.name, name);
}
name_matches(&node.name, selector)
}
pub(crate) fn exact_selector_matches_node(node: &SemanticNode, selector: &str) -> bool {
if let Some(dom_id) = selector_dom_id(selector) {
return node.dom_id.as_deref() == Some(dom_id);
}
if let Some(slot) = selector_slot(selector) {
return node.slot.as_deref() == Some(slot);
}
if let Some((role, name)) = selector.split_once(':')
&& role.eq_ignore_ascii_case(&node.role)
{
return node.name.eq_ignore_ascii_case(name);
}
node.name.eq_ignore_ascii_case(selector)
}
pub(crate) fn retain_exact_candidates(
candidates: &mut Vec<(&SemanticNode, [f64; 4])>,
selector: &str,
) {
if candidates
.iter()
.any(|(node, _)| exact_selector_matches_node(node, selector))
{
candidates.retain(|(node, _)| exact_selector_matches_node(node, selector));
}
}
#[cfg(test)]
mod tests {
use super::*;
fn node(dom_id: Option<&str>, name: &str) -> SemanticNode {
SemanticNode {
dom_id: dom_id.map(str::to_owned),
id: 1,
parent: None,
role: "button".into(),
name: name.into(),
value: None,
enabled: true,
visible: true,
selected: false,
bounds: Some([0.0, 0.0, 20.0, 20.0]),
slot: Some("button".into()),
}
}
#[test]
fn dom_id_is_an_exact_stable_selector() {
let save = node(Some("settings-save"), "Save settings");
assert!(selector_matches_node(&save, "#settings-save"));
assert!(exact_selector_matches_node(&save, "#settings-save"));
assert!(!selector_matches_node(&save, "#settings"));
assert!(!selector_matches_node(&save, "#"));
}
#[test]
fn dom_id_does_not_fall_back_to_the_accessible_name() {
let anonymous = node(None, "settings-save");
assert!(!selector_matches_node(&anonymous, "#settings-save"));
}
#[test]
fn role_and_name_globs_have_one_case_insensitive_semantics() {
let save = node(None, "Save Settings");
assert!(selector_matches_node(&save, "BUTTON:save*"));
assert!(selector_matches_node(&save, "button:*settings"));
assert!(selector_matches_node(&save, "*VE SET*"));
assert!(painted_named(&[save], "BuTtOn:*SETTINGS"));
}
#[test]
fn a_bare_selector_matches_names_not_roles() {
let save = node(None, "Save settings");
assert!(!selector_matches_node(&save, "button"));
assert!(selector_matches_node(&save, "save"));
}
#[test]
fn a_colon_in_a_bare_name_is_not_misread_as_a_role() {
let permission = node(None, "Default permission: Auto");
assert!(selector_matches_node(
&permission,
"Default permission: Auto"
));
assert!(exact_selector_matches_node(
&permission,
"Default permission: Auto"
));
}
}