use std::collections::HashSet;
use std::time::Instant;
use blitz_control_protocol::{
AgentAction, AgentControlRequest, InputCommand, KeyPhase, Modifiers, PointerPhase,
SemanticNode, WheelPhase,
};
use eyre::{Result, bail, eyre};
use crate::diagnostics::metrics_if_supported;
use crate::inspector::{Client, inspect};
use crate::target::{locate_control, selector_matches_node};
use crate::timing::{pace, sleep_pace};
use crate::{cli, reach, report};
pub(crate) async fn hover_over(client: &mut Client, want: &str) -> Result<bool> {
if let Some((x, y)) = want.split_once(',')
&& let (Ok(x), Ok(y)) = (x.trim().parse::<f64>(), y.trim().parse::<f64>())
{
{
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Pointer {
phase: PointerPhase::Move,
x,
y,
button: 0,
modifiers: Modifiers::default(),
},
)))
.await?;
println!("pointer at {x:.0},{y:.0}");
return Ok(true);
}
}
let (snapshot, _) = inspect(client).await?;
let Some(node) = snapshot
.nodes
.iter()
.filter(|node| node.visible && selector_matches_node(node, want))
.filter_map(|node| node.bounds.map(|bounds| (node, bounds)))
.max_by(|a, b| {
(a.1[2] * a.1[3])
.partial_cmp(&(b.1[2] * b.1[3]))
.unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(node, bounds)| (node.name.clone(), bounds))
else {
println!("no visible node named {want:?} to hover; wheel goes wherever it goes");
return Ok(false);
};
let (name, bounds) = node;
let (x, y) = (bounds[0] + bounds[2] / 2.0, bounds[1] + bounds[3] / 2.0);
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Pointer {
phase: PointerPhase::Move,
x,
y,
button: 0,
modifiers: Modifiers::default(),
},
)))
.await?;
println!("pointer over {name:?} at {x:.0},{y:.0}");
Ok(true)
}
pub(crate) async fn pointer_drag(
client: &mut Client,
want: &str,
dx: f64,
dy: f64,
steps: u32,
cancel: bool,
to: Option<&str>,
) -> Result<()> {
if !dx.is_finite() || !dy.is_finite() || steps == 0 || steps > 240 {
bail!("pointer drag needs finite offsets and 1..=240 steps");
}
let destination = if let Some(to) = to {
let (_, bounds) = locate_control(client, to, &[]).await?;
Some((bounds[0] + bounds[2] / 2.0, bounds[1] + bounds[3] / 2.0))
} else {
None
};
let (_, bounds) = locate_control(client, want, &[]).await?;
let start = (bounds[0] + bounds[2] / 2.0, bounds[1] + bounds[3] / 2.0);
let (dx, dy) = destination.map_or((dx, dy), |end| (end.0 - start.0, end.1 - start.1));
let request = |phase, x, y| {
AgentControlRequest::Act(AgentAction::Input(InputCommand::Pointer {
phase,
x,
y,
button: 0,
modifiers: Modifiers::default(),
}))
};
for phase in [PointerPhase::Move, PointerPhase::Down] {
client.agent(&request(phase, start.0, start.1)).await?;
}
let mut position = start;
for step in 1..=steps {
let fraction = f64::from(step) / f64::from(steps);
position = (start.0 + dx * fraction, start.1 + dy * fraction);
if let Err(error) = client
.agent(&request(PointerPhase::Move, position.0, position.1))
.await
{
let _ = client
.agent(&request(PointerPhase::Cancel, position.0, position.1))
.await;
return Err(error);
}
sleep_pace().await;
}
client
.agent(&request(
if cancel {
PointerPhase::Cancel
} else {
PointerPhase::Up
},
position.0,
position.1,
))
.await?;
println!(
"pointer drag {want:?}: {:.1},{:.1} -> {:.1},{:.1} ({})",
start.0,
start.1,
position.0,
position.1,
if cancel { "cancelled" } else { "released" }
);
Ok(())
}
pub(crate) async fn scroll(client: &mut Client, ticks: usize, delta: f64) -> Result<()> {
let pace = pace();
if pace.is_zero() {
println!("pace: unpaced (--pace 0) - measures app throughput");
} else {
println!(
"pace: {:.2}ms between events ({:.0} Hz requested); fps and missed_refreshes \
below describe this pace, not the app's limit. Pass --pace 0 to remove it",
pace.as_secs_f64() * 1000.0,
1.0 / pace.as_secs_f64(),
);
}
let mut latencies = scroll_events(client, ticks, delta).await?;
report::show_latencies("wheel events", ticks, &mut latencies);
Ok(())
}
pub(crate) async fn scroll_events(
client: &mut Client,
ticks: usize,
delta: f64,
) -> Result<Vec<f64>> {
let mut latencies = Vec::with_capacity(ticks);
for _ in 0..ticks {
let started = Instant::now();
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Wheel {
delta_x: 0.0,
delta_y: delta,
phase: WheelPhase::Moved,
modifiers: Modifiers::default(),
},
)))
.await?;
latencies.push(started.elapsed().as_secs_f64() * 1000.0);
sleep_pace().await;
}
Ok(latencies)
}
fn is_text_field_role(role: &str) -> bool {
matches!(
role,
"textbox" | "textarea" | "input" | "combobox" | "searchbox" | "spinbutton"
)
}
fn find_text_field<'a>(nodes: &'a [SemanticNode], want: &str) -> Option<&'a SemanticNode> {
let modal_scope: HashSet<u64> = reach::dismissers(nodes)
.first()
.map(|(id, _)| reach::enclosing_dialog(nodes, *id))
.unwrap_or_default()
.into_iter()
.collect();
let surface_scope: HashSet<u64> = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(nodes, surface))
.map(|surface| reach::on_surface_subtree(nodes, surface))
.unwrap_or_default()
.into_iter()
.collect();
let fields: Vec<&SemanticNode> = nodes
.iter()
.filter(|node| {
is_text_field_role(node.role.as_str())
&& node.enabled
&& node
.bounds
.is_some_and(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
})
.collect();
let matches_name = |node: &&SemanticNode| want.is_empty() || selector_matches_node(node, want);
for scope in [&modal_scope, &surface_scope] {
if let Some(field) = fields
.iter()
.find(|node| matches_name(node) && scope.contains(&node.id))
{
return Some(field);
}
}
fields.into_iter().find(matches_name)
}
pub(crate) async fn press_key(
client: &mut Client,
name: &str,
count: usize,
over: &str,
require_target: bool,
) -> Result<()> {
let (snapshot, _) = inspect(client).await?;
let by_id = over.parse::<u64>().ok().filter(|id| {
snapshot.nodes.iter().any(|node| {
node.id == *id
&& node
.bounds
.is_some_and(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
})
});
let target = if by_id.is_some() || over.is_empty() {
by_id
} else {
match locate_control(client, over, &[]).await {
Ok((node_id, _)) => Some(node_id),
Err(error) if require_target => {
bail!("no enabled, painted key target matching {over:?}: {error}")
}
Err(_) => None,
}
};
if let Some(target) = target {
client
.agent(&AgentControlRequest::Act(AgentAction::Focus {
node_id: target,
}))
.await?;
println!("focused node {target} for {name} x{count}");
} else if require_target {
bail!("no enabled, painted key target matching {over:?}");
} else {
println!("no visible node named {over:?}; sending {name} to whatever has focus");
}
let (key, code, modifiers) = parse_key_chord(name)?;
for _ in 0..count {
for phase in [KeyPhase::Down, KeyPhase::Up] {
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Key {
phase,
key: key.clone(),
code: code.clone(),
modifiers,
},
)))
.await?;
}
sleep_pace().await;
}
Ok(())
}
pub(crate) fn parse_key_chord(name: &str) -> Result<(String, String, Modifiers)> {
let mut parts = name
.split('+')
.map(str::trim)
.filter(|part| !part.is_empty())
.peekable();
let mut modifiers = Modifiers::default();
let mut key_name = None;
while let Some(part) = parts.next() {
let lower = part.to_ascii_lowercase();
let is_last = parts.peek().is_none();
if is_last {
key_name = Some(lower);
break;
}
match lower.as_str() {
"meta" | "cmd" | "command" => modifiers.meta = true,
"ctrl" | "control" => modifiers.control = true,
"alt" | "option" => modifiers.alt = true,
"shift" => modifiers.shift = true,
other => bail!("unknown key modifier {other:?}: meta, cmd, ctrl, alt, shift"),
}
}
let key_name = key_name.ok_or_else(|| eyre!("key chord is empty"))?;
if key_name.len() == 1 {
let character = key_name.as_bytes()[0];
if character.is_ascii_alphabetic() {
return Ok((
if modifiers.shift {
key_name.to_ascii_uppercase()
} else {
key_name.clone()
},
format!("Key{}", (character as char).to_ascii_uppercase()),
modifiers,
));
}
if character.is_ascii_digit() {
return Ok((key_name.clone(), format!("Digit{key_name}"), modifiers));
}
}
let (key, code) = match key_name.as_str() {
"pageup" | "pgup" => ("PageUp", "PageUp"),
"pagedown" | "pgdn" => ("PageDown", "PageDown"),
"home" => ("Home", "Home"),
"end" => ("End", "End"),
"up" | "arrowup" => ("ArrowUp", "ArrowUp"),
"down" | "arrowdown" => ("ArrowDown", "ArrowDown"),
"left" | "arrowleft" => ("ArrowLeft", "ArrowLeft"),
"right" | "arrowright" => ("ArrowRight", "ArrowRight"),
"tab" => ("Tab", "Tab"),
"enter" => ("Enter", "Enter"),
"escape" | "esc" => ("Escape", "Escape"),
"delete" | "del" => ("Delete", "Delete"),
"backspace" => ("Backspace", "Backspace"),
"space" | "spacebar" => (" ", "Space"),
other => {
bail!(
"unknown key {other:?}: letters, digits, pageup, pagedown, home, end, arrows, tab, enter, escape, delete, backspace, space"
)
}
};
Ok((key.to_owned(), code.to_owned(), modifiers))
}
pub(crate) async fn type_keys(client: &mut Client, count: usize, want: &str) -> Result<()> {
let (snapshot, _) = inspect(client).await?;
let Some(field) = find_text_field(&snapshot.nodes, want) else {
bail!("no enabled, visible text field found; open a tab with a composer");
};
println!(
"typing into node {} role={} name={}",
field.id,
field.role,
report::py_repr(&field.name.chars().take(40).collect::<String>())
);
client
.agent(&AgentControlRequest::Act(AgentAction::Click {
node_id: field.id,
}))
.await?;
let before = metrics_if_supported(client).await?;
let mut latencies = Vec::with_capacity(count);
for index in 0..count {
let letter = (b'a' + (index % 26) as u8) as char;
let started = Instant::now();
for phase in [KeyPhase::Down, KeyPhase::Up] {
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Key {
phase,
key: letter.to_string(),
code: format!("Key{}", letter.to_ascii_uppercase()),
modifiers: Modifiers::default(),
},
)))
.await?;
}
latencies.push(started.elapsed().as_secs_f64() * 1000.0);
sleep_pace().await;
}
let after = metrics_if_supported(client).await?;
report::show_latencies("keystrokes", count, &mut latencies);
report_frames(before.as_ref(), after.as_ref(), count);
Ok(())
}
pub(crate) async fn type_text(client: &mut Client, want: &str, text: &str) -> Result<u64> {
let (snapshot, _) = inspect(client).await?;
let field = find_text_field(&snapshot.nodes, want)
.ok_or_else(|| eyre!("no enabled, visible text field matching {want:?}"))?;
if cli::trace() {
println!(" setting {want:?} (id {})", field.id);
}
client
.agent(&AgentControlRequest::Act(AgentAction::SetValue {
node_id: field.id,
value: text.to_owned(),
}))
.await?;
Ok(field.id)
}
pub(crate) async fn click_named(client: &mut Client, want: &str) -> Result<()> {
let (target_id, _) = locate_control(client, want, &[]).await?;
let (snapshot, _) = inspect(client).await?;
let target = snapshot
.nodes
.iter()
.find(|node| node.id == target_id)
.ok_or_else(|| eyre!("node {target_id} disappeared after it was located"))?;
println!(
"clicking node {} role={} name={}",
target.id,
target.role,
report::py_repr(&target.name.chars().take(50).collect::<String>())
);
let before = metrics_if_supported(client).await?;
let started = Instant::now();
client
.agent(&AgentControlRequest::Act(AgentAction::Click {
node_id: target_id,
}))
.await?;
let ack = started.elapsed().as_secs_f64() * 1000.0;
let after = metrics_if_supported(client).await?;
println!("click acked in {ack:.1}ms");
report_frames(before.as_ref(), after.as_ref(), 1);
Ok(())
}
fn report_frames(
before: Option<&blitz_control_protocol::RendererMetrics>,
after: Option<&blitz_control_protocol::RendererMetrics>,
actions: usize,
) {
let (Some(before), Some(after)) = (before, after) else {
println!("frames: the host reports no renderer metrics; it has no compositor");
return;
};
report::show("before", before);
report::show("after", after);
report::show_delta(before, after, actions);
}
#[cfg(test)]
mod tests {
use super::is_text_field_role;
#[test]
fn editable_aria_roles_are_literal_text_targets() {
for role in [
"textbox",
"textarea",
"input",
"combobox",
"searchbox",
"spinbutton",
] {
assert!(is_text_field_role(role), "{role}");
}
for role in ["button", "listbox", "option", "slider"] {
assert!(!is_text_field_role(role), "{role}");
}
}
}