use std::collections::{HashMap, HashSet};
use std::time::{Duration, Instant};
use blitz_control_protocol::{
AgentAction, AgentControlRequest, AgentSnapshot, CaptureRequest, CapturedImage, DebugResponse,
DebugStream, DiagnosticsRequest, InputCommand, KeyPhase, Modifiers, PointerPhase,
RendererMetrics, SemanticNode, SnapshotRequest, WheelPhase,
};
use eyre::{Result, bail, eyre};
mod app;
mod audit;
mod cli;
mod inspector;
mod qa;
mod reach;
mod report;
mod sweep;
use inspector::Client;
fn pace() -> Duration {
Duration::from_secs_f64(cli::pace().max(0.0))
}
async fn sleep_pace() {
let pace = pace();
if !pace.is_zero() {
tokio::time::sleep(pace).await;
}
}
async fn metrics(client: &mut Client) -> Result<RendererMetrics> {
match client
.diagnostics(&DiagnosticsRequest::Metrics)
.await?
.response
{
DebugResponse::Metrics(metrics) => Ok(metrics),
other => bail!("asked for metrics, got {other:?}"),
}
}
async fn layout(client: &mut Client, want: &str) -> Result<()> {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: true,
include_layout: true,
include_computed_style: false,
}))
.await?;
let DebugResponse::Snapshot(snapshot) = answer.response else {
bail!("asked for a layout snapshot, got {:?}", answer.response);
};
let bounds: HashMap<u64, serde_json::Value> = snapshot
.layout
.as_ref()
.and_then(|value| value.as_array())
.map(|rows| {
rows.iter()
.filter_map(|row| {
let id = row.get("nodeId")?.as_u64()?;
Some((id, row.get("bounds")?.clone()))
})
.collect()
})
.unwrap_or_default();
let nodes = snapshot
.dom
.as_ref()
.and_then(|value| value.as_array())
.cloned()
.unwrap_or_default();
let mut shown = 0usize;
for node in &nodes {
let name = node.get("name").and_then(|v| v.as_str()).unwrap_or("");
let role = node.get("role").and_then(|v| v.as_str()).unwrap_or("");
if !want.is_empty() && !name.contains(want) && !role.contains(want) {
continue;
}
let Some(id) = node.get("id").and_then(|v| v.as_u64()) else {
continue;
};
let Some(box_) = bounds.get(&id) else {
continue;
};
let read = |key: &str, index: usize| {
box_.get(key)
.or_else(|| box_.get(index))
.and_then(|v| v.as_f64())
.unwrap_or(f64::NAN)
};
let row = snapshot
.layout
.as_ref()
.and_then(|value| value.as_array())
.and_then(|rows| {
rows.iter()
.find(|row| row.get("nodeId").and_then(|value| value.as_u64()) == Some(id))
});
let pair = |field: &str, index: usize| {
row.and_then(|row| row.get(field))
.and_then(|value| value.get(index))
.and_then(|value| value.as_f64())
.unwrap_or(f64::NAN)
};
println!(
"{:>6} {:<16} {:>8.1} {:>8.1} {:>8.1} {:>8.1} scroll={:.1},{:.1} range={:.1},{:.1} client={:.1},{:.1} content={:.1},{:.1} {}",
id,
role,
read("x", 0),
read("y", 1),
read("width", 2),
read("height", 3),
pair("scrollOffset", 0),
pair("scrollOffset", 1),
pair("scrollRange", 0),
pair("scrollRange", 1),
pair("clientSize", 0),
pair("clientSize", 1),
pair("scrollSize", 0),
pair("scrollSize", 1),
name.chars().take(60).collect::<String>()
);
shown += 1;
}
if shown == 0 {
println!(
"no named node matched {want:?} ({} in the tree)",
nodes.len()
);
}
Ok(())
}
async fn paint(client: &mut Client, want: &str, min_area: f64) -> Result<()> {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: true,
include_layout: true,
include_computed_style: true,
}))
.await?;
let DebugResponse::Snapshot(snapshot) = answer.response else {
bail!("asked for a paint snapshot, got {:?}", answer.response);
};
let styles: HashMap<u64, serde_json::Value> = snapshot
.computed_style
.as_ref()
.and_then(|value| value.as_array())
.map(|rows| {
rows.iter()
.filter_map(|row| Some((row.get("nodeId")?.as_u64()?, row.clone())))
.collect()
})
.unwrap_or_default();
if styles.is_empty() {
bail!("the snapshot carried no computed styles; is this build's diagnostics feature on?");
}
let bounds: HashMap<u64, (f64, f64, f64, f64)> = snapshot
.layout
.as_ref()
.and_then(|value| value.as_array())
.map(|rows| {
rows.iter()
.filter_map(|row| {
let id = row.get("nodeId")?.as_u64()?;
let read = |key: &str, index: usize| {
row.get("bounds")
.and_then(|b| b.get(key).or_else(|| b.get(index)))
.and_then(|v| v.as_f64())
.unwrap_or(0.0)
};
Some((
id,
(
read("x", 0),
read("y", 1),
read("width", 2),
read("height", 3),
),
))
})
.collect()
})
.unwrap_or_default();
let nodes = snapshot
.dom
.as_ref()
.and_then(|value| value.as_array())
.cloned()
.unwrap_or_default();
let mut rows: Vec<(f64, String)> = Vec::new();
for node in &nodes {
let Some(id) = node.get("id").and_then(|v| v.as_u64()) else {
continue;
};
let name = node.get("name").and_then(|v| v.as_str()).unwrap_or("");
let role = node.get("role").and_then(|v| v.as_str()).unwrap_or("");
if !want.is_empty() && !name.contains(want) && !role.contains(want) {
continue;
}
let (Some(style), Some(&(x, y, w, h))) = (styles.get(&id), bounds.get(&id)) else {
continue;
};
let area = w * h;
if area < min_area {
continue;
}
let field = |key: &str| {
style
.get(key)
.and_then(|v| v.as_str())
.unwrap_or("-")
.to_string()
};
let opacity = style
.get("opacity")
.and_then(|v| v.as_f64())
.unwrap_or(f64::NAN);
rows.push((
area,
format!(
" {id:>11} {role:<12} {w:>7.1}x{h:<7.1} at {x:.0},{y:.0} bg={:<10} fg={:<10} \
opacity={opacity:.2} {:<12} {name}",
field("backgroundColor"),
field("color"),
field("visibility"),
),
));
}
rows.sort_by(|a, b| b.0.total_cmp(&a.0));
println!(
"{} nodes, {} with computed styles, showing boxes of {min_area}px2 or more",
nodes.len(),
styles.len()
);
for (_, row) in &rows {
println!("{row}");
}
if rows.is_empty() {
println!("nothing matched");
}
Ok(())
}
async fn transcript(client: &mut Client) -> Result<()> {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: true,
include_layout: true,
include_computed_style: false,
}))
.await?;
let DebugResponse::Snapshot(snapshot) = answer.response else {
bail!("asked for a transcript snapshot, got {:?}", answer.response);
};
let nodes = snapshot
.dom
.as_ref()
.and_then(|value| value.as_array())
.ok_or_else(|| eyre::eyre!("snapshot omitted DOM rows"))?;
let rows = snapshot
.layout
.as_ref()
.and_then(|value| value.as_array())
.ok_or_else(|| eyre::eyre!("snapshot omitted layout rows"))?;
let conversation = nodes
.iter()
.find(|node| {
node.get("name").and_then(|value| value.as_str())
== reach::profile().transcript_region.as_deref()
})
.and_then(|node| node.get("id").and_then(|value| value.as_u64()))
.ok_or_else(|| eyre::eyre!("configured transcript region is absent"))?;
let parent: HashMap<u64, Option<u64>> = nodes
.iter()
.filter_map(|node| {
Some((
node.get("id")?.as_u64()?,
node.get("parent").and_then(|value| value.as_u64()),
))
})
.collect();
let named: HashMap<u64, (&str, &str)> = nodes
.iter()
.filter_map(|node| {
Some((
node.get("id")?.as_u64()?,
(
node.get("role")
.and_then(|value| value.as_str())
.unwrap_or(""),
node.get("name")
.and_then(|value| value.as_str())
.unwrap_or(""),
),
))
})
.collect();
let layout: HashMap<u64, &serde_json::Value> = rows
.iter()
.filter_map(|row| Some((row.get("nodeId")?.as_u64()?, row)))
.collect();
let conversation_row = layout
.get(&conversation)
.ok_or_else(|| eyre::eyre!("configured transcript region has no layout row"))?;
let pair = |row: &serde_json::Value, field: &str, index: usize| {
row.get(field)
.and_then(|value| value.get(index))
.and_then(|value| value.as_f64())
.unwrap_or(f64::NAN)
};
let bounds = conversation_row
.get("bounds")
.ok_or_else(|| eyre::eyre!("configured transcript region has no bounds"))?;
let viewport_bottom = bounds.get(1).and_then(|v| v.as_f64()).unwrap_or(f64::NAN)
+ bounds.get(3).and_then(|v| v.as_f64()).unwrap_or(f64::NAN);
println!(
"transcript id={conversation} top={:.1} bottom={viewport_bottom:.1} scrollTop={:.1} max={:.1} clientHeight={:.1} scrollHeight={:.1} gapToMax={:.1}",
bounds.get(1).and_then(|v| v.as_f64()).unwrap_or(f64::NAN),
pair(conversation_row, "scrollOffset", 1),
pair(conversation_row, "scrollRange", 1),
pair(conversation_row, "clientSize", 1),
pair(conversation_row, "scrollSize", 1),
pair(conversation_row, "scrollRange", 1) - pair(conversation_row, "scrollOffset", 1),
);
let is_descendant = |mut id: u64| {
for _ in 0..512 {
let Some(Some(next)) = parent.get(&id) else {
return false;
};
if *next == conversation {
return true;
}
id = *next;
}
false
};
let mut descendants: Vec<(f64, u64, f64)> = layout
.iter()
.filter_map(|(id, row)| {
if *id == conversation || !is_descendant(*id) {
return None;
}
let box_ = row.get("bounds")?;
let top = box_.get(1)?.as_f64()?;
let height = box_.get(3)?.as_f64()?;
Some((top + height, *id, top))
})
.collect();
descendants.sort_by(|left, right| right.0.total_cmp(&left.0));
for (bottom, id, top) in descendants.into_iter().take(12) {
let (role, name) = named.get(&id).copied().unwrap_or(("", ""));
println!(
" id={id} top={top:.1} bottom={bottom:.1} fromViewportBottom={:.1} role={role} name={}",
bottom - viewport_bottom,
name.chars().take(100).collect::<String>()
);
}
Ok(())
}
async fn spill(client: &mut Client, axis: &str, tolerance: f64) -> Result<()> {
let (snapshot, elapsed) = inspect(client).await?;
let boxes: HashMap<u64, [f64; 4]> = snapshot
.nodes
.iter()
.filter_map(|node| node.bounds.map(|bounds| (node.id, bounds)))
.collect();
let by_id: HashMap<u64, &SemanticNode> =
snapshot.nodes.iter().map(|node| (node.id, node)).collect();
let describe = |mut id: u64| -> String {
for _ in 0..12 {
let Some(node) = by_id.get(&id) else { break };
if !node.name.is_empty() {
return format!(
"in {} \"{}\"",
node.role,
node.name.chars().take(60).collect::<String>()
);
}
let Some(parent) = node.parent else { break };
id = parent;
}
String::from("(no named ancestor)")
};
let scroll_of: HashMap<u64, (f64, f64)> = {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: false,
include_layout: true,
include_computed_style: false,
}))
.await?;
match answer.response {
DebugResponse::Snapshot(layout) => layout
.layout
.as_ref()
.and_then(|value| value.as_array())
.map(|rows| {
rows.iter()
.filter_map(|row| {
let id = row.get("nodeId")?.as_u64()?;
let range = row.get("scrollRange")?;
let x = range.get(0)?.as_f64()?;
let y = range.get(1)?.as_f64()?;
Some((id, (x, y)))
})
.collect()
})
.unwrap_or_default(),
_ => {
eprintln!("no layout snapshot: scrolled content may read as spill");
HashMap::new()
}
}
};
let vertical = axis.starts_with('v') || axis.starts_with('a');
let mut by_owner: HashMap<String, (usize, f64)> = HashMap::new();
let mut rows: Vec<(f64, String)> = Vec::new();
for node in &snapshot.nodes {
let (Some(child), Some(parent_id)) = (node.bounds, node.parent) else {
continue;
};
let Some(parent) = boxes.get(&parent_id) else {
continue;
};
let (range_x, range_y) = scroll_of.get(&parent_id).copied().unwrap_or((0.0, 0.0));
if parent[2] <= 0.0 || parent[3] <= 0.0 || child[2] <= 0.0 {
continue;
}
let scrolls_x = range_x > 0.5;
let scrolls_y = range_y > 0.5;
let mut worst = f64::NEG_INFINITY;
let mut how = "right";
if !scrolls_x {
let left = parent[0] - child[0];
let right = (child[0] + child[2]) - (parent[0] + parent[2]);
worst = left.max(right);
how = if right >= left { "right" } else { "left" };
}
if vertical && !scrolls_y {
let top = parent[1] - child[1];
let bottom = (child[1] + child[3]) - (parent[1] + parent[3]);
if top > worst {
worst = top;
how = "top";
}
if bottom > worst {
worst = bottom;
how = "bottom";
}
}
if worst <= tolerance {
continue;
}
let owner = describe(parent_id);
let entry = by_owner.entry(owner).or_insert((0, 0.0));
entry.0 += 1;
entry.1 = entry.1.max(worst);
rows.push((
worst,
format!(
"{:>8.1}px {how:<6} {:<11} child[{:.0},{:.0} {:.0}x{:.0}] parent[{:.0},{:.0} {:.0}x{:.0}] {} {}",
worst,
node.role,
child[0], child[1], child[2], child[3],
parent[0], parent[1], parent[2], parent[3],
node.name.chars().take(40).collect::<String>(),
describe(parent_id),
),
));
}
rows.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
println!(
"{} nodes inspected in {elapsed:.1}ms, {} axis, tolerance {tolerance}px",
snapshot.nodes.len(),
if vertical { "both" } else { "horizontal" }
);
if rows.is_empty() {
println!("nothing sticks out of its container");
}
for (_, row) in rows.iter().take(40) {
println!("{row}");
}
if rows.len() > 40 {
println!("... and {} more", rows.len() - 40);
}
if let Some((pane, pane_box)) = snapshot
.nodes
.iter()
.filter(|node| {
reach::profile()
.transcript_region
.as_deref()
.is_some_and(|r| node.name.contains(r))
})
.filter_map(|node| node.bounds.map(|bounds| (node, bounds)))
.max_by(|a, b| {
(a.1[2] * a.1[3])
.partial_cmp(&(b.1[2] * b.1[3]))
.unwrap_or(std::cmp::Ordering::Equal)
})
{
let right = pane_box[0] + pane_box[2];
let mut out: Vec<(f64, u64, String)> = snapshot
.nodes
.iter()
.filter_map(|node| node.bounds.map(|b| (node, b)))
.filter(|(node, _)| {
let mut id = node.parent;
for _ in 0..64 {
match id {
Some(current) if current == pane.id => return true,
Some(current) => id = by_id.get(¤t).and_then(|n| n.parent),
None => return false,
}
}
false
})
.filter_map(|(node, b)| {
let over = (b[0] + b[2]) - right;
(over > 0.5 && b[2] > 0.0 && b[3] > 0.0).then(|| {
(
over,
node.id,
format!("{} {}", node.role, describe(node.id)),
)
})
})
.collect();
out.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
let pane_bottom = pane_box[1] + pane_box[3];
let deepest = snapshot
.nodes
.iter()
.filter_map(|node| node.bounds.map(|b| (node, b)))
.filter(|(node, b)| node.id != pane.id && b[2] > 0.0 && b[3] > 0.0)
.filter(|(node, _)| {
let mut id = node.parent;
for _ in 0..64 {
match id {
Some(current) if current == pane.id => return true,
Some(current) => id = by_id.get(¤t).and_then(|n| n.parent),
None => return false,
}
}
false
})
.map(|(_, b)| b[1] + b[3])
.fold(f64::NEG_INFINITY, f64::max);
if deepest.is_finite() {
println!(
"tail: last content ends at {deepest:.1}, pane ends at {pane_bottom:.1}, gap {:.1}",
pane_bottom - deepest
);
}
println!(
"\ntranscript pane [{:.0},{:.0} {:.0}x{:.0}], right edge {right:.0}",
pane_box[0], pane_box[1], pane_box[2], pane_box[3]
);
if out.is_empty() {
println!(" nothing reaches past it");
}
for (over, id, what) in out.iter().take(15) {
println!(" {over:>8.1}px past {id:>12} {what}");
}
for (_, id, _) in out.iter().take(3) {
println!(" chain for {id}:");
let mut current = Some(*id);
for _ in 0..16 {
let Some(node) = current.and_then(|id| by_id.get(&id)) else {
break;
};
let b = node.bounds.unwrap_or([f64::NAN; 4]);
println!(
" {:>12} {:<12} [{:>7.1},{:>7.1} {:>7.1}x{:>6.1}] {}",
node.id,
node.role,
b[0],
b[1],
b[2],
b[3],
node.name.chars().take(40).collect::<String>()
);
if node.id == pane.id {
break;
}
current = node.parent;
}
}
}
if !by_owner.is_empty() {
let mut owners: Vec<(String, (usize, f64))> = by_owner.into_iter().collect();
owners.sort_by(|a, b| {
b.1.1
.partial_cmp(&a.1.1)
.unwrap_or(std::cmp::Ordering::Equal)
});
println!("\nby container, worst first:");
for (owner, (count, worst)) in owners {
println!(" {count:>4} nodes worst {worst:>7.1}px {owner}");
}
}
Ok(())
}
async fn inspect(client: &mut Client) -> Result<(AgentSnapshot, f64)> {
let started = Instant::now();
let answer = client
.agent(&AgentControlRequest::Inspect {
root: None,
max_depth: 40,
})
.await?;
let elapsed = started.elapsed().as_secs_f64() * 1000.0;
match answer.response {
DebugResponse::AgentSnapshot(snapshot) => Ok((snapshot, elapsed)),
other => bail!("asked for a semantic snapshot, got {other:?}"),
}
}
async fn dom(client: &mut Client, want: &str, depth: usize) -> Result<()> {
if want.is_empty() {
bail!("dom needs a substring to match");
}
let (snapshot, elapsed) = inspect(client).await?;
let by_id: HashMap<u64, &SemanticNode> =
snapshot.nodes.iter().map(|node| (node.id, node)).collect();
let describe = |node: &SemanticNode| -> String {
let bounds = node
.bounds
.map(|b| format!("[{:.0},{:.0} {:.0}x{:.0}]", b[0], b[1], b[2], b[3]))
.unwrap_or_else(|| "[no box]".into());
format!(
"{} {:<10} {:<28} {bounds}{}\n attrs: {}",
node.id,
node.role,
format!("{:?}", node.name),
if node.visible { "" } else { " HIDDEN" },
node.value.as_deref().unwrap_or("(none)")
)
};
let matched: Vec<&SemanticNode> = snapshot
.nodes
.iter()
.filter(|node| {
node.name.contains(want)
|| node.role.contains(want)
|| node.value.as_deref().is_some_and(|v| v.contains(want))
})
.collect();
println!(
"{} of {} nodes match {want:?} (inspect {elapsed:.1}ms)\n",
matched.len(),
snapshot.nodes.len()
);
for node in &matched {
println!("{}", describe(node));
let mut parent = node.parent;
for level in 0..depth {
let Some(current) = parent.and_then(|id| by_id.get(&id)) else {
break;
};
println!(
" {}^{} {}",
" ".repeat(level),
level + 1,
describe(current)
);
parent = current.parent;
}
println!();
}
Ok(())
}
async fn nodes(client: &mut Client) -> Result<usize> {
let (snapshot, elapsed) = inspect(client).await?;
report::show_nodes(&snapshot.nodes, elapsed);
Ok(snapshot.nodes.len())
}
async fn panes(client: &mut Client) -> Result<()> {
let (snapshot, elapsed) = inspect(client).await?;
let by_id: HashMap<u64, &SemanticNode> =
snapshot.nodes.iter().map(|node| (node.id, node)).collect();
let anchor = reach::profile()
.transcript_region
.clone()
.unwrap_or_default();
let anchor: &str = &anchor;
let depth_of = |start: u64| -> usize {
let mut cursor = Some(start);
let mut depth = 0usize;
for _ in 0..256 {
let Some(current) = cursor.and_then(|id| by_id.get(&id)) else {
break;
};
let Some(parent) = current.parent else { break };
cursor = Some(parent);
depth += 1;
}
depth
};
let anchors: Vec<&SemanticNode> = snapshot
.nodes
.iter()
.filter(|node| node.name.contains(anchor))
.collect();
let mut roots: HashMap<u64, (bool, Option<[f64; 4]>)> = HashMap::new();
for anchor in &anchors {
let mut cursor = anchor.parent;
let mut root = anchor.id;
for _ in 0..12 {
let Some(current) = cursor.and_then(|id| by_id.get(&id)) else {
break;
};
let descendants = anchors
.iter()
.filter(|other| {
let mut walk = Some(other.id);
for _ in 0..256 {
let Some(step) = walk.and_then(|id| by_id.get(&id)) else {
return false;
};
if step.id == current.id {
return true;
}
walk = step.parent;
}
false
})
.count();
if descendants > 1 {
break;
}
root = current.id;
cursor = current.parent;
}
let node = by_id.get(&root).copied();
roots.insert(
root,
(
node.map(|n| n.visible).unwrap_or(false),
node.and_then(|n| n.bounds),
),
);
}
let mut totals: HashMap<u64, usize> = HashMap::new();
let mut unattributed = 0usize;
for node in &snapshot.nodes {
let mut cursor = Some(node.id);
let mut found = None;
for _ in 0..256 {
let Some(current) = cursor.and_then(|id| by_id.get(&id)) else {
break;
};
if roots.contains_key(¤t.id) {
found = Some(current.id);
break;
}
cursor = current.parent;
}
match found {
Some(root) => *totals.entry(root).or_default() += 1,
None => unattributed += 1,
}
}
let mut rows: Vec<(u64, usize)> = totals.into_iter().collect();
rows.sort_by_key(|row| std::cmp::Reverse(row.1));
println!(
"{} nodes total, {} panes found via {anchor:?} (inspect {elapsed:.1}ms)\n",
snapshot.nodes.len(),
rows.len()
);
let mut hidden_cost = 0usize;
for (root, count) in &rows {
let (visible, bounds) = roots.get(root).copied().unwrap_or((false, None));
let box_text = bounds
.map(|b| format!("[{:.0},{:.0} {:.0}x{:.0}]", b[0], b[1], b[2], b[3]))
.unwrap_or_else(|| "[no box]".into());
println!(
" {count:>6} node {root:<14} {:<8} depth {:<3} {box_text}",
if visible { "VISIBLE" } else { "hidden" },
depth_of(*root)
);
if !visible {
hidden_cost += count;
}
}
println!("\n {unattributed:>6} outside any pane (chrome, tab strip, overlays)");
println!(
" {hidden_cost:>6} in hidden panes = {:.0}% of the tree",
100.0 * hidden_cost as f64 / snapshot.nodes.len().max(1) as f64
);
Ok(())
}
async fn hover_over(client: &mut Client, want: &str) -> Result<bool> {
if let Some((x, y)) = want.split_once(',')
&& let (Ok(x), Ok(y)) = (x.trim().parse::<f64>(), y.trim().parse::<f64>())
{
{
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Pointer {
phase: PointerPhase::Move,
x,
y,
button: 0,
modifiers: Modifiers::default(),
},
)))
.await?;
println!("pointer at {x:.0},{y:.0}");
return Ok(true);
}
}
let (snapshot, _) = inspect(client).await?;
let Some(node) = snapshot
.nodes
.iter()
.filter(|node| node.visible && node.name.contains(want))
.filter_map(|node| node.bounds.map(|bounds| (node, bounds)))
.max_by(|a, b| {
(a.1[2] * a.1[3])
.partial_cmp(&(b.1[2] * b.1[3]))
.unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(node, bounds)| (node.name.clone(), bounds))
else {
println!("no visible node named {want:?} to hover; wheel goes wherever it goes");
return Ok(false);
};
let (name, bounds) = node;
let (x, y) = (bounds[0] + bounds[2] / 2.0, bounds[1] + bounds[3] / 2.0);
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Pointer {
phase: PointerPhase::Move,
x,
y,
button: 0,
modifiers: Modifiers::default(),
},
)))
.await?;
println!("pointer over {name:?} at {x:.0},{y:.0}");
Ok(true)
}
async fn scroll(client: &mut Client, ticks: usize, delta: f64) -> Result<()> {
let pace = pace();
if pace.is_zero() {
println!("pace: unpaced (BENCH_PACE=0) - measures app throughput");
} else {
println!(
"pace: {:.2}ms between events ({:.0} Hz requested); fps and missed_refreshes \
below describe this pace, not the app's limit. BENCH_PACE=0 to remove it",
pace.as_secs_f64() * 1000.0,
1.0 / pace.as_secs_f64(),
);
}
let mut latencies = Vec::with_capacity(ticks);
for _ in 0..ticks {
let started = Instant::now();
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Wheel {
delta_x: 0.0,
delta_y: delta,
phase: WheelPhase::Moved,
modifiers: Modifiers::default(),
},
)))
.await?;
latencies.push(started.elapsed().as_secs_f64() * 1000.0);
sleep_pace().await;
}
report::show_latencies("wheel events", ticks, &mut latencies);
Ok(())
}
fn find_text_field<'a>(nodes: &'a [SemanticNode], want: &str) -> Option<&'a SemanticNode> {
let modal_scope: HashSet<u64> = reach::dismissers(nodes)
.first()
.map(|(id, _)| reach::enclosing_dialog(nodes, *id))
.unwrap_or_default()
.into_iter()
.collect();
let surface_scope: HashSet<u64> = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(nodes, surface))
.map(|surface| reach::on_surface_subtree(nodes, surface))
.unwrap_or_default()
.into_iter()
.collect();
let fields: Vec<&SemanticNode> = nodes
.iter()
.filter(|node| {
matches!(node.role.as_str(), "textbox" | "textarea" | "input")
&& node.enabled
&& reach::onscreen(node)
})
.collect();
let matches_name = |node: &&SemanticNode| {
want.is_empty() || node.name.to_lowercase().contains(&want.to_lowercase())
};
for scope in [&modal_scope, &surface_scope] {
if let Some(field) = fields
.iter()
.find(|node| matches_name(node) && scope.contains(&node.id))
{
return Some(field);
}
}
fields.into_iter().find(matches_name)
}
async fn press_key(client: &mut Client, name: &str, count: usize, over: &str) -> Result<()> {
let (snapshot, _) = inspect(client).await?;
let by_id = over.parse::<u64>().ok().filter(|id| {
snapshot
.nodes
.iter()
.any(|node| node.id == *id && node.visible)
});
if let Some(target) = by_id.or_else(|| {
snapshot
.nodes
.iter()
.filter(|node| node.visible && !over.is_empty() && node.name.contains(over))
.filter_map(|node| node.bounds.map(|b| (node, b)))
.max_by(|a, b| {
(a.1[2] * a.1[3])
.partial_cmp(&(b.1[2] * b.1[3]))
.unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(node, _)| node.id)
}) {
client
.agent(&AgentControlRequest::Act(AgentAction::Click {
node_id: target,
}))
.await?;
tokio::time::sleep(Duration::from_millis(150)).await;
println!("focused node {target} for {name} x{count}");
} else {
println!("no visible node named {over:?}; sending {name} to whatever has focus");
}
let (key, code) = match name.to_ascii_lowercase().as_str() {
"pageup" | "pgup" => ("PageUp", "PageUp"),
"pagedown" | "pgdn" => ("PageDown", "PageDown"),
"home" => ("Home", "Home"),
"end" => ("End", "End"),
"up" | "arrowup" => ("ArrowUp", "ArrowUp"),
"down" | "arrowdown" => ("ArrowDown", "ArrowDown"),
"left" | "arrowleft" => ("ArrowLeft", "ArrowLeft"),
"right" | "arrowright" => ("ArrowRight", "ArrowRight"),
"tab" => ("Tab", "Tab"),
"enter" => ("Enter", "Enter"),
"escape" | "esc" => ("Escape", "Escape"),
other => {
bail!(
"unknown key {other:?}: pageup, pagedown, home, end, up, down, left, right, tab, enter, escape"
)
}
};
for _ in 0..count {
for phase in [KeyPhase::Down, KeyPhase::Up] {
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Key {
phase,
key: key.to_string(),
code: code.to_string(),
modifiers: Modifiers::default(),
},
)))
.await?;
}
sleep_pace().await;
}
tokio::time::sleep(Duration::from_millis(300)).await;
Ok(())
}
async fn type_keys(client: &mut Client, count: usize, want: &str) -> Result<()> {
let (snapshot, _) = inspect(client).await?;
let Some(field) = find_text_field(&snapshot.nodes, want) else {
bail!("no enabled, visible text field found; open a tab with a composer");
};
println!(
"typing into node {} role={} name={}",
field.id,
field.role,
report::py_repr(&field.name.chars().take(40).collect::<String>())
);
client
.agent(&AgentControlRequest::Act(AgentAction::Click {
node_id: field.id,
}))
.await?;
tokio::time::sleep(Duration::from_millis(200)).await;
let before = metrics(client).await?;
let mut latencies = Vec::with_capacity(count);
for index in 0..count {
let letter = (b'a' + (index % 26) as u8) as char;
let started = Instant::now();
for phase in [KeyPhase::Down, KeyPhase::Up] {
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Key {
phase,
key: letter.to_string(),
code: format!("Key{}", letter.to_ascii_uppercase()),
modifiers: Modifiers::default(),
},
)))
.await?;
}
latencies.push(started.elapsed().as_secs_f64() * 1000.0);
sleep_pace().await;
}
let after = metrics(client).await?;
report::show_latencies("keystrokes", count, &mut latencies);
report::show("before", &before);
report::show("after", &after);
report::show_delta(&before, &after, count);
Ok(())
}
async fn type_text(client: &mut Client, want: &str, text: &str) -> Result<()> {
let (snapshot, _) = inspect(client).await?;
let field = find_text_field(&snapshot.nodes, want)
.ok_or_else(|| eyre!("no enabled, visible text field matching {want:?}"))?;
if cli::trace() {
println!(" setting {want:?} (id {})", field.id);
}
let answer = client
.agent(&AgentControlRequest::Act(AgentAction::SetValue {
node_id: field.id,
value: text.to_owned(),
}))
.await?;
if let DebugResponse::Error(error) = answer.response {
bail!("{} ({})", error.message, error.code);
}
tokio::time::sleep(Duration::from_millis(150)).await;
Ok(())
}
async fn click_named(client: &mut Client, want: &str) -> Result<()> {
let (snapshot, _) = inspect(client).await?;
let wanted = want.to_lowercase();
let Some(target) = snapshot
.nodes
.iter()
.find(|node| node.name.to_lowercase().contains(&wanted) && node.visible && node.enabled)
else {
bail!(
"no visible, enabled node whose name contains {}",
report::py_repr(want)
);
};
println!(
"clicking node {} role={} name={}",
target.id,
target.role,
report::py_repr(&target.name.chars().take(50).collect::<String>())
);
let offscreen = target
.bounds
.is_some_and(|b| b[1] + b[3] < 0.0 || b[0] + b[2] < 0.0);
let target_id = target.id;
if offscreen {
println!(" offscreen, scrolling it into view first");
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id: target_id,
}))
.await?;
tokio::time::sleep(Duration::from_millis(300)).await;
}
let before = metrics(client).await?;
let started = Instant::now();
client
.agent(&AgentControlRequest::Act(AgentAction::Click {
node_id: target_id,
}))
.await?;
let ack = started.elapsed().as_secs_f64() * 1000.0;
tokio::time::sleep(Duration::from_millis(500)).await;
let after = metrics(client).await?;
println!("click acked in {ack:.1}ms");
report::show("before", &before);
report::show("after", &after);
report::show_delta(&before, &after, 1);
Ok(())
}
async fn run_qa(
client: &mut Client,
group: Option<&str>,
checks_dir: Option<&std::path::Path>,
) -> Result<usize> {
let all = qa::checks(checks_dir).map_err(|error| eyre!(error))?;
let selected: Vec<&qa::Check> = all
.iter()
.filter(|check| group.is_none_or(|want| check.group == want || check.id == want))
.collect();
if selected.is_empty() {
let mut names: Vec<String> = all
.iter()
.map(|check| format!("{} ({})", check.id, check.group))
.collect();
names.sort();
bail!(
"no check or group matching {group:?}. known:\n {}",
names.join("\n ")
);
}
let mut results: Vec<(&qa::Check, std::result::Result<(), String>)> = Vec::new();
for check in selected {
let mut open_error = None;
if let Some(want) = check.open.as_deref() {
let want_here: &str = check.click.as_deref().unwrap_or(&check.subject);
let (here, _) = inspect(client).await?;
let arrived = here.nodes.iter().any(|n| {
n.name.contains(want_here) && n.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0)
});
if !arrived {
let _ = click_named_quiet(client, want).await;
settle(client, None).await?;
let (now, _) = inspect(client).await?;
let there = now.nodes.iter().any(|n| {
n.name.contains(want_here) && n.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0)
});
if !there && open_named(client, want).await.is_err() {
if let Some(home) = reach::profile().home_opener.as_deref() {
let _ = click_named_quiet(client, home).await;
}
settle(client, None).await?;
if let Err(error) = open_named(client, want).await {
open_error = Some(format!("could not open {want:?}: {error}"));
}
}
}
settle(client, None).await?;
}
let (before, _) = inspect(client).await?;
if let Some(want) = check.hover.as_deref() {
let (tree, _) = inspect(client).await?;
let target = tree
.nodes
.iter()
.find(|node| {
node.name.contains(want)
&& node.visible
&& node.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0)
})
.map(|node| node.id);
let Some(node_id) = target else {
bail!("no visible, sized node matching {want:?} to hover");
};
client
.agent(&AgentControlRequest::Act(AgentAction::Hover { node_id }))
.await?;
tokio::time::sleep(Duration::from_millis(150)).await;
}
let mut click_error = None;
let mut action_target = None;
if let Some(want) = check.click.as_deref() {
if check.expect == qa::Expect::TargetPaints && !check.press {
let (tree, _) = inspect(client).await?;
let wanted = want.to_lowercase();
action_target = tree
.nodes
.iter()
.find(|node| {
node.name.to_lowercase().contains(&wanted) && node.visible && node.enabled
})
.map(|node| node.name.clone());
}
let driven = if check.press {
press_named(client, want).await
} else {
click_named_quiet(client, want).await
};
if let Err(error) = driven {
let how = if check.press { "press" } else { "click" };
click_error = Some(format!("could not {how} {want:?}: {error}"));
}
settle(client, Some(&check.subject)).await?;
}
if click_error.is_none()
&& let Some(text) = check.text.as_deref()
{
if let Some(field) = check.type_into.as_deref() {
if let Err(error) = type_text(client, field, text).await {
click_error = Some(format!("could not type into {field:?}: {error}"));
}
} else {
click_error = Some("text requires type_into".to_owned());
}
}
if click_error.is_none()
&& let Some(key) = check.key.as_deref()
{
let target = check
.key_on
.as_deref()
.or(check.type_into.as_deref())
.unwrap_or("");
if let Err(error) = press_key(client, key, 1, target).await {
click_error = Some(format!("could not send {key:?}: {error}"));
}
}
if check.text.is_some() || check.key.is_some() {
settle(client, Some(&check.subject)).await?;
}
let (after, _) = inspect(client).await?;
let outcome = match open_error.or(click_error) {
Some(error) => Err(error),
None if check.expect == qa::Expect::TargetPaints => {
let Some(subject) = action_target else {
results.push((
check,
Err("could not resolve the exact click target".to_owned()),
));
continue;
};
let mut targeted = (*check).clone();
targeted.subject = subject;
targeted.expect = qa::Expect::Paints;
qa::verdict(&targeted, &before.nodes, &after.nodes)
}
None => qa::verdict(check, &before.nodes, &after.nodes),
};
results.push((check, outcome));
}
let failed = results.iter().filter(|(_, out)| out.is_err()).count();
let tally = qa::tally(&results);
let mut groups: Vec<_> = tally.iter().collect();
groups.sort_by_key(|(name, _)| *name);
let report = Report {
passed: results.len() - failed,
failed,
groups: groups
.into_iter()
.map(|(name, (passed, total))| GroupRow {
name: name.to_string(),
passed: *passed,
total: *total,
})
.collect(),
checks: results.iter().map(CheckRow::from).collect(),
};
println!(
"{}",
toon_format::encode_default(&report).map_err(|e| eyre!(e.to_string()))?
);
Ok(failed)
}
async fn click_by_id(client: &mut Client, node_id: u64) -> Result<()> {
let answer = client
.agent(&AgentControlRequest::Act(AgentAction::Click { node_id }))
.await?;
if let DebugResponse::Error(error) = answer.response {
bail!("{} ({})", error.message, error.code);
}
Ok(())
}
async fn settle(client: &mut Client, want: Option<&str>) -> Result<()> {
if let Some(subject) = want {
let (role, name) = subject.split_once(':').unwrap_or(("", subject));
for _ in 0..40 {
tokio::time::sleep(Duration::from_millis(100)).await;
let (snapshot, _) = inspect(client).await?;
let painted = snapshot.nodes.iter().any(|n| {
(role.is_empty() || n.role == role)
&& n.name.contains(name)
&& n.visible
&& n.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0)
});
if painted {
return Ok(());
}
}
return Ok(());
}
let mut last = 0usize;
let mut stable = 0;
for _ in 0..40 {
tokio::time::sleep(Duration::from_millis(100)).await;
let (snapshot, _) = inspect(client).await?;
let now = snapshot
.nodes
.iter()
.filter(|n| n.visible && n.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0))
.count();
if now == last {
stable += 1;
if stable == 2 {
return Ok(());
}
} else {
stable = 0;
last = now;
}
}
Ok(())
}
#[derive(serde::Serialize)]
struct Report {
passed: usize,
failed: usize,
groups: Vec<GroupRow>,
checks: Vec<CheckRow>,
}
#[derive(serde::Serialize)]
struct GroupRow {
name: String,
passed: usize,
total: usize,
}
#[derive(serde::Serialize)]
struct CheckRow {
verdict: &'static str,
group: String,
id: String,
error: String,
what: String,
}
impl From<&(&qa::Check, std::result::Result<(), String>)> for CheckRow {
fn from((check, outcome): &(&qa::Check, std::result::Result<(), String>)) -> Self {
Self {
verdict: if outcome.is_ok() { "pass" } else { "fail" },
group: check.group.clone(),
id: check.id.clone(),
error: outcome.clone().err().unwrap_or_default(),
what: check.what.clone(),
}
}
}
#[derive(serde::Serialize)]
struct FoundRow {
id: u64,
role: String,
name: String,
x: f64,
y: f64,
w: f64,
h: f64,
state: String,
}
#[expect(
clippy::too_many_arguments,
reason = "each one is a filter the caller asks for by name"
)]
async fn find(
client: &mut Client,
pattern: &str,
roles: &[String],
visible: bool,
hidden: bool,
painted: bool,
offscreen_only: bool,
disabled: bool,
count_only: bool,
limit: Option<usize>,
) -> Result<()> {
let (snapshot, elapsed) = inspect(client).await?;
let viewport = viewport_of(&snapshot);
let rows: Vec<FoundRow> = snapshot
.nodes
.iter()
.filter(|node| roles.is_empty() || roles.iter().any(|role| role == &node.role))
.filter(|node| name_matches(&node.name, pattern))
.filter(|node| !visible || node.visible)
.filter(|node| !hidden || !node.visible)
.filter(|node| !disabled || !node.enabled)
.filter(|node| {
!painted
|| node
.bounds
.is_some_and(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
})
.filter(|node| {
!offscreen_only
|| node
.bounds
.is_some_and(|bounds| offscreen(bounds, viewport))
})
.map(|node| {
let bounds = node.bounds.unwrap_or([0.0; 4]);
let mut state: Vec<&str> = Vec::new();
state.push(if node.visible { "visible" } else { "hidden" });
if !node.enabled {
state.push("disabled");
}
if bounds[2] <= 0.0 || bounds[3] <= 0.0 {
state.push("0x0");
} else if offscreen(bounds, viewport) {
state.push("offscreen");
}
let round = |value: f64| (value * 10.0).round() / 10.0;
FoundRow {
id: node.id,
role: node.role.clone(),
name: node.name.clone(),
x: round(bounds[0]),
y: round(bounds[1]),
w: round(bounds[2]),
h: round(bounds[3]),
state: state.join(","),
}
})
.collect();
let matched = rows.len();
let shown: Vec<FoundRow> = match limit {
Some(limit) => rows.into_iter().take(limit).collect(),
None => rows,
};
#[derive(serde::Serialize)]
struct Found {
matched: usize,
of: usize,
inspect_ms: f64,
controls: Vec<FoundRow>,
}
let report = Found {
matched,
of: snapshot.nodes.len(),
inspect_ms: (elapsed * 10.0).round() / 10.0,
controls: if count_only { Vec::new() } else { shown },
};
println!(
"{}",
toon_format::encode_default(&report).map_err(|error| eyre!(error.to_string()))?
);
Ok(())
}
fn name_matches(name: &str, pattern: &str) -> bool {
let (name, pattern) = (name.to_lowercase(), pattern.to_lowercase());
if pattern == "*" {
return true;
}
let Some(stripped) = pattern.strip_prefix('*') else {
return match pattern.strip_suffix('*') {
Some(prefix) => name.starts_with(prefix),
None => name.contains(&pattern),
};
};
match stripped.strip_suffix('*') {
Some(middle) => name.contains(middle),
None => name.ends_with(stripped),
}
}
fn viewport_of(snapshot: &AgentSnapshot) -> (f64, f64) {
let bottom = snapshot
.nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| b[1] + b[3])
.fold(f64::MIN, f64::max);
(0.0, if bottom > f64::MIN { bottom } else { f64::MAX })
}
fn offscreen(bounds: [f64; 4], viewport: (f64, f64)) -> bool {
bounds[1] + bounds[3] < viewport.0 || bounds[0] + bounds[2] < 0.0 || bounds[1] > viewport.1
}
async fn locate_button(client: &mut Client, want: &str) -> Result<(u64, [f64; 4])> {
let wanted = want.to_lowercase();
let pick = |snapshot: &AgentSnapshot, viewport: (f64, f64)| -> Option<(u64, [f64; 4])> {
let modal_scope: HashSet<u64> = reach::dismissers(&snapshot.nodes)
.first()
.map(|(id, _)| reach::enclosing_dialog(&snapshot.nodes, *id))
.unwrap_or_default()
.into_iter()
.collect();
let surface_scope: HashSet<u64> = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(&snapshot.nodes, surface))
.map(|surface| reach::on_surface_subtree(&snapshot.nodes, surface))
.unwrap_or_default()
.into_iter()
.collect();
let candidates: Vec<_> = snapshot
.nodes
.iter()
.filter(|n| n.role == "button")
.filter(|n| n.name.to_lowercase().contains(&wanted))
.filter(|n| n.visible && n.enabled)
.filter_map(|node| {
node.bounds
.filter(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
.map(|bounds| (node, bounds))
})
.collect();
for scope in [&modal_scope, &surface_scope] {
if let Some((node, bounds)) = candidates
.iter()
.find(|(node, bounds)| scope.contains(&node.id) && !offscreen(*bounds, viewport))
{
return Some((node.id, *bounds));
}
}
if let Some((node, bounds)) = candidates
.iter()
.find(|(_, bounds)| !offscreen(*bounds, viewport))
{
return Some((node.id, *bounds));
}
let mut fallback = None;
for (node, bounds) in candidates {
let recoverable = bounds[0] + bounds[2] > 0.0;
if recoverable && !fallback.is_some_and(|(_, b): (u64, [f64; 4])| b[0] + b[2] > 0.0) {
fallback = Some((node.id, bounds));
} else {
fallback.get_or_insert((node.id, bounds));
}
}
fallback
};
let (snapshot, _) = inspect(client).await?;
let viewport = viewport_of(&snapshot);
let Some((id, bounds)) = pick(&snapshot, viewport) else {
bail!("no visible, enabled, sized button matching it");
};
if !offscreen(bounds, viewport) {
return Ok((id, bounds));
}
if cli::trace() {
println!(" {want:?} is off-screen at {bounds:?}, scrolling it in");
}
let mut target = (id, bounds);
let mut latest = snapshot;
for _ in 0..4 {
for node_id in reach::reveal_chain(&latest.nodes, target.0) {
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id,
}))
.await?;
}
tokio::time::sleep(Duration::from_millis(250)).await;
let (settled, _) = inspect(client).await?;
let viewport = viewport_of(&settled);
let Some(found) = pick(&settled, viewport) else {
bail!("no visible, enabled, sized button matching it");
};
target = found;
if !offscreen(target.1, viewport) {
return Ok(target);
}
let delta_y = if target.1[1] > viewport.1 {
target.1[1] + target.1[3] - viewport.1 + 16.0
} else if target.1[1] + target.1[3] < viewport.0 {
target.1[1] - viewport.0 - 16.0
} else {
0.0
};
if delta_y != 0.0
&& let Some(node_id) = reach::reveal_chain(&settled.nodes, target.0).first()
{
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollBy {
node_id: *node_id,
delta_x: 0.0,
delta_y,
}))
.await?;
tokio::time::sleep(Duration::from_millis(150)).await;
}
latest = settled;
}
bail!(
"{want:?} is still off-screen at {:?} after four semantic reveal attempts",
target.1
)
}
async fn open_named(client: &mut Client, want: &str) -> Result<()> {
let (id, _) = locate_button(client, want).await?;
if cli::trace() {
println!(" opening {want:?} (id {id})");
}
client
.agent(&AgentControlRequest::Act(AgentAction::DoubleClick {
node_id: id,
}))
.await?;
Ok(())
}
async fn press_named(client: &mut Client, want: &str) -> Result<()> {
let (id, b) = locate_button(client, want).await?;
let (x, y) = (b[0] + b[2] / 2.0, b[1] + b[3] / 2.0);
if cli::trace() {
println!(" pressing {want:?} (id {id}) at {x:.0},{y:.0}");
}
for phase in [PointerPhase::Move, PointerPhase::Down, PointerPhase::Up] {
let answer = client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Pointer {
phase,
x,
y,
button: 0,
modifiers: Modifiers::default(),
},
)))
.await?;
if let DebugResponse::Error(error) = answer.response {
bail!("{} ({})", error.message, error.code);
}
}
Ok(())
}
async fn click_named_quiet(client: &mut Client, want: &str) -> Result<()> {
let (target_id, _) = locate_button(client, want).await?;
if cli::trace() {
println!(" activating {want:?} (id {target_id})");
}
client
.agent(&AgentControlRequest::Act(AgentAction::Click {
node_id: target_id,
}))
.await?;
Ok(())
}
struct Ink {
visible: usize,
total: usize,
background: (u8, u8, u8),
}
impl Ink {
fn fraction(&self) -> f64 {
if self.total == 0 {
0.0
} else {
self.visible as f64 / self.total as f64
}
}
}
fn measure_ink(image: &CapturedImage) -> Result<Ink> {
use base64::Engine as _;
let rgba = base64::engine::general_purpose::STANDARD
.decode(&image.rgba_base64)
.map_err(|error| eyre::eyre!("the capture was not valid base64: {error}"))?;
let expected = (image.width as usize) * (image.height as usize) * 4;
if rgba.len() != expected {
bail!(
"capture is {} bytes, expected {expected} for {}x{}",
rgba.len(),
image.width,
image.height
);
}
let mut histogram: HashMap<(u8, u8, u8), usize> = HashMap::new();
for pixel in rgba.as_chunks::<4>().0 {
*histogram.entry((pixel[0], pixel[1], pixel[2])).or_default() += 1;
}
let background = histogram
.iter()
.max_by_key(|(_, count)| **count)
.map(|(colour, _)| *colour)
.unwrap_or((0, 0, 0));
let luminance = |(r, g, b): (u8, u8, u8)| {
0.299 * f64::from(r) + 0.587 * f64::from(g) + 0.114 * f64::from(b)
};
let background_luminance = luminance(background);
let visible = rgba
.as_chunks::<4>()
.0
.iter()
.filter(|pixel| {
if pixel[3] < 32 {
return false;
}
(luminance((pixel[0], pixel[1], pixel[2])) - background_luminance).abs() > 24.0
})
.count();
Ok(Ink {
visible,
total: (image.width as usize) * (image.height as usize),
background,
})
}
async fn capture(client: &mut Client, want: &str, scale: f32) -> Result<()> {
let node_id = if want.is_empty() {
None
} else {
let (snapshot, _) = inspect(client).await?;
let node = snapshot
.nodes
.iter()
.filter(|node| node.name.contains(want) && node.visible)
.filter_map(|node| node.bounds.map(|bounds| (node, bounds)))
.filter(|(_, bounds)| bounds[2] > 0.0 && bounds[3] > 0.0)
.max_by(|a, b| {
(a.1[2] * a.1[3])
.partial_cmp(&(b.1[2] * b.1[3]))
.unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(node, _)| node);
let Some(node) = node else {
bail!("no visible node with a box whose name contains {want:?}");
};
println!(
"capturing {} role={} name={}",
node.id,
node.role,
report::py_repr(&node.name.chars().take(50).collect::<String>())
);
Some(node.id)
};
let answer = client
.diagnostics(&DiagnosticsRequest::Capture(CaptureRequest {
node_id,
scale,
}))
.await?;
let image = match answer.response {
DebugResponse::Captured(image) => image,
DebugResponse::Error(error) => bail!("capture refused: {} ({})", error.message, error.code),
other => bail!("asked for a capture, got {other:?}"),
};
let ink = measure_ink(&image)?;
println!(
"{}x{} at {scale}x, background #{:02x}{:02x}{:02x}",
image.width, image.height, ink.background.0, ink.background.1, ink.background.2
);
println!(
"visible ink: {} of {} pixels ({:.2}%)",
ink.visible,
ink.total,
ink.fraction() * 100.0
);
if ink.visible == 0 {
println!("nothing was drawn: every pixel is the background colour");
}
Ok(())
}
async fn run_audit(client: &mut Client, family: Option<&str>) -> Result<usize> {
use audit::{Audited, Verdict};
let (snapshot, _) = inspect(client).await?;
let viewport = snapshot
.nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| (b[1], b[1] + b[3]))
.max_by(|a, b| {
(a.1 - a.0)
.partial_cmp(&(b.1 - b.0))
.unwrap_or(std::cmp::Ordering::Equal)
})
.unwrap_or((0.0, f64::MAX));
let painted = painted_nodes(client).await?;
let mut rows: Vec<Audited> = Vec::new();
for node in audit::buttons(&snapshot.nodes) {
let family_name = audit::family_of(&node.name);
if family.is_some_and(|want| family_name != want) {
continue;
}
let (width, height) = node.bounds.map(|b| (b[2], b[3])).unwrap_or((0.0, 0.0));
let verdict = if !node.visible {
Verdict::Hidden
} else if width <= 0.0 || height <= 0.0 {
Verdict::NoBox
} else if node
.bounds
.is_some_and(|b| b[1] + b[3] < viewport.0 || b[0] + b[2] < 0.0 || b[1] > viewport.1)
{
Verdict::Offscreen
} else if painted.contains(&node.id) {
Verdict::Drawn
} else {
Verdict::Blank
};
rows.push(Audited {
name: node.name.clone(),
family: family_name,
width,
height,
verdict,
});
}
if rows.is_empty() {
bail!("no buttons matched {family:?}");
}
println!("auditing {} buttons in the running app\n", rows.len());
let faults: Vec<&Audited> = rows.iter().filter(|row| row.verdict.is_fault()).collect();
if faults.is_empty() {
println!("no faults: every visible button was painted");
} else {
println!("{} button(s) nobody can see:\n", faults.len());
for row in &faults {
println!(
" {:<8} {:<52} {:.0}x{:.0}",
row.verdict.label(),
row.name.chars().take(52).collect::<String>(),
row.width,
row.height
);
}
println!();
}
let mut families: Vec<_> = audit::by_family(&rows).into_iter().collect();
families.sort_by_key(|(name, _)| *name);
for (name, (passed, total)) in families {
let mark = if passed == total { " " } else { "!" };
println!("{mark} {name:<12} {passed}/{total}");
}
println!("\n{} audited, {} faults", rows.len(), faults.len());
Ok(faults.len())
}
async fn painted_nodes(client: &mut Client) -> Result<HashSet<u64>> {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: false,
include_layout: false,
include_computed_style: true,
}))
.await?;
let DebugResponse::Snapshot(snapshot) = answer.response else {
bail!("asked for a paint snapshot, got {:?}", answer.response);
};
Ok(snapshot
.computed_style
.as_ref()
.and_then(|value| value.as_array())
.map(|rows| {
rows.iter()
.filter_map(|row| row.get("nodeId")?.as_u64())
.collect()
})
.unwrap_or_default())
}
async fn run_sweep(client: &mut Client, family: Option<&str>) -> Result<usize> {
let (snapshot, _) = inspect(client).await?;
let viewport = snapshot
.nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| (b[1], b[1] + b[3]))
.max_by(|a, b| {
(a.1 - a.0)
.partial_cmp(&(b.1 - b.0))
.unwrap_or(std::cmp::Ordering::Equal)
})
.unwrap_or((0.0, f64::MAX));
let planned = sweep::cases(
&snapshot.nodes,
family,
audit::family_of,
reach::is_inert_control,
);
if planned.is_empty() {
bail!("no clickable buttons matched {family:?}");
}
println!("clicking {} buttons\n", planned.len());
let mut outcomes: Vec<sweep::Outcome> = Vec::new();
for case in planned {
let (before, _) = inspect(client).await?;
let Some(node) = before.nodes.iter().find(|node| node.id == case.id) else {
continue;
};
if !node.visible || !node.enabled {
continue;
}
if node
.bounds
.is_some_and(|b| b[1] + b[3] < viewport.0 || b[0] + b[2] < 0.0 || b[1] > viewport.1)
{
continue;
}
if let Err(error) = click_by_id(client, case.id).await {
outcomes.push(sweep::Outcome {
case,
failure: Some(format!("could not be clicked: {error}")),
});
continue;
}
tokio::time::sleep(Duration::from_millis(250)).await;
let (after, _) = inspect(client).await?;
let failure = sweep::judge(&case, &before.nodes, &after.nodes);
outcomes.push(sweep::Outcome { case, failure });
}
let failures: Vec<&sweep::Outcome> = outcomes.iter().filter(|o| o.failure.is_some()).collect();
if failures.is_empty() {
println!("every button acted");
} else {
println!("{} button(s) did not act:\n", failures.len());
for outcome in &failures {
println!(
" {:<48} {}",
outcome.case.name.chars().take(48).collect::<String>(),
outcome.failure.as_deref().unwrap_or("")
);
}
println!();
}
let mut by_family: HashMap<&'static str, (usize, usize)> = HashMap::new();
for outcome in &outcomes {
let entry = by_family.entry(outcome.case.family).or_insert((0, 0));
entry.1 += 1;
if outcome.failure.is_none() {
entry.0 += 1;
}
}
let mut families: Vec<_> = by_family.into_iter().collect();
families.sort_by_key(|(name, _)| *name);
for (name, (passed, total)) in families {
let mark = if passed == total { " " } else { "!" };
println!("{mark} {name:<12} {passed}/{total}");
}
println!(
"\n{} clicked, {} did not act",
outcomes.len(),
failures.len()
);
Ok(failures.len())
}
async fn expand_everything(client: &mut Client, surface: &reach::Surface) -> Result<usize> {
let mut opened = 0;
for _ in 0..6 {
let (tree, _) = inspect(client).await?;
let mine: std::collections::HashSet<u64> = reach::on_surface_subtree(&tree.nodes, surface)
.into_iter()
.collect();
let todo: Vec<(u64, String)> = reach::expanders(&tree.nodes)
.into_iter()
.filter(|(id, _)| mine.contains(id))
.collect();
if todo.is_empty() {
break;
}
for (id, _name) in todo {
if click_by_id(client, id).await.is_ok() {
opened += 1;
tokio::time::sleep(Duration::from_millis(80)).await;
}
}
}
Ok(opened)
}
async fn hover_all_rows(client: &mut Client) -> Result<usize> {
let (tree, _) = inspect(client).await?;
let window = tree
.nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| (b[1], b[1] + b[3]))
.max_by(|a, b| {
(a.1 - a.0)
.partial_cmp(&(b.1 - b.0))
.unwrap_or(std::cmp::Ordering::Equal)
})
.unwrap_or((0.0, 4000.0));
let mut revealed = 0;
for node_id in reach::hover_row_ids(&tree.nodes, "listitem", window) {
if client
.agent(&AgentControlRequest::Act(AgentAction::Hover { node_id }))
.await
.is_ok()
{
revealed += 1;
tokio::time::sleep(Duration::from_millis(40)).await;
}
}
Ok(revealed)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum InventoryClass {
Manual,
Isolated,
Anonymous,
Unreachable,
Disabled,
Reachable,
}
fn inventory_class(node: &SemanticNode, manual: bool, isolated: bool) -> InventoryClass {
if manual {
InventoryClass::Manual
} else if isolated {
InventoryClass::Isolated
} else if node.name.trim().is_empty() {
InventoryClass::Anonymous
} else if !reach::onscreen(node) {
InventoryClass::Unreachable
} else if !node.enabled {
InventoryClass::Disabled
} else {
InventoryClass::Reachable
}
}
async fn run_inventory(client: &mut Client, only: Option<&str>) -> Result<usize> {
#[derive(serde::Serialize)]
struct SurfaceRow {
surface: String,
opened: bool,
components: usize,
reachable: usize,
unreachable: usize,
anonymous: usize,
disabled: usize,
manual: usize,
isolated: usize,
unverified: usize,
sections_opened: usize,
rows_hovered: usize,
}
#[derive(serde::Serialize)]
struct ControlRow {
surface: String,
id: u64,
role: String,
name: String,
classification: String,
reason: String,
}
#[derive(serde::Serialize)]
struct RoleRow {
role: String,
components: usize,
reachable: usize,
unreachable: usize,
anonymous: usize,
disabled: usize,
manual: usize,
isolated: usize,
}
#[derive(serde::Serialize)]
struct InventoryReport {
components: usize,
reachable: usize,
unreachable: usize,
anonymous: usize,
disabled: usize,
manual: usize,
isolated: usize,
unverified: usize,
surfaces: Vec<SurfaceRow>,
roles: Vec<RoleRow>,
controls: Vec<ControlRow>,
}
let mut rows = Vec::new();
let mut controls = Vec::new();
let mut role_counts: std::collections::BTreeMap<String, [usize; 7]> =
std::collections::BTreeMap::new();
for surface in reach::surfaces() {
if only.is_some_and(|want| want != surface.name) {
continue;
}
if !open_surface(client, surface).await? {
rows.push(SurfaceRow {
surface: surface.name.clone(),
opened: false,
components: 0,
reachable: 0,
unreachable: 0,
anonymous: 0,
disabled: 0,
manual: 0,
isolated: 0,
unverified: 0,
sections_opened: 0,
rows_hovered: 0,
});
continue;
}
let sections_opened = expand_everything(client, surface).await?;
if !open_surface(client, surface).await? {
rows.push(SurfaceRow {
surface: surface.name.clone(),
opened: false,
components: 0,
reachable: 0,
unreachable: 0,
anonymous: 0,
disabled: 0,
manual: 0,
isolated: 0,
unverified: 0,
sections_opened,
rows_hovered: 0,
});
continue;
}
let rows_hovered = hover_all_rows(client).await?;
let (tree, _) = inspect(client).await?;
let mine: std::collections::HashSet<u64> = reach::on_surface_subtree(&tree.nodes, surface)
.into_iter()
.collect();
let components: Vec<_> = tree
.nodes
.iter()
.filter(|node| reach::interactive(node) && mine.contains(&node.id))
.collect();
let classes: Vec<_> = components
.iter()
.map(|node| {
inventory_class(
node,
reach::requires_manual_release_check(&node.name),
reach::requires_isolated_outcome(&node.name),
)
})
.collect();
let count = |class| classes.iter().filter(|found| **found == class).count();
let reachable = count(InventoryClass::Reachable);
let unreachable = count(InventoryClass::Unreachable);
let anonymous = count(InventoryClass::Anonymous);
let disabled = count(InventoryClass::Disabled);
let manual = count(InventoryClass::Manual);
let isolated = count(InventoryClass::Isolated);
let unverified = reachable + isolated;
for node in &components {
let manual = reach::requires_manual_release_check(&node.name);
let isolated = reach::requires_isolated_outcome(&node.name);
let class = inventory_class(node, manual, isolated);
let counts = role_counts.entry(node.role.clone()).or_default();
counts[0] += 1;
match class {
InventoryClass::Reachable => counts[1] += 1,
InventoryClass::Unreachable => counts[2] += 1,
InventoryClass::Anonymous => counts[3] += 1,
InventoryClass::Disabled => counts[4] += 1,
InventoryClass::Manual => counts[5] += 1,
InventoryClass::Isolated => counts[6] += 1,
}
let (classification, reason) = match class {
InventoryClass::Manual => ("excluded-manual", "native-dialog-or-external"),
InventoryClass::Isolated => (
"isolated-unverified",
"requires disposable-process outcome check",
),
InventoryClass::Anonymous => ("failed-anonymous", "no accessible name"),
InventoryClass::Unreachable if !node.visible => ("failed-reachability", "hidden"),
InventoryClass::Unreachable => ("failed-reachability", "no-box"),
InventoryClass::Disabled => ("state-disabled", "disabled in current state"),
InventoryClass::Reachable => ("reachable-unverified", "no outcome check matched"),
};
controls.push(ControlRow {
surface: surface.name.clone(),
id: node.id,
role: node.role.clone(),
name: node.name.clone(),
classification: classification.to_owned(),
reason: reason.to_owned(),
});
}
rows.push(SurfaceRow {
surface: surface.name.clone(),
opened: true,
components: components.len(),
reachable,
unreachable,
anonymous,
disabled,
manual,
isolated,
unverified,
sections_opened,
rows_hovered,
});
}
let report = InventoryReport {
components: rows.iter().map(|row| row.components).sum(),
reachable: rows.iter().map(|row| row.reachable).sum(),
unreachable: rows.iter().map(|row| row.unreachable).sum(),
anonymous: rows.iter().map(|row| row.anonymous).sum(),
disabled: rows.iter().map(|row| row.disabled).sum(),
manual: rows.iter().map(|row| row.manual).sum(),
isolated: rows.iter().map(|row| row.isolated).sum(),
unverified: rows.iter().map(|row| row.unverified).sum(),
surfaces: rows,
roles: role_counts
.into_iter()
.map(|(role, counts)| RoleRow {
role,
components: counts[0],
reachable: counts[1],
unreachable: counts[2],
anonymous: counts[3],
disabled: counts[4],
manual: counts[5],
isolated: counts[6],
})
.collect(),
controls,
};
let failures = report.unreachable + report.anonymous;
println!(
"{}",
toon_format::encode_default(&report).map_err(|error| eyre!(error.to_string()))?
);
Ok(failures)
}
async fn open_surface(client: &mut Client, surface: &reach::Surface) -> Result<bool> {
if surface.opener.is_empty() {
return Ok(true);
}
let opener = if surface.opener == reach::DYNAMIC_DOCUMENT {
let (here, _) = inspect(client).await?;
if reach::document_opener(&here.nodes).is_none() {
if let Some(home) = reach::profile().home_opener.as_deref() {
let _ = click_named_quiet(client, home).await;
}
tokio::time::sleep(Duration::from_millis(600)).await;
}
let (tree, _) = inspect(client).await?;
match reach::document_opener(&tree.nodes) {
Some(name) => name,
None => return Ok(false),
}
} else {
surface.opener.to_owned()
};
if surface.opener == reach::DYNAMIC_DOCUMENT {
let (tree, _) = inspect(client).await?;
let Some(id) = tree
.nodes
.iter()
.find(|n| n.name == opener && reach::onscreen(n))
.map(|n| n.id)
else {
return Ok(false);
};
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id: id,
}))
.await?;
tokio::time::sleep(Duration::from_millis(400)).await;
let (tree, _) = inspect(client).await?;
if !tree.nodes.iter().any(|n| n.id == id && reach::onscreen(n)) {
return Ok(false);
}
client
.agent(&AgentControlRequest::Act(AgentAction::DoubleClick {
node_id: id,
}))
.await?;
} else if click_named_quiet(client, &opener).await.is_err() {
return Ok(false);
}
if settle_on(client, surface).await? {
return Ok(true);
}
if surface.opener == reach::DYNAMIC_DOCUMENT {
return Ok(false);
}
if let Some(home) = reach::profile().home_opener.as_deref() {
let _ = click_named_quiet(client, home).await;
}
tokio::time::sleep(Duration::from_millis(400)).await;
if click_named_quiet(client, &opener).await.is_err() {
return Ok(false);
}
settle_on(client, surface).await
}
async fn sweep_modal(
client: &mut Client,
opener: &str,
here: &mut reach::Coverage,
failures: &mut Vec<(String, String, String)>,
surface: &str,
) -> Result<bool> {
let (tree, _) = inspect(client).await?;
let dismiss_ids: Vec<u64> = reach::dismissers(&tree.nodes)
.into_iter()
.map(|(id, _)| id)
.collect();
let scope = dismiss_ids
.first()
.map(|id| reach::enclosing_dialog(&tree.nodes, *id))
.unwrap_or_default();
if cli::trace() {
for node in tree.nodes.iter().filter(|node| {
scope.contains(&node.id)
&& (reach::profile()
.deferred_controls
.iter()
.any(|name| node.name.eq_ignore_ascii_case(name))
|| reach::requires_isolated_outcome(&node.name))
}) {
println!(
" [modal] deferred to an outcome check: {:?}",
node.name
);
}
}
let inner: Vec<(u64, String)> = tree
.nodes
.iter()
.filter(|n| n.role == "button" && reach::onscreen(n) && n.enabled)
.filter(|n| !n.name.trim().is_empty())
.filter(|n| !dismiss_ids.contains(&n.id))
.filter(|n| scope.contains(&n.id))
.filter(|n| !reach::requires_manual_release_check(&n.name))
.filter(|n| !reach::requires_isolated_outcome(&n.name))
.filter(|n| {
!reach::profile()
.deferred_controls
.iter()
.any(|name| n.name.eq_ignore_ascii_case(name))
})
.map(|n| (n.id, n.name.clone()))
.collect();
for (id, name) in inner {
let (before, _) = inspect(client).await?;
if !before
.nodes
.iter()
.any(|n| n.id == id && reach::onscreen(n))
{
continue;
}
if click_by_id(client, id).await.is_err() {
continue;
}
here.revealed += 1;
if cli::trace() {
println!(" [modal] clicked: {name:?}");
}
tokio::time::sleep(Duration::from_millis(180)).await;
let (after, _) = inspect(client).await?;
let case = sweep::Case {
id,
name: name.clone(),
family: audit::family_of(&name),
expect: sweep::expectation_for(&name, reach::is_inert_control(&name)),
};
if let Some(why) = sweep::judge(&case, &before.nodes, &after.nodes) {
failures.push((
surface.to_owned(),
format!("{name} (in {opener} dialog)"),
why,
));
}
let (now, _) = inspect(client).await?;
if !reach::modal_open(&now.nodes) {
return Ok(false);
}
}
let (tree, _) = inspect(client).await?;
for (id, name) in reach::dismissers(&tree.nodes) {
if click_by_id(client, id).await.is_err() {
continue;
}
here.revealed += 1;
tokio::time::sleep(Duration::from_millis(250)).await;
let (after, _) = inspect(client).await?;
if !reach::modal_open(&after.nodes) {
return Ok(false);
}
failures.push((
surface.to_owned(),
format!("{name} (in {opener} dialog)"),
"the dialog is still open; it did not dismiss".to_owned(),
));
}
let _ = press_key(client, "escape", 1, "").await;
tokio::time::sleep(Duration::from_millis(300)).await;
let (after, _) = inspect(client).await?;
if !reach::modal_open(&after.nodes) {
return Ok(false);
}
failures.push((
surface.to_owned(),
format!("{opener} dialog"),
"TRAPPED: no dismiss control and no Escape closes it".to_owned(),
));
Ok(true)
}
async fn settle_on(client: &mut Client, surface: &reach::Surface) -> Result<bool> {
for _ in 0..12 {
tokio::time::sleep(Duration::from_millis(400)).await;
let (tree, _) = inspect(client).await?;
if reach::on_surface(&tree.nodes, surface) {
return Ok(true);
}
}
Ok(false)
}
async fn run_cover(client: &mut Client, only: Option<&str>) -> Result<usize> {
let mut total = reach::Coverage::default();
let mut failures: Vec<(String, String, String)> = Vec::new();
let mut skipped_manual: Vec<String> = Vec::new();
let mut skipped_isolated: Vec<String> = Vec::new();
for surface in reach::surfaces() {
if only.is_some_and(|want| want != surface.name) {
continue;
}
if !open_surface(client, surface).await? {
println!("- {:<10} could not be opened, skipping\n", surface.name);
continue;
}
let opened = expand_everything(client, surface).await?;
if !open_surface(client, surface).await? {
println!("- {:<10} left during expansion, skipping\n", surface.name);
continue;
}
let hovered = hover_all_rows(client).await?;
let (tree, _) = inspect(client).await?;
let mine: std::collections::HashSet<u64> = reach::on_surface_subtree(&tree.nodes, surface)
.into_iter()
.collect();
let buttons: Vec<&blitz_control_protocol::SemanticNode> = tree
.nodes
.iter()
.filter(|n| n.role == "button" && !n.name.trim().is_empty())
.filter(|n| n.visible)
.filter(|n| mine.contains(&n.id))
.collect();
let retained = tree
.nodes
.iter()
.filter(|n| n.role == "button" && !n.name.trim().is_empty() && !n.visible)
.count();
let mut here = reach::Coverage {
in_tree: buttons.len(),
hidden: 0,
..Default::default()
};
let mut plan: Vec<(u64, String)> = Vec::new();
let mut collapsers: Vec<(u64, String)> = Vec::new();
let mut closers: Vec<(u64, String)> = Vec::new();
for node in &buttons {
if !reach::onscreen(node) {
here.unreachable += 1;
} else if reach::requires_isolated_outcome(&node.name) {
here.isolated += 1;
skipped_isolated.push(node.name.clone());
} else if reach::profile()
.fold_prefixes
.iter()
.any(|p| node.name.to_lowercase().starts_with(&p.to_lowercase()))
|| reach::profile()
.deferred_controls
.iter()
.any(|c| node.name.eq_ignore_ascii_case(c))
|| reach::folds_a_section(&node.name)
{
collapsers.push((node.id, node.name.clone()));
} else if reach::navigates(&node.name)
|| reach::opens_document_row(&tree.nodes, node.id)
{
here.navigation += 1;
} else if reach::requires_manual_release_check(&node.name) {
here.manual += 1;
skipped_manual.push(node.name.clone());
} else if reach::closes_a_surface(&node.name) {
closers.push((node.id, node.name.clone()));
} else {
plan.push((node.id, node.name.clone()));
}
}
plan.sort_by_key(|(_, name)| {
let lower = name.to_lowercase();
u8::from(
lower.starts_with("delete ")
|| lower.starts_with("close ")
|| lower.starts_with("remove ")
|| lower.starts_with("retire "),
)
});
plan.extend(collapsers);
plan.extend(closers);
let mut done: HashMap<String, usize> = HashMap::new();
let planned_total = plan.len();
for (index, (planned_id, name)) in plan.into_iter().enumerate() {
let remaining = planned_total.saturating_sub(index + 1);
let (mut before, _) = inspect(client).await?;
if !reach::on_surface(&before.nodes, surface) {
if !open_surface(client, surface).await? {
here.vanished += 1;
if cli::trace() {
println!(" left surface, could not return: {name:?}");
}
continue;
}
let (fresh, _) = inspect(client).await?;
before = fresh;
}
let seen = done.entry(name.clone()).or_insert(0);
let found = before
.nodes
.iter()
.find(|n| n.id == planned_id && n.name == name && reach::onscreen(n))
.or_else(|| {
before
.nodes
.iter()
.filter(|n| n.role == "button" && n.name == name && reach::onscreen(n))
.nth(*seen)
});
let Some(node) = found else {
here.vanished += 1;
if cli::trace() {
println!(" vanished: {name:?} (id {planned_id})");
}
continue;
};
*seen += 1;
let id = node.id;
if !reach::onscreen(node) || !node.enabled {
here.vanished += 1;
if cli::trace() {
let visible_buttons = before
.nodes
.iter()
.filter(|n| n.role == "button" && reach::onscreen(n))
.count();
println!(
" offscreen/disabled: {name:?} visible={} bounds={:?} \
[on_surface={}, {visible_buttons} buttons on screen]",
node.visible,
node.bounds,
reach::on_surface(&before.nodes, surface),
);
}
continue;
}
let case = sweep::Case {
id,
name: name.clone(),
family: audit::family_of(&name),
expect: sweep::expectation_for(&name, reach::is_inert_control(&name)),
};
if click_by_id(client, id).await.is_err() {
here.vanished += 1;
continue;
}
here.swept += 1;
if cli::trace() {
println!(" clicked: {name:?}");
}
tokio::time::sleep(Duration::from_millis(200)).await;
let (after_click, _) = inspect(client).await?;
let after = if reach::modal_open(&after_click.nodes) {
let trapped =
sweep_modal(client, &name, &mut here, &mut failures, &surface.name).await?;
if trapped {
println!(
" ! {:?} opened a dialog that will not dismiss - \
the rest of this surface is unreachable behind it",
name
);
here.blocked += remaining;
break;
}
inspect(client).await?.0
} else {
after_click
};
if sweep::judge(&case, &before.nodes, &after.nodes).is_some() {
tokio::time::sleep(Duration::from_millis(800)).await;
let (settled, _) = inspect(client).await?;
if let Some(why) = sweep::judge(&case, &before.nodes, &settled.nodes) {
failures.push((surface.name.to_owned(), name, why));
}
}
}
println!(
"= {:<10} {} ({} sections opened, {} rows hovered, {} retained elsewhere)",
surface.name,
here.line(),
opened,
hovered,
retained
);
total.in_tree += here.in_tree;
total.swept += here.swept;
total.unreachable += here.unreachable;
total.hidden += here.hidden;
total.vanished += here.vanished;
total.navigation += here.navigation;
total.manual += here.manual;
total.isolated += here.isolated;
total.blocked += here.blocked;
total.revealed += here.revealed;
}
println!("\n{}", total.line());
if total.manual > 0 {
println!(
"\n{} control(s) need the manual release pass:",
total.manual
);
let mut seen: Vec<&str> = skipped_manual.iter().map(String::as_str).collect();
seen.sort_unstable();
seen.dedup();
for label in seen {
let command = reach::profile()
.manual_controls
.iter()
.find(|exception| label.starts_with(exception.label.as_str()))
.map(|exception| exception.command.as_str())
.unwrap_or("(unmapped manual control)");
println!(" {label:<38} {command}");
}
}
if total.isolated > 0 {
println!(
"\n{} control(s) require an isolated outcome check:",
total.isolated
);
skipped_isolated.sort_unstable();
skipped_isolated.dedup();
for label in skipped_isolated {
println!(" {label}");
}
}
if failures.is_empty() {
if total.isolated > 0 {
println!("every broadly swept button acted; isolated controls remain listed above");
} else {
println!("every reached button acted");
}
} else {
println!("\n{} did not act:\n", failures.len());
for (surface, name, why) in &failures {
println!(
" [{surface}] {:<40} {why}",
name.chars().take(40).collect::<String>()
);
}
}
Ok(failures.len())
}
#[tokio::main(flavor = "current_thread")]
async fn main() -> Result<()> {
let cli = <cli::Cli as clap::Parser>::parse();
cli::set_trace(cli.trace);
cli::set_pace(cli.pace);
cli::set_app_profile(cli.app.clone());
if let cli::Command::List { checks } = &cli.command {
print!(
"{}",
qa::manifest(checks.as_deref()).map_err(|error| eyre!(error))?
);
return Ok(());
}
app::AppProfile::load(cli.app.as_deref()).map_err(|error| eyre!(error))?;
let descriptor = inspector::discover(cli.descriptor.as_deref().and_then(|p| p.to_str()))?;
descriptor.warn_if_stale();
let verbose = cli.command.is_dump();
if verbose {
println!("descriptor: {}", descriptor.path.display());
println!("{}", report::dump(&descriptor.raw, usize::MAX));
}
let mut client = Client::connect(&descriptor.socket_path()).await?;
let initialize = client.initialize().await?;
if verbose {
println!("\n== initialize ==");
println!("{}", report::dump(&initialize, 800));
println!("\n== tools ==");
let tools = client.tools_list().await?;
println!("{}", report::dump(&tools, 1200));
}
match cli.command {
cli::Command::Metrics => {
println!("\n== metrics ==");
let answer = client.diagnostics(&DiagnosticsRequest::Metrics).await?;
println!("{}", report::dump(&answer.envelope, 2000));
}
cli::Command::Watch { seconds } => {
println!("\n== observing metrics/console/runtimeErrors for {seconds}s ==");
let answer = client
.diagnostics_envelope(&DiagnosticsRequest::Observe {
streams: vec![
DebugStream::Metrics,
DebugStream::Console,
DebugStream::RuntimeErrors,
],
})
.await?;
println!("{}", report::dump(&answer, 400));
for message in client.drain(seconds).await? {
println!("{}", report::dump(&message, 400));
}
}
cli::Command::Frames => report::show_frames(&metrics(&mut client).await?),
cli::Command::Tree => {
println!("\n== semantic tree ==");
let answer = client
.agent(&AgentControlRequest::Inspect {
root: None,
max_depth: 3,
})
.await?;
println!("{}", report::dump(&answer.envelope, 3000));
}
cli::Command::Find {
pattern,
role,
visible,
hidden,
painted,
offscreen,
disabled,
count,
limit,
} => {
find(
&mut client,
&pattern,
&role,
visible,
hidden,
painted,
offscreen,
disabled,
count,
limit,
)
.await?;
}
cli::Command::Layout { name } => {
layout(&mut client, &name).await?;
}
cli::Command::Transcript => transcript(&mut client).await?,
cli::Command::Paint { name, min_area } => {
paint(&mut client, &name, min_area).await?;
}
cli::Command::Nodes => {
nodes(&mut client).await?;
}
cli::Command::Panes => panes(&mut client).await?,
cli::Command::Dom { name, depth } => {
dom(&mut client, &name, depth).await?;
}
cli::Command::Spill { axis, tolerance } => {
spill(&mut client, &axis, tolerance).await?;
}
cli::Command::Idle => report::show("idle", &metrics(&mut client).await?),
cli::Command::Drift { seconds } => {
let before = metrics(&mut client).await?;
println!("== holding still for {seconds}s, nothing driven ==");
tokio::time::sleep(Duration::from_secs_f64(seconds)).await;
let after = metrics(&mut client).await?;
let frames_of =
|m: &RendererMetrics| m.frame_window.as_ref().map(|w| w.frames_total).unwrap_or(0);
let frames = frames_of(&after).saturating_sub(frames_of(&before));
println!(
"frames={frames} over {seconds}s = {:.1}fps with no input",
frames as f64 / seconds
);
report::show_delta(&before, &after, 0);
}
cli::Command::Ghost { min_area, max } => {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: true,
include_layout: true,
include_computed_style: false,
}))
.await?;
let DebugResponse::Snapshot(snapshot) = answer.response else {
bail!("asked for a layout snapshot, got {:?}", answer.response);
};
let mut boxes: HashMap<u64, (f64, f64, f64, f64)> = HashMap::new();
if let Some(rows) = snapshot.layout.as_ref().and_then(|v| v.as_array()) {
for row in rows {
let Some(id) = row.get("nodeId").and_then(|v| v.as_u64()) else {
continue;
};
let read = |key: &str, index: usize| {
row.get("bounds")
.and_then(|b| b.get(key).or_else(|| b.get(index)))
.and_then(|v| v.as_f64())
.unwrap_or(0.0)
};
boxes.insert(
id,
(
read("x", 0),
read("y", 1),
read("width", 2),
read("height", 3),
),
);
}
}
let nodes = snapshot
.dom
.as_ref()
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
let mut ghosts = Vec::new();
for node in &nodes {
let visible = node
.get("visible")
.and_then(|v| v.as_bool())
.unwrap_or(true);
if visible {
continue;
}
let Some(id) = node.get("id").and_then(|v| v.as_u64()) else {
continue;
};
let Some(&(x, y, w, h)) = boxes.get(&id) else {
continue;
};
if w * h >= min_area {
let name = node
.get("name")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
ghosts.push((id, name, x, y, w, h));
}
}
ghosts.sort_by(|a, b| (b.4 * b.5).total_cmp(&(a.4 * a.5)));
println!(
"{} nodes inspected, reporting hidden boxes of {min_area}px2 or more",
nodes.len()
);
for (id, name, x, y, w, h) in &ghosts {
println!(" {id:>10} {w:>7.1}x{h:<7.1} at {x:.0},{y:.0} {name}");
}
if ghosts.is_empty() {
println!("\nno ghosts: nothing hidden is holding a painted box");
} else {
println!(
"\nGHOSTS PRESENT: {} hidden node(s) still occupy layout",
ghosts.len()
);
if ghosts.len() > max {
println!(
"over budget: {} ghosts, limit {max}. Hidden subtrees are not \
being unmounted.",
ghosts.len()
);
std::process::exit(1);
}
println!("within budget: limit {max}");
}
}
cli::Command::Blink { allowed_missed } => {
let reading = metrics(&mut client).await?;
report::show("blink", &reading);
let Some(window) = reading.frame_window.as_ref() else {
eyre::bail!(
"the app published no frame window, so there is nothing to assert; \
launch it with --blitz-deep-profiling"
);
};
let period_ms = 1000.0
/ window
.display_refresh_hz
.filter(|hz| *hz > 0.0)
.unwrap_or(60.0);
let interval_budget = period_ms * 2.0;
let worst_interval = window.interval.max_ms;
println!();
println!("== the reported repro: a project with 0 items, item list expanded ==");
println!(
" missed refreshes : {} over {} frames (allowed {allowed_missed})",
window.missed_refreshes, window.window_frames
);
println!(
" worst interval : {worst_interval:.1}ms against a {period_ms:.1}ms refresh \
(budget {interval_budget:.1}ms)"
);
let mut faults = Vec::new();
if window.missed_refreshes > allowed_missed {
faults.push(format!(
"{} missed refreshes over {} frames",
window.missed_refreshes, window.window_frames
));
}
if worst_interval > interval_budget {
faults.push(format!(
"a {worst_interval:.1}ms frame interval, {:.1}x the refresh period",
worst_interval / period_ms
));
}
if faults.is_empty() {
println!("\nno blink: the window is quiet by both measures");
} else {
println!("\nBLINK PRESENT: {}", faults.join(", "));
std::process::exit(1);
}
}
cli::Command::Click { name, id } => {
nodes(&mut client).await?;
match (name, id) {
(Some(name), None) => click_named(&mut client, &name).await?,
(None, Some(node_id)) => {
click_by_id(&mut client, node_id).await?;
tokio::time::sleep(Duration::from_millis(400)).await;
println!("activated node {node_id}");
}
_ => unreachable!("clap requires exactly one click selector"),
}
}
cli::Command::Capture { name, scale } => {
capture(&mut client, &name, scale as f32).await?;
}
cli::Command::Audit { family } => {
let faults = run_audit(&mut client, family.as_deref()).await?;
if faults > 0 {
std::process::exit(1);
}
}
cli::Command::Sweep { family } => {
let failures = run_sweep(&mut client, family.as_deref()).await?;
if failures > 0 {
std::process::exit(1);
}
}
cli::Command::Press { name } => {
let (snapshot, _) = inspect(&mut client).await?;
let wanted = name.to_lowercase();
let Some(node) = snapshot
.nodes
.iter()
.filter(|n| n.name.to_lowercase().contains(&wanted))
.filter(|n| n.visible && n.enabled)
.find(|n| n.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0))
else {
bail!("no visible, enabled, sized node matching {name:?}");
};
let b = node.bounds.unwrap();
let (x, y) = (b[0] + b[2] / 2.0, b[1] + b[3] / 2.0);
println!("pressing {:?} at {x:.0},{y:.0}", node.name);
for phase in [PointerPhase::Move, PointerPhase::Down, PointerPhase::Up] {
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Pointer {
phase,
x,
y,
button: 0,
modifiers: Modifiers::default(),
},
)))
.await?;
}
tokio::time::sleep(Duration::from_millis(400)).await;
println!("pressed");
}
cli::Command::Cover { surface } => {
let failures = run_cover(&mut client, surface.as_deref()).await?;
if failures > 0 {
std::process::exit(1);
}
}
cli::Command::Inventory { surface } => {
let failures = run_inventory(&mut client, surface.as_deref()).await?;
if failures > 0 {
std::process::exit(1);
}
}
cli::Command::Qa { selector, checks } => {
let failed = run_qa(&mut client, selector.as_deref(), checks.as_deref()).await?;
if failed > 0 {
std::process::exit(1);
}
}
cli::Command::Reveal { name } => {
let (snapshot, _) = inspect(&mut client).await?;
let Some(target) = snapshot
.nodes
.iter()
.find(|node| node.name.contains(&name) && node.bounds.is_some())
else {
bail!("no node named {name:?}");
};
let before = target.bounds.unwrap();
let id = target.id;
println!("{id} {:?} y={:.1}", target.role, before[1]);
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id: id,
}))
.await?;
tokio::time::sleep(Duration::from_millis(400)).await;
let (after_snapshot, _) = inspect(&mut client).await?;
let after = after_snapshot
.nodes
.iter()
.find(|node| node.id == id)
.and_then(|node| node.bounds);
match after {
Some(b) => println!("after: y={:.1} (moved {:.1})", b[1], b[1] - before[1]),
None => println!("after: the node is gone from the tree"),
}
}
cli::Command::Key { name, count, over } => {
let fallback = reach::profile()
.transcript_region
.clone()
.unwrap_or_default();
let over = if over.is_empty() { &fallback } else { &over };
press_key(&mut client, &name, count as usize, over).await?;
}
cli::Command::Type { count, name } => {
nodes(&mut client).await?;
type_keys(&mut client, count as usize, &name).await?;
}
cli::Command::Drag { name, dy, steps } => {
let fallback = reach::profile()
.transcript_region
.clone()
.unwrap_or_default();
let name = if name.is_empty() { &fallback } else { &name };
let times = steps as usize;
let (snapshot, _) = inspect(&mut client).await?;
let Some(target) = snapshot
.nodes
.iter()
.filter(|node| node.visible && node.name.contains(name))
.filter_map(|node| node.bounds.map(|b| (node, b)))
.max_by(|a, b| {
(a.1[2] * a.1[3])
.partial_cmp(&(b.1[2] * b.1[3]))
.unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(node, _)| node.id)
else {
bail!("no visible node named {name:?}");
};
println!("scrolling node {target} by {dy} x{times}");
for _ in 0..times {
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollBy {
node_id: target,
delta_x: 0.0,
delta_y: dy,
}))
.await?;
sleep_pace().await;
}
tokio::time::sleep(Duration::from_millis(300)).await;
}
cli::Command::Scroll { ticks, delta, over } => {
let fallback = reach::profile()
.transcript_region
.clone()
.unwrap_or_default();
let over = if over.is_empty() { &fallback } else { &over };
let count = nodes(&mut client).await?;
hover_over(&mut client, over).await?;
report::show("before", &metrics(&mut client).await?);
scroll(&mut client, ticks as usize, delta).await?;
report::show("after", &metrics(&mut client).await?);
println!("tree size during run: {count} nodes");
}
cli::Command::List { .. } => unreachable!("handled before the client connects"),
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{InventoryClass, inventory_class, name_matches};
use blitz_control_protocol::SemanticNode;
fn component(name: &str, enabled: bool, visible: bool) -> SemanticNode {
SemanticNode {
id: 1,
parent: None,
role: "button".into(),
name: name.into(),
value: None,
enabled,
visible,
selected: false,
bounds: Some([0.0, 0.0, 20.0, 20.0]),
}
}
#[test]
fn inventory_categories_are_mutually_exclusive() {
assert_eq!(
inventory_class(&component("Import data", true, true), true, false),
InventoryClass::Manual
);
assert_eq!(
inventory_class(&component("Restart application", true, true), false, true),
InventoryClass::Isolated
);
assert_eq!(
inventory_class(&component("", false, false), false, false),
InventoryClass::Anonymous
);
assert_eq!(
inventory_class(&component("Save", false, true), false, false),
InventoryClass::Disabled
);
assert_eq!(
inventory_class(&component("Hidden", true, false), false, false),
InventoryClass::Unreachable
);
assert_eq!(
inventory_class(&component("Synchronize", true, true), false, false),
InventoryClass::Reachable
);
}
#[test]
fn a_bare_pattern_is_a_substring() {
assert!(name_matches("Rename project", "rename"));
assert!(name_matches("Rename project", "project"));
assert!(!name_matches("Rename project", "delete"));
}
#[test]
fn a_trailing_star_anchors_the_front() {
assert!(name_matches("chat with agent", "chat*"));
assert!(!name_matches("open chat", "chat*"));
}
#[test]
fn a_leading_star_anchors_the_end() {
assert!(name_matches("open chat", "*chat"));
assert!(!name_matches("chat with agent", "*chat"));
}
#[test]
fn stars_at_both_ends_match_anywhere() {
assert!(name_matches("the chat panel", "*chat*"));
assert!(!name_matches("Rename project", "*chat*"));
}
#[test]
fn matching_ignores_case() {
assert!(name_matches("Rename Project", "rename project"));
assert!(name_matches("CHAT", "chat*"));
}
#[test]
fn a_lone_star_matches_everything() {
assert!(name_matches("anything at all", "*"));
assert!(name_matches("", "*"));
}
}