use std::collections::{HashMap, HashSet};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::time::{Duration, Instant};
use blitz_control_protocol::{
AgentAction, AgentControlRequest, AgentSnapshot, CaptureRequest, CapturedImage, DebugResponse,
DebugStream, DiagnosticsRequest, InputCommand, Modifiers, PointerPhase, RendererMetrics,
SemanticNode, SnapshotRequest, WindowComposition,
};
use eyre::{Context, Result, bail, eyre};
use crate::capture_analysis::{
CHANNEL_TOLERANCE as CAPTURE_CHANNEL_TOLERANCE, measure_ink, measure_interior_ink,
pixel_delta as captured_pixel_delta, pixels_change, pixels_hold,
pixels_hold_with_tolerance as captured_pixels_hold, rgb_pixels_hold,
save_artifacts as save_pixel_artifacts, write_ppm as write_capture_ppm,
};
use crate::computed_style::{
font_size, full_opacity, opaque_background, transparent_background, wait_for_larger_font,
};
use crate::diagnostics::{dom, metrics, nodes, panes, spill, transcript};
use crate::inspector::{Client, inspect, inspect_subtree};
use crate::interaction::{
TextEntryFailure, click_named, hover_over, key_steps, pointer_drag, press_key, scroll,
scroll_events, type_keys, type_text,
};
use crate::layout_report::layout;
use crate::target::{
exact_selector_matches_node, locate_control, name_matches, offscreen, painted_bounds,
painted_named, resolved_action_target, selector_matches_node, viewport_of,
};
use crate::timing::{check_timeout, sleep_pace};
use crate::{app, audit, cli, inspector, paint_audit, plan, qa, reach, report, sweep};
async fn wait_for_arrival(
client: &mut Client,
destination: Option<&reach::Surface>,
want_here: &str,
within: Duration,
) -> Result<bool> {
let deadline = std::time::Instant::now() + within;
let mut painted_streak = 0;
let mut root = None;
loop {
let (tree, scoped) = if let Some(node_id) = root {
match inspect_subtree(client, node_id).await {
Ok((tree, _)) => (tree, true),
Err(_) => {
root = None;
painted_streak = 0;
(inspect(client).await?.0, false)
}
}
} else {
(inspect(client).await?.0, false)
};
let arrived = arrival_sample_matches(&tree.nodes, destination, want_here, scoped);
if arrived && root.is_none() {
root = arrival_anchor(&tree.nodes, destination, want_here);
} else if !arrived && root.is_some() {
root = None;
}
if stable_arrival(&mut painted_streak, arrived) {
return Ok(true);
}
if std::time::Instant::now() >= deadline {
return Ok(false);
}
nagoya::sleep(Duration::from_millis(25)).await;
}
}
async fn wait_for_visible_arrival(
client: &mut Client,
want_here: &str,
within: Duration,
) -> Result<bool> {
let deadline = std::time::Instant::now() + within;
let mut painted_streak = 0;
loop {
let (tree, _) = inspect(client).await?;
let arrived = arrival_anchor(&tree.nodes, None, want_here).is_some();
if stable_arrival(&mut painted_streak, arrived) {
return Ok(true);
}
if std::time::Instant::now() >= deadline {
return Ok(false);
}
nagoya::sleep(Duration::from_millis(25)).await;
}
}
fn arrival_sample_matches(
nodes: &[SemanticNode],
destination: Option<&reach::Surface>,
want_here: &str,
scoped: bool,
) -> bool {
if scoped {
arrival_anchor(nodes, destination, want_here).is_some()
} else {
destination.map_or_else(
|| painted_named(nodes, want_here),
|surface| reach::on_surface(nodes, surface),
)
}
}
fn arrival_anchor(
nodes: &[SemanticNode],
destination: Option<&reach::Surface>,
want_here: &str,
) -> Option<u64> {
let marker = destination.and_then(|surface| surface.marker.as_deref());
nodes
.iter()
.find(|node| {
reach::onscreen(node)
&& marker.map_or_else(
|| selector_matches_node(node, want_here),
|marker| node.name.contains(marker),
)
})
.map(|node| node.id)
}
async fn wait_for_navigation_arrival(
client: &mut Client,
destination: Option<&reach::Surface>,
want_here: &str,
named_document: bool,
document_name: &str,
within: Duration,
) -> Result<bool> {
if !named_document {
return wait_for_arrival(client, destination, want_here, within).await;
}
let deadline = std::time::Instant::now() + within;
let mut painted_streak = 0;
let mut selected_tab = None;
loop {
let (tree, scoped) = if let Some(node_id) = selected_tab {
match inspect_subtree(client, node_id).await {
Ok((tree, _)) => (tree, true),
Err(_) => {
selected_tab = None;
painted_streak = 0;
(inspect(client).await?.0, false)
}
}
} else {
(inspect(client).await?.0, false)
};
let arrived = named_document_is_active(&tree.nodes, document_name)
&& (scoped
|| destination.is_some_and(|surface| reach::on_surface(&tree.nodes, surface)));
if arrived && selected_tab.is_none() {
let tab_name = format!("{document_name}{document_name}");
selected_tab = tree
.nodes
.iter()
.find(|node| {
node.role.eq_ignore_ascii_case("button")
&& (node.name.eq_ignore_ascii_case(document_name)
|| node.name.eq_ignore_ascii_case(&tab_name))
&& node.selected
&& reach::onscreen(node)
})
.map(|node| node.id);
} else if !arrived && selected_tab.is_some() {
selected_tab = None;
}
if stable_arrival(&mut painted_streak, arrived) {
return Ok(true);
}
if std::time::Instant::now() >= deadline {
return Ok(false);
}
nagoya::sleep(Duration::from_millis(25)).await;
}
}
fn stable_arrival(painted_streak: &mut u8, arrived: bool) -> bool {
if arrived {
*painted_streak = painted_streak.saturating_add(1);
} else {
*painted_streak = 0;
}
*painted_streak >= 3
}
async fn wait_for_semantic_condition(
client: &mut Client,
within: Duration,
mut matches: impl FnMut(&[SemanticNode]) -> bool,
) -> Result<AgentSnapshot> {
let deadline = std::time::Instant::now() + within;
let event_driven = client.arm_paint_events().await.unwrap_or(false);
loop {
let snapshot = inspect(client).await?.0;
if matches(&snapshot.nodes) || std::time::Instant::now() >= deadline {
return Ok(snapshot);
}
if event_driven {
let remaining = deadline.saturating_duration_since(std::time::Instant::now());
if !remaining.is_zero() {
let _ = client.wait_for_paint(remaining).await?;
}
} else {
nagoya::sleep(Duration::from_millis(25)).await;
}
}
}
fn setup_value_landed(nodes: &[SemanticNode], node_id: u64, value: &str) -> bool {
nodes
.iter()
.any(|node| node.id == node_id && node.value.as_deref() == Some(value))
}
async fn settle_sweep_case(
client: &mut Client,
case: &sweep::Case,
before: &[SemanticNode],
) -> Result<AgentSnapshot> {
wait_for_semantic_condition(client, check_timeout(900), |after| {
sweep::judge(case, before, after).is_none()
})
.await
}
struct HostProcess {
child: std::process::Child,
control_dir: Option<std::path::PathBuf>,
}
impl Drop for HostProcess {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
if let Some(path) = &self.control_dir
&& control_directory_is_real(path)
{
let _ = std::fs::remove_dir_all(path);
}
}
}
struct HostedSession {
host: std::path::PathBuf,
page: std::path::PathBuf,
startup_timeout: Duration,
quiet_output: bool,
clean_environment: bool,
storage: Option<std::path::PathBuf>,
seeds_storage: bool,
process: HostProcess,
descriptor_path: std::path::PathBuf,
}
fn launch_host(
host: &std::path::Path,
page: &std::path::Path,
startup_timeout: Duration,
quiet_output: bool,
clean_environment: bool,
storage: Option<&std::path::Path>,
) -> std::result::Result<(HostProcess, std::path::PathBuf), String> {
if !quiet_output && !clean_environment {
start_host(host, page, startup_timeout, storage)
} else {
start_host_with_policy(
host,
page,
startup_timeout,
quiet_output,
clean_environment,
storage,
)
}
}
impl HostedSession {
fn start(
host: &std::path::Path,
page: &std::path::Path,
startup_timeout: Duration,
quiet_output: bool,
clean_environment: bool,
) -> std::result::Result<Self, String> {
let (process, descriptor_path) = launch_host(
host,
page,
startup_timeout,
quiet_output,
clean_environment,
None,
)?;
Ok(Self {
host: host.to_path_buf(),
page: page.to_path_buf(),
startup_timeout,
quiet_output,
clean_environment,
storage: None,
seeds_storage: false,
process,
descriptor_path,
})
}
fn restart(&mut self) -> std::result::Result<(), String> {
let _ = self.process.child.kill();
let _ = self.process.child.wait();
if let Some(path) = self.process.control_dir.take()
&& control_directory_is_real(&path)
{
let _ = std::fs::remove_dir_all(path);
}
let (process, descriptor_path) = launch_host(
&self.host,
&self.page,
self.startup_timeout,
self.quiet_output,
self.clean_environment,
self.storage.as_deref(),
)?;
self.process = process;
self.descriptor_path = descriptor_path;
Ok(())
}
}
impl Drop for HostedSession {
fn drop(&mut self) {
if let Some(path) = self.storage.take() {
let _ = std::fs::remove_file(path);
}
}
}
type OneCheck = (std::result::Result<(), String>, u32, u32);
fn host_gave_up(error: &str) -> bool {
const MARKERS: &[&str] = &[
"pageUnresponsive",
"the page did not answer the control request",
"the page exceeded its Poll deadline",
"inspector did not answer within",
"the inspector closed the connection",
"sending to the inspector failed",
"reading from the inspector failed",
];
MARKERS.iter().any(|marker| error.contains(*marker))
}
fn host_gave_up_report(error: &eyre::Report) -> bool {
let head = error.to_string();
host_gave_up(&head) || host_gave_up(&format!("{error:#}"))
}
fn host_failure_text(error: &eyre::Report) -> String {
let head = error.to_string();
if host_gave_up(&head) {
return head;
}
let chain = format!("{error:#}");
if host_gave_up(&chain) { chain } else { head }
}
fn check_recovery_budget(client: &Client, check: &qa::Check) -> Duration {
client
.request_timeout()
.max(declared_open_timeout(check))
.max(declared_outcome_timeout(check))
}
async fn reconnect_hosted(client: &mut Client, hosted: &mut HostedSession) -> Result<()> {
hosted.restart().map_err(|error| eyre!(error))?;
let descriptor_path = hosted
.descriptor_path
.to_str()
.ok_or_else(|| eyre!("host descriptor path is not valid utf-8"))?;
let descriptor = inspector::discover(Some(descriptor_path))?;
let mut fresh = Client::connect(&descriptor.socket_path()).await?;
fresh.initialize().await?;
*client = fresh;
Ok(())
}
async fn inspector_answers_again(client: &mut Client, budget: Duration) -> bool {
if budget.is_zero() {
return false;
}
let previous = client.request_timeout();
client.set_request_timeout(budget);
let answered = match client.tools_list().await {
Ok(_) => true,
Err(error) => !host_gave_up_report(&error),
};
client.set_request_timeout(previous);
answered
}
async fn away_target(client: &mut Client) -> Result<Option<u64>> {
let (snapshot, _) = inspect(client).await?;
Ok(away_target_in(&snapshot.nodes))
}
fn away_target_in(nodes: &[SemanticNode]) -> Option<u64> {
nodes
.iter()
.filter(|node| node.visible)
.filter_map(|node| painted_bounds(node).map(|b| (node.id, b[2] * b[3])))
.max_by(|a, b| a.1.total_cmp(&b.1))
.map(|(id, _)| id)
}
async fn park_pointer(client: &mut Client) -> std::result::Result<(), String> {
if let Some(root) = away_target(client)
.await
.map_err(|error| error.to_string())?
{
client
.agent(&AgentControlRequest::Act(AgentAction::Hover {
node_id: root,
}))
.await
.map_err(|error| error.to_string())?;
}
Ok(())
}
async fn repeat_hover(
client: &mut Client,
hover: &qa::Hover,
leave_after: bool,
) -> std::result::Result<(), String> {
let want = hover.target();
let node_id = scroll_hover_target_into_view(client, want).await?;
if cli::trace() {
println!(" hovering {want:?} (id {node_id})");
}
if let Some(root) = away_target(client)
.await
.map_err(|error| error.to_string())?
{
client
.agent(&AgentControlRequest::Act(AgentAction::Hover {
node_id: root,
}))
.await
.map_err(|error| error.to_string())?;
}
for turn in 0..hover.times() {
if turn > 0
&& let Some(root) = away_target(client)
.await
.map_err(|error| error.to_string())?
{
client
.agent(&AgentControlRequest::Act(AgentAction::Hover {
node_id: root,
}))
.await
.map_err(|error| error.to_string())?;
}
client
.agent(&AgentControlRequest::Act(AgentAction::Hover { node_id }))
.await
.map_err(|error| error.to_string())?;
}
if leave_after
&& let Some(root) = away_target(client)
.await
.map_err(|error| error.to_string())?
{
client
.agent(&AgentControlRequest::Act(AgentAction::Hover {
node_id: root,
}))
.await
.map_err(|error| error.to_string())?;
}
Ok(())
}
type HoverSignature = (String, String, Option<String>, Option<String>);
fn hover_signature_counts(
nodes: &[SemanticNode],
) -> std::collections::BTreeMap<HoverSignature, usize> {
let scope = outcome_poll_scope(nodes).map(|scope| scope.baseline_ids);
let mut counts = std::collections::BTreeMap::new();
for node in nodes.iter().filter(|node| {
scope
.as_ref()
.is_none_or(|surface_ids| surface_ids.contains(&node.id))
}) {
if node.name.is_empty() && node.slot.is_none() && node.dom_id.is_none() {
continue;
}
let signature = (
node.role.to_ascii_lowercase(),
node.name.clone(),
node.slot.clone(),
node.dom_id.clone(),
);
*counts.entry(signature).or_insert(0) += 1;
}
counts
}
fn accumulated_hover_signatures(
baseline: &std::collections::BTreeMap<HoverSignature, usize>,
after: &std::collections::BTreeMap<HoverSignature, usize>,
) -> Vec<(HoverSignature, usize)> {
after
.iter()
.filter_map(|(signature, count)| {
let before = baseline.get(signature).copied().unwrap_or_default();
(before > 0 && *count > before).then(|| (signature.clone(), count - before))
})
.collect()
}
async fn scroll_hover_target_into_view(
client: &mut Client,
want: &str,
) -> std::result::Result<u64, String> {
let explicit_role = want.split_once(':').map(|(role, _)| role);
let any_role = ["*"];
let roles = explicit_role.as_slice();
let roles = if roles.is_empty() { &any_role } else { roles };
locate_control(client, want, roles)
.await
.map(|(node_id, _)| node_id)
.map_err(|error| format!("could not hover {want:?}: {error}"))
}
fn capture_node_id(nodes: &[SemanticNode], selector: &str) -> std::result::Result<u64, String> {
let captures_decorative_art = selector
.split_once(':')
.is_some_and(|(role, _)| role.eq_ignore_ascii_case("presentation"));
nodes
.iter()
.filter(|node| {
selector_matches_node(node, selector)
&& (node.visible || captures_decorative_art && node.role == "presentation")
&& node.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0)
})
.max_by(|a, b| {
let area = |node: &&SemanticNode| {
node.bounds
.map(|bounds| bounds[2] * bounds[3])
.unwrap_or_default()
};
area(a).total_cmp(&area(b))
})
.map(|node| node.id)
.ok_or_else(|| format!("no painted capture region matching {selector:?}"))
}
async fn capture_region(
client: &mut Client,
selector: &str,
) -> std::result::Result<CapturedImage, String> {
let (tree, _) = inspect(client).await.map_err(|error| error.to_string())?;
let node_id = capture_node_id(&tree.nodes, selector)?;
capture_node_region(client, node_id, selector).await
}
async fn reveal_capture_region(
client: &mut Client,
selector: &str,
) -> std::result::Result<u64, String> {
let (tree, _) = inspect(client).await.map_err(|error| error.to_string())?;
let node_id = capture_node_id(&tree.nodes, selector)?;
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id,
}))
.await
.map_err(|error| error.to_string())?;
Ok(node_id)
}
async fn visible_ink(client: &mut Client, selector: &str) -> std::result::Result<(), String> {
let image = capture_region(client, selector).await?;
let ink = measure_ink(&image).map_err(|error| error.to_string())?;
if ink.visible == 0 {
Err(format!(
"{selector:?} occupies {}x{} but draws no pixels distinguishable from its background",
image.width, image.height
))
} else {
Ok(())
}
}
async fn interior_ink(client: &mut Client, selector: &str) -> std::result::Result<(), String> {
let image = capture_region(client, selector).await?;
let ink = measure_interior_ink(&image).map_err(|error| error.to_string())?;
if ink.visible == 0 {
Err(format!(
"{selector:?} occupies {}x{} but its interior contains no pixels distinguishable from its background",
image.width, image.height
))
} else {
Ok(())
}
}
async fn transparent_window_tint(client: &mut Client) -> std::result::Result<(), String> {
let answer = client
.diagnostics(&DiagnosticsRequest::WindowComposition)
.await
.map_err(|error| format!("could not inspect native window composition: {error}"))?;
let composition = match answer.response {
DebugResponse::WindowComposition(composition) => composition,
other => {
return Err(format!(
"asked for native window composition, got {other:?}"
));
}
};
if cli::trace() {
eprintln!(
" native window: transparent={} glass={} backend={} tint={:?} radius={:?}",
composition.surface_transparent,
composition.glass_enabled,
composition.glass_backend.as_deref().unwrap_or("unknown"),
composition.tint_rgba,
composition.radius,
);
}
require_transparent_window_tint(&composition)
}
fn require_transparent_window_tint(
composition: &WindowComposition,
) -> std::result::Result<(), String> {
if !composition.supported {
return Err("this host cannot inspect native window composition".into());
}
if !composition.surface_transparent {
return Err("the native window surface is opaque".into());
}
if !composition.glass_enabled {
return Err("the native glass layer was not installed".into());
}
let backend = composition.glass_backend.as_deref().unwrap_or("unknown");
let Some([red, green, blue, alpha]) = composition.tint_rgba else {
if backend == "vibrancy" {
return Ok(());
}
return Err(format!(
"native glass backend {backend:?} did not expose an applied tint"
));
};
if alpha != 0 {
return Err(format!(
"native glass backend {backend:?} retains tint rgba({red}, {green}, {blue}, {alpha}) at zero opacity"
));
}
Ok(())
}
async fn capture_node_region(
client: &mut Client,
node_id: u64,
selector: &str,
) -> std::result::Result<CapturedImage, String> {
let previous_timeout = client.request_timeout();
client.set_request_timeout(Duration::from_secs(15));
let started = std::time::Instant::now();
let answer = client
.diagnostics(&DiagnosticsRequest::Capture(CaptureRequest {
node_id: Some(node_id),
scale: 1.0,
}))
.await;
client.set_request_timeout(previous_timeout);
if cli::trace_capture() {
eprintln!("capture {selector:?}: request took {:?}", started.elapsed());
}
let answer = answer.map_err(|error| format!("could not capture {selector:?}: {error}"))?;
match answer.response {
DebugResponse::Captured(image) => Ok(image),
other => Err(format!("asked for a rendered frame, got {other:?}")),
}
}
async fn capture_stable_region(
client: &mut Client,
selector: &str,
settle_timeout: Duration,
) -> std::result::Result<CapturedImage, String> {
const QUIET_SAMPLES: usize = 4;
let (tree, _) = inspect(client).await.map_err(|error| error.to_string())?;
let node_id = capture_node_id(&tree.nodes, selector)?;
let mut previous = capture_node_region(client, node_id, selector).await?;
let mut deadline = std::time::Instant::now() + settle_timeout;
let mut matching = 1;
loop {
nagoya::sleep(Duration::from_millis(16)).await;
let capture_started = std::time::Instant::now();
let current = capture_node_region(client, node_id, selector).await?;
deadline += std::time::Instant::now().duration_since(capture_started);
let held =
captured_pixels_hold(&previous, ¤t, CAPTURE_CHANNEL_TOLERANCE).unwrap_or(false);
if cli::trace_capture() {
let delta = captured_pixel_delta(&previous, ¤t)
.map(|(pixels, max)| format!("{pixels} pixel(s), max {max}"))
.unwrap_or_else(|error| error);
eprintln!(
"capture {selector:?}: {}x{} -> {}x{}, held={held}, streak={matching}, {delta}",
previous.width, previous.height, current.width, current.height
);
}
if held {
matching += 1;
if matching >= QUIET_SAMPLES {
return Ok(current);
}
} else {
matching = 1;
}
if std::time::Instant::now() >= deadline {
let detail = if previous.width != current.width || previous.height != current.height {
format!(
"; last frame changed size from {}x{} to {}x{}",
previous.width, previous.height, current.width, current.height
)
} else {
captured_pixel_delta(&previous, ¤t)
.map(|(pixels, max)| {
format!("; last frame changed {pixels} pixel(s), max channel delta {max}")
})
.unwrap_or_else(|error| format!("; {error}"))
};
return Err(format!(
"rendered region {selector:?} did not settle within {}ms{detail}",
settle_timeout.as_millis()
));
}
previous = current;
}
}
async fn wait_for_pixels_change(
client: &mut Client,
node_id: u64,
selector: &str,
before: &CapturedImage,
deadline: Instant,
event_driven: bool,
) -> std::result::Result<(), String> {
loop {
let after = capture_node_region(client, node_id, selector).await?;
let now = Instant::now();
match assess_pixel_change(before, &after, now >= deadline)? {
Some(()) => return Ok(()),
None => {
let remaining = deadline.saturating_duration_since(now);
if event_driven {
let _ = client
.wait_for_paint(remaining)
.await
.map_err(|error| error.to_string())?;
} else {
nagoya::sleep(remaining.min(Duration::from_millis(25))).await;
}
}
}
}
}
fn assess_pixel_change(
before: &CapturedImage,
after: &CapturedImage,
timed_out: bool,
) -> std::result::Result<Option<()>, String> {
match pixels_change(before, after) {
Ok(()) => Ok(Some(())),
Err(error) if timed_out => Err(error),
Err(_) => Ok(None),
}
}
fn start_host(
host: &std::path::Path,
page: &std::path::Path,
startup_timeout: Duration,
storage: Option<&std::path::Path>,
) -> std::result::Result<(HostProcess, std::path::PathBuf), String> {
start_host_with_policy(host, page, startup_timeout, false, false, storage)
}
fn plan_control_directory() -> std::result::Result<std::path::PathBuf, String> {
static NEXT_CONTROL_DIRECTORY: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
let current = std::env::current_dir()
.map_err(|error| format!("could not locate QA artifact directory: {error}"))?
.canonicalize()
.map_err(|error| format!("could not resolve QA artifact directory: {error}"))?;
let target = ensure_plan_directory(¤t, "target")?;
let root = ensure_plan_directory(&target, "q")?;
for _ in 0..32 {
let sequence = NEXT_CONTROL_DIRECTORY.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let path = root.join(format!("{:x}-{sequence:x}", std::process::id()));
let mut directory = std::fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
directory.mode(0o700);
}
match directory.create(&path) {
Ok(()) => {
if !control_directory_is_real(&path) {
let _ = std::fs::remove_dir(&path);
return Err("QA host control directory cannot be a symlink".into());
}
return path.canonicalize().map_err(|error| {
format!("could not resolve QA host control directory: {error}")
});
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(error) => {
return Err(format!(
"could not create QA host control directory: {error}"
));
}
}
}
Err("could not allocate a unique QA host control directory".into())
}
fn ensure_plan_directory(
parent: &std::path::Path,
name: &str,
) -> std::result::Result<std::path::PathBuf, String> {
let path = parent.join(name);
match std::fs::create_dir(&path) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(error) => {
return Err(format!(
"could not create QA control artifact directory: {error}"
));
}
}
let parent = parent
.canonicalize()
.map_err(|error| format!("could not resolve QA control artifact directory: {error}"))?;
let path = path
.canonicalize()
.map_err(|error| format!("could not resolve QA control artifact directory: {error}"))?;
if !path.starts_with(parent) {
return Err("QA control artifact directory must stay under its parent".into());
}
Ok(path)
}
fn control_directory_is_real(path: &std::path::Path) -> bool {
std::fs::symlink_metadata(path)
.is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink())
}
fn write_storage_seed(
storage: &std::collections::BTreeMap<String, String>,
) -> std::result::Result<std::path::PathBuf, String> {
let path = std::env::temp_dir().join(format!("qa-inspect-storage-{}.json", std::process::id()));
let encoded = serde_json::to_vec(storage)
.map_err(|error| format!("could not encode QA storage seed: {error}"))?;
std::fs::write(&path, encoded).map_err(|error| {
format!(
"could not write QA storage seed {}: {error}",
path.display()
)
})?;
Ok(path)
}
async fn stage_check_storage(
client: &mut Client,
session: &mut HostedSession,
storage: &std::collections::BTreeMap<String, String>,
) -> Result<()> {
if storage.is_empty() {
let Some(path) = session.storage.take() else {
return Ok(());
};
let _ = std::fs::remove_file(path);
reconnect_hosted(client, session).await?;
return Ok(());
}
let path = write_storage_seed(storage).map_err(eyre::Report::msg)?;
session.storage = Some(path);
reconnect_hosted(client, session).await?;
Ok(())
}
fn contained_control_file(
path: &std::path::Path,
root: &std::path::Path,
) -> std::result::Result<std::path::PathBuf, String> {
let meta = std::fs::symlink_metadata(path)
.map_err(|error| format!("could not inspect host descriptor: {error}"))?;
if meta.file_type().is_symlink() || !meta.is_file() {
return Err("host descriptor must be a regular file under the QA control directory".into());
}
let path = path
.canonicalize()
.map_err(|error| format!("could not resolve host descriptor: {error}"))?;
let root = root
.canonicalize()
.map_err(|error| format!("could not resolve QA host control directory: {error}"))?;
if !path.starts_with(&root) {
return Err("host descriptor must stay under the QA control directory".into());
}
Ok(path)
}
fn start_host_with_policy(
host: &std::path::Path,
page: &std::path::Path,
startup_timeout: Duration,
quiet_output: bool,
clean_environment: bool,
storage: Option<&std::path::Path>,
) -> std::result::Result<(HostProcess, std::path::PathBuf), String> {
use std::io::BufRead;
let mut command = std::process::Command::new(host);
let control_dir = if clean_environment {
Some(plan_control_directory()?)
} else {
None
};
if clean_environment {
command.env_clear();
}
command.env("QA_INSPECT_PAGE", page);
if let Some(path) = storage {
command.env("QA_INSPECT_STORAGE", path);
}
if let Some(path) = &control_dir {
command.env("CHUZZ_CONTROL_DIR", path);
command
.env("TMPDIR", path)
.env("TMP", path)
.env("TEMP", path);
}
command.stdout(std::process::Stdio::piped());
command.stderr(if quiet_output {
std::process::Stdio::null()
} else {
std::process::Stdio::inherit()
});
let spawned = match command.spawn() {
Ok(child) => child,
Err(error) => {
if let Some(path) = &control_dir {
let _ = std::fs::remove_dir_all(path);
}
return Err(format!("launching {}: {error}", host.display()));
}
};
let mut child = HostProcess {
child: spawned,
control_dir,
};
let stdout = child.child.stdout.take().expect("stdout was piped");
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let mut reader = std::io::BufReader::new(stdout);
let mut line = String::new();
let mut announced = false;
while reader.read_line(&mut line).unwrap_or_default() > 0 {
let text = line.trim();
if !announced && text.ends_with(".json") && std::path::Path::new(text).is_file() {
announced = true;
let _ = tx.send(text.to_owned());
} else if !quiet_output && !text.is_empty() {
eprintln!("host: {text}");
}
line.clear();
}
});
match rx.recv_timeout(startup_timeout) {
Ok(line) if !line.trim().is_empty() => {
let announced = std::path::PathBuf::from(line.trim());
let announced = if let Some(root) = &child.control_dir {
contained_control_file(&announced, root)?
} else {
announced
};
Ok((child, announced))
}
_ => Err("the host never announced a descriptor".to_owned()),
}
}
async fn sweep_components(
ids: &[String],
host: &std::path::Path,
dists: &std::path::Path,
checks_dir: Option<&std::path::Path>,
startup_timeout: Duration,
mode: cli::CheckMode,
) -> Result<usize> {
let ids: Vec<String> = if ids.is_empty() {
let mut found: Vec<String> = std::fs::read_dir(dists)
.with_context(|| format!("reading {}", dists.display()))?
.filter_map(|entry| entry.ok())
.filter_map(|entry| {
let path = entry.path();
if path.is_dir() {
entry.file_name().into_string().ok()
} else if path.extension().is_some_and(|ext| ext == "html") {
path.file_stem()
.and_then(|stem| stem.to_str())
.map(str::to_owned)
} else {
None
}
})
.collect();
found.sort();
found.dedup();
found
} else {
ids.to_vec()
};
if ids.is_empty() {
bail!("no components to sweep under {}", dists.display());
}
let mut passed = 0_usize;
let mut failed = 0_usize;
let mut verdicts: Vec<(String, bool, String)> = Vec::new();
for id in &ids {
let as_dir = dists.join(id);
let as_page = dists.join(format!("{id}.html"));
let dist = if as_dir.is_dir() {
as_dir
} else if as_page.is_file() {
as_page
} else {
println!(
"FAIL {id}: no built page at {} or {}",
as_dir.display(),
as_page.display()
);
verdicts.push((id.clone(), false, "no built page".to_owned()));
failed += 1;
continue;
};
let check_ids: Vec<String> = match qa::checks(checks_dir) {
Ok(all) => all
.iter()
.filter(|check| check.group == *id)
.map(|check| check.id.clone())
.collect(),
Err(error) => {
println!("FAIL {id}: {error}");
verdicts.push((id.clone(), false, error));
failed += 1;
continue;
}
};
if check_ids.is_empty() {
println!("FAIL {id}: no checks in group {id:?}");
verdicts.push((id.clone(), false, "no checks".to_owned()));
failed += 1;
continue;
}
let runs: Vec<String> = if mode == cli::CheckMode::Sweep {
vec![id.clone()]
} else {
check_ids.clone()
};
let mut component_failed = 0_usize;
let mut launch_error: Option<String> = None;
for check_id in &runs {
let mut hosted = match HostedSession::start(host, &dist, startup_timeout, false, false)
{
Ok(hosted) => hosted,
Err(error) => {
launch_error = Some(error);
break;
}
};
let descriptor_path = hosted.descriptor_path.clone();
let outcome = run_component(
&descriptor_path,
Some(check_id),
checks_dir,
Some(&mut hosted),
)
.await;
drop(hosted);
match outcome {
Ok(count) => component_failed += count,
Err(error) => {
launch_error = Some(error.to_string());
break;
}
}
}
let outcome: Result<usize> = match launch_error {
Some(error) => Err(eyre!(error)),
None => Ok(component_failed),
};
match outcome {
Ok(0) => {
println!("PASS {id}");
verdicts.push((id.clone(), true, String::new()));
passed += 1;
}
Ok(count) => {
println!("FAIL {id}: {count} check(s) failed");
verdicts.push((id.clone(), false, format!("{count} check(s) failed")));
failed += 1;
}
Err(error) => {
println!("FAIL {id}: {error}");
verdicts.push((id.clone(), false, error.to_string()));
failed += 1;
}
}
}
println!();
println!("passed: {passed} failed: {failed} of {}", ids.len());
Ok(failed)
}
async fn run_component(
descriptor_path: &std::path::Path,
selector: Option<&str>,
checks_dir: Option<&std::path::Path>,
hosted: Option<&mut HostedSession>,
) -> Result<usize> {
let descriptor = inspector::discover(descriptor_path.to_str())?;
let mut client = Client::connect(&descriptor.socket_path()).await?;
client.initialize().await?;
let baseline = inspect(&mut client).await?.0;
let all_checks = qa::checks(checks_dir).map_err(eyre::Report::msg)?;
let uncovered: Vec<&SemanticNode> = baseline
.nodes
.iter()
.filter(|node| reach::operable(node))
.filter(|node| !reach::requires_manual_release_check(&node.name))
.filter(|node| outcome_check_ids(node, &all_checks).is_empty())
.collect();
for node in &uncovered {
eprintln!(
"uncovered component control: id={} role={} name={:?} visible={} enabled={}",
node.id, node.role, node.name, node.visible, node.enabled
);
}
let failed = run_qa(&mut client, selector, checks_dir, hosted).await?;
Ok(failed + uncovered.len())
}
fn ordered_checks<'a>(all: &'a [qa::Check], group: Option<&str>) -> Vec<&'a qa::Check> {
let group = group.filter(|group| *group != "*");
let mut selected: Vec<&qa::Check> = all
.iter()
.filter(|check| group.is_none_or(|want| check.group == want || check.id == want))
.collect();
selected.sort_by_key(|check| check.destructive);
selected
}
fn resolve_assertion_subjects(
checks: &mut [qa::Check],
values: &HashMap<String, String>,
) -> std::result::Result<HashSet<String>, String> {
let mut used = HashSet::new();
for check in checks {
if let Some(binding) = check.subject_env.as_deref() {
let value = values.get(binding).ok_or_else(|| {
format!(
"check {:?} references an unknown assertion binding",
check.id
)
})?;
check.subject.clone_from(value);
used.insert(binding.to_owned());
}
if let Some(binding) = check.setup_text_env.as_deref() {
let value = values.get(binding).ok_or_else(|| {
format!(
"check {:?} references an unknown assertion binding",
check.id
)
})?;
check.setup_text = Some(value.clone());
used.insert(binding.to_owned());
}
}
Ok(used)
}
fn resolve_secret_subjects(
checks: &mut [qa::Check],
secrets: Option<&plan::SessionSecrets>,
) -> std::result::Result<(), String> {
for check in checks {
if let Some(input) = check.subject_secret {
let secrets = secrets.ok_or_else(|| {
format!(
"check {:?} requires a signed-in session credential",
check.id
)
})?;
check.subject = secrets.value(input).to_owned();
}
}
Ok(())
}
fn redact_session_text(text: &str, secrets: Option<&plan::SessionSecrets>) -> String {
secrets.map_or_else(|| text.to_owned(), |secrets| secrets.redact(text))
}
async fn run_qa(
client: &mut Client,
group: Option<&str>,
checks_dir: Option<&std::path::Path>,
hosted: Option<&mut HostedSession>,
) -> Result<usize> {
run_qa_with_secrets(client, group, checks_dir, None, hosted).await
}
async fn run_qa_with_secrets(
client: &mut Client,
group: Option<&str>,
checks_dir: Option<&std::path::Path>,
secrets: Option<&plan::SessionSecrets>,
hosted: Option<&mut HostedSession>,
) -> Result<usize> {
let checks_paths = checks_dir.map(|path| vec![path.to_path_buf()]);
run_qa_with_plan_values(
client,
group,
checks_paths.as_deref(),
secrets,
None,
hosted,
)
.await
}
async fn run_qa_with_plan_values(
client: &mut Client,
group: Option<&str>,
checks_paths: Option<&[std::path::PathBuf]>,
secrets: Option<&plan::SessionSecrets>,
assertion_values: Option<&HashMap<String, String>>,
hosted: Option<&mut HostedSession>,
) -> Result<usize> {
match run_qa_inner(
client,
group,
checks_paths,
secrets,
assertion_values,
hosted,
)
.await
{
Err(error) => match secrets {
Some(secrets) => Err(eyre!(secrets.redact(&error.to_string()))),
None => Err(error),
},
result => result,
}
}
async fn run_qa_inner(
client: &mut Client,
group: Option<&str>,
checks_paths: Option<&[std::path::PathBuf]>,
secrets: Option<&plan::SessionSecrets>,
assertion_values: Option<&HashMap<String, String>>,
mut hosted: Option<&mut HostedSession>,
) -> Result<usize> {
let all = if let Some(paths) = checks_paths {
qa::checks_from_paths(paths).map_err(eyre::Report::msg)?
} else {
qa::checks(None).map_err(eyre::Report::msg)?
};
let selected = ordered_checks(&all, group);
if selected.is_empty() {
let mut names: Vec<String> = all
.iter()
.map(|check| format!("{} ({})", check.id, check.group))
.collect();
names.sort();
bail!(
"no check or group matching {group:?}. known:\n {}",
names.join("\n ")
);
}
let mut selected: Vec<qa::Check> = selected.into_iter().cloned().collect();
if let Some(values) = assertion_values {
resolve_assertion_subjects(&mut selected, values).map_err(eyre::Report::msg)?;
}
resolve_secret_subjects(&mut selected, secrets).map_err(eyre::Report::msg)?;
if secrets.is_none() && selected.iter().any(|check| check.setup_secret.is_some()) {
bail!("checks with runtime credential setup must run through `ps-qa run-plan`");
}
let mut results: Vec<CheckResult<'_>> = Vec::new();
for index in 0..selected.len() {
if let Some(session) = hosted.as_deref_mut()
&& session.seeds_storage
{
stage_check_storage(client, session, &selected[index].storage).await?;
}
let check = &selected[index];
let full_check_started = Instant::now();
let produced: std::result::Result<OneCheck, eyre::Report> = async {
let mut retries = 0;
client.set_request_timeout(Duration::from_secs(15));
let mut open_error = None;
let mut pixel_outcome: Option<std::result::Result<(), String>> = None;
let open_budget = declared_open_timeout(check);
if let Some(url) = check.navigate.as_deref() {
let navigated = client
.agent(&AgentControlRequest::Navigate {
url: url.to_owned(),
})
.await;
if let Err(error) = navigated {
if host_gave_up_report(&error) {
return Err(error);
}
open_error = Some(format!("could not navigate to {url:?}: {error}"));
} else {
let target = check
.arrival_subject
.as_deref()
.or(check.open.as_deref())
.or_else(|| check.hover.as_ref().map(qa::Hover::target))
.or(check.prepare.as_deref())
.or(check.click.as_deref())
.or(check.type_into.as_deref())
.or(check.key_on.as_deref())
.unwrap_or(&check.subject);
if !wait_for_arrival(client, None, target, open_budget).await? {
open_error = Some(format!(
"could not navigate to {url:?}: the arrival target {target:?} did not paint within \
{}ms. Raise open_timeout_ms if this route fetches.",
open_budget.as_millis()
));
}
}
}
if open_error.is_none()
&& let Some(want) = check.open.as_deref()
{
let want_here: &str = check
.hover
.as_ref()
.map(qa::Hover::target)
.or(check.prepare.as_deref())
.or(check.click.as_deref())
.or(check.type_into.as_deref())
.or(check.key_on.as_deref())
.unwrap_or(&check.subject);
let destination = surface_for_opener(want);
let named_document = is_named_document_opener(want);
let (here, _) = inspect(client).await?;
let active_document_matches = named_document_is_active(&here.nodes, want);
let arrived = arrived_without_navigation(
&here.nodes,
destination,
want_here,
named_document,
active_document_matches,
);
if cli::trace() {
let surface = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(&here.nodes, surface))
.map(|surface| surface.name.as_str())
.unwrap_or("none");
let trace = format!(
" arrival want={want:?} destination={} target={want_here:?} \
named_document={named_document} arrived={arrived} on_surface={surface}",
destination.map_or("dynamic", |surface| surface.name.as_str()),
);
println!("{}", redact_session_text(&trace, secrets));
}
if !arrived {
let first_click = click_opener_quiet(client, want, named_document).await;
if let Err(error) = &first_click
&& host_gave_up_report(error)
{
return Err(eyre!(error.to_string()));
}
let mut there = wait_for_navigation_arrival(
client,
destination,
want_here,
named_document,
want,
open_budget,
)
.await?;
if !there && first_click.is_err() {
let _ = click_named_quiet(client, want).await;
there = wait_for_navigation_arrival(
client,
destination,
want_here,
named_document,
want,
open_budget,
)
.await?;
}
if !there && open_named(client, want).await.is_ok() {
there = wait_for_navigation_arrival(
client,
destination,
want_here,
named_document,
want,
open_budget,
)
.await?;
}
if !there {
if let Some(home) = reach::profile().home_opener.as_deref() {
let _ = click_named_quiet(client, home).await;
}
if let Err(error) = open_named(client, want).await {
open_error = Some(format!("could not open {want:?}: {error}"));
} else if !wait_for_navigation_arrival(
client,
destination,
want_here,
named_document,
want,
open_budget,
)
.await?
{
open_error = Some(format!(
"could not open {want:?}: the control was activated but the \
destination did not paint within {}ms. Raise open_timeout_ms \
if this route fetches.",
open_budget.as_millis()
));
}
}
}
}
let setup_value = check.setup_type_into.as_deref().and_then(|field| {
let value = check.setup_text.as_deref().or_else(|| {
check
.setup_secret
.and_then(|kind| secrets.map(|secrets| secrets.value(kind)))
});
value.map(|value| (field, value, check.setup_secret.is_some()))
});
if open_error.is_none()
&& let Some((field, value, is_secret)) = setup_value
{
match type_text(client, field, value).await {
Err(error) => {
let category = error
.downcast_ref::<TextEntryFailure>()
.map(TextEntryFailure::category)
.unwrap_or("unclassified_text_entry_failure");
open_error = Some(if is_secret {
format!("could not establish credential setup in {field:?} ({category})")
} else {
format!("could not establish setup value in {field:?}: {error}")
});
}
Ok(node_id) => {
let settled =
wait_for_semantic_condition(client, check_timeout(900), |nodes| {
setup_value_landed(nodes, node_id, value)
})
.await?;
if !setup_value_landed(&settled.nodes, node_id, value) {
open_error = Some(if is_secret {
format!(
"credential setup had not reached {field:?} within {}ms; \
a baseline taken before it lands lets setup satisfy the \
check's own outcome",
check_timeout(900).as_millis()
)
} else {
format!(
"the setup value {value:?} had not reached {field:?} within {}ms; \
a baseline taken before it lands lets the setup satisfy the \
check's own outcome",
check_timeout(900).as_millis()
)
});
}
}
}
}
let setup_target = check
.hover
.as_ref()
.map(qa::Hover::target)
.or(check.after_prepare_hover.as_ref().map(qa::Hover::target))
.or(check.reveal_before_capture.as_deref())
.or(check.prepare.as_deref())
.or(check.click.as_deref())
.or(check.type_into.as_deref())
.or(check.key_on.as_deref())
.unwrap_or(&check.subject);
let (current, _) = inspect(client).await?;
if !painted_named(¤t.nodes, setup_target)
&& let Some(surface) = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(¤t.nodes, surface))
{
let opened = expand_everything(client, surface).await?;
if cli::trace() && opened > 0 {
let trace =
format!(" opened {opened} collapsed section(s) for {setup_target:?}");
println!("{}", redact_session_text(&trace, secrets));
}
}
let (expanded, _) = inspect(client).await?;
if open_error.is_none()
&& !painted_named(&expanded.nodes, setup_target)
&& let Some(surface) = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(&expanded.nodes, surface))
{
match reveal_deferred_content(client, surface, setup_target).await {
Ok(reveals) => {
if cli::trace() && reveals > 0 {
let trace = format!(
" revealed deferred content for {setup_target:?} in {reveals} step(s)"
);
println!("{}", redact_session_text(&trace, secrets));
}
}
Err(error) => {
open_error = Some(format!(
"could not materialize pagination for {setup_target:?}: {error}"
));
}
}
}
if open_error.is_none()
&& let Some(reveal) = check.reveal_before_capture.as_deref()
{
let arrived = wait_for_arrival(client, None, reveal, check_timeout(900)).await?;
if !arrived {
open_error = Some(format!(
"could not reveal {reveal:?}: it did not paint within {}ms",
check_timeout(900).as_millis()
));
} else if let Err(error) = scroll_hover_target_into_view(client, reveal).await {
open_error = Some(error);
}
}
let mut nodes_after_first_hover = None;
if open_error.is_none()
&& let Some(hover) = check.hover.as_ref()
{
let already_hovered = if let Some(unless) = check.hover_unless.as_deref() {
let (snapshot, _) = inspect(client).await?;
painted_named(&snapshot.nodes, unless)
} else {
false
};
if already_hovered {
} else {
let hovered = if hover.times() > 1 {
let first = qa::Hover::Once(hover.target().to_owned());
match repeat_hover(client, &first, false).await {
Err(error) => Err(error),
Ok(()) => {
let snapshot = inspect(client).await?.0;
nodes_after_first_hover = Some(hover_signature_counts(&snapshot.nodes));
match park_pointer(client).await {
Err(error) => Err(error),
Ok(()) => {
let remaining = qa::Hover::Times(
hover.target().to_owned(),
hover.times() - 1,
);
repeat_hover(client, &remaining, false).await
}
}
}
}
} else {
repeat_hover(client, hover, false).await
};
if let Err(error) = hovered {
open_error = Some(error);
} else if let Some(next) = check.prepare.as_deref().or(check.click.as_deref())
&& !wait_for_arrival(client, None, next, check_timeout(900)).await?
{
if let Err(error) = repeat_hover(client, hover, false).await {
open_error = Some(error);
} else if !wait_for_arrival(client, None, next, check_timeout(900)).await? {
open_error = Some(format!(
"hovering {:?} did not reveal {next:?}",
hover.target()
));
}
}
}
}
if open_error.is_none()
&& let Some(want) = check.prepare.as_deref()
{
let already_prepared = if let Some(unless) = check.prepare_unless.as_deref() {
let (snapshot, _) = inspect(client).await?;
snapshot.nodes.iter().any(|node| {
node.visible
&& selector_matches_node(node, unless)
&& painted_bounds(node).is_some()
})
} else {
false
};
let mut prepared = if already_prepared {
Ok(())
} else if let Some(key) = check.prepare_key.as_deref() {
press_key(client, key, 1, want, true).await.map(|_| ())
} else if check.prepare_press {
press_named(client, want).await
} else {
click_named_quiet(client, want).await.map(|_| ())
};
if let Err(error) = &prepared
&& host_gave_up_report(error)
{
return Err(eyre!(error.to_string()));
}
if prepared.is_err() {
retries += 1;
let declared_surface = check.open.as_deref().and_then(surface_for_opener);
let live_surface = if declared_surface.is_some() {
declared_surface
} else {
let (snapshot, _) = inspect(client).await?;
reach::surfaces()
.iter()
.find(|surface| reach::on_surface(&snapshot.nodes, surface))
};
if let Some(surface) = live_surface {
let _ = expand_everything(client, surface).await?;
let _ = reveal_deferred_content(client, surface, want).await?;
prepared = if let Some(key) = check.prepare_key.as_deref() {
press_key(client, key, 1, want, true).await.map(|_| ())
} else if check.prepare_press {
press_named(client, want).await
} else {
click_named_quiet(client, want).await.map(|_| ())
};
}
}
if let Err(error) = prepared {
let nearby = match inspect(client).await {
Ok((snapshot, _)) => {
let mut names: Vec<String> = snapshot
.nodes
.iter()
.filter(|node| {
!node.name.is_empty()
&& node.visible
&& node.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0)
})
.map(|node| format!("{}:{}", node.role, node.name))
.collect();
names.sort();
names.dedup();
names.truncate(8);
if names.is_empty() {
String::new()
} else {
format!("; on screen: {}", names.join(", "))
}
}
Err(_) => String::new(),
};
open_error = Some(format!("could not prepare {want:?}: {error}{nearby}"));
}
if open_error.is_none()
&& let Some(until) = check.prepare_until.as_deref()
&& !wait_for_visible_arrival(client, until, check_timeout(900)).await?
{
open_error = Some(format!(
"preparing {want:?} did not render the declared precondition {until:?}"
));
}
if open_error.is_none()
&& let Some(next) = check
.click
.as_deref()
.or(check.type_into.as_deref())
.or(check.key_on.as_deref())
{
let _ = wait_for_arrival(client, None, next, check_timeout(900)).await?;
}
if open_error.is_none() && check.prepare_wait_ms > 0 {
nagoya::sleep(Duration::from_millis(check.prepare_wait_ms)).await;
}
}
if open_error.is_none()
&& let Some(hover) = check.after_prepare_hover.as_ref()
{
if check.expect == qa::Expect::PixelsHold {
let measured = async {
park_pointer(client).await?;
let establish = qa::Hover::Once(hover.target().to_owned());
repeat_hover(client, &establish, true).await?;
let before = capture_stable_region(
client,
&check.subject,
declared_outcome_timeout(check),
)
.await
.map_err(|error| format!("before hover: {error}"))?;
repeat_hover(client, hover, true).await?;
let after = capture_stable_region(
client,
&check.subject,
declared_outcome_timeout(check),
)
.await
.map_err(|error| format!("after hover: {error}"))?;
pixels_hold(&before, &after)
}
.await;
pixel_outcome = Some(measured);
} else if check.expect == qa::Expect::PixelsHoldAfterHover {
let measured = async {
match wait_for_arrival(client, None, hover.target(), check_timeout(900)).await {
Ok(true) => {}
Ok(false) => {
return Err(format!(
"could not prepare hover target {:?}: it did not finish painting",
hover.target()
));
}
Err(error) => return Err(error.to_string()),
}
scroll_hover_target_into_view(client, hover.target()).await?;
park_pointer(client).await?;
let before = capture_stable_region(
client,
&check.subject,
declared_outcome_timeout(check),
)
.await
.map_err(|error| format!("before hover: {error}"))?;
repeat_hover(client, hover, true).await?;
let after = capture_stable_region(
client,
&check.subject,
declared_outcome_timeout(check),
)
.await
.map_err(|error| format!("after hover: {error}"))?;
let comparison = rgb_pixels_hold(&before, &after);
if comparison.is_err()
&& let Err(error) = save_pixel_artifacts(&check.id, &before, &after)
{
eprintln!("could not save pixel artifacts for {}: {error}", check.id);
}
comparison
}
.await;
pixel_outcome = Some(measured);
} else if check.expect == qa::Expect::PixelsChange {
let measured = async {
reveal_capture_region(client, &check.subject).await?;
park_pointer(client).await?;
let node_id = scroll_hover_target_into_view(client, hover.target()).await?;
let before = capture_region(client, &check.subject).await?;
let subject_id = before.node_id.ok_or_else(|| {
format!(
"capture of {:?} did not identify the subject",
check.subject
)
})?;
let event_driven = client
.arm_paint_events()
.await
.map_err(|error| error.to_string())?;
let require_events = cli::require_paint_events();
if require_events && !event_driven {
return Err("the inspector does not provide paint events".to_owned());
}
client
.agent(&AgentControlRequest::Act(AgentAction::Hover { node_id }))
.await
.map_err(|error| error.to_string())?;
let paint_committed = if event_driven {
client
.wait_for_paint(declared_outcome_timeout(check))
.await
.map_err(|error| error.to_string())?
} else {
false
};
if require_events && !paint_committed {
return Err("no paint event arrived after hover".to_owned());
}
let comparison = wait_for_pixels_change(
client,
subject_id,
&check.subject,
&before,
Instant::now() + declared_outcome_timeout(check),
event_driven,
)
.await;
if comparison.is_err()
&& let Ok(after) =
capture_node_region(client, subject_id, &check.subject).await
&& let Err(error) = save_pixel_artifacts(&check.id, &before, &after)
{
eprintln!("could not save pixel artifacts for {}: {error}", check.id);
}
comparison
}
.await;
pixel_outcome = Some(measured);
} else if let Err(error) = repeat_hover(client, hover, true).await {
open_error = Some(error);
}
}
if let Some(after_first) = nodes_after_first_hover
&& open_error.is_none()
{
let snapshot = inspect(client).await?.0;
let after_repeats = hover_signature_counts(&snapshot.nodes);
let retained = accumulated_hover_signatures(&after_first, &after_repeats);
if !retained.is_empty() {
let details = retained
.iter()
.take(3)
.map(|((role, name, slot, dom_id), extra)| {
format!("{role}:{name:?} slot={slot:?} id={dom_id:?} +{extra}")
})
.collect::<Vec<_>>()
.join(", ");
open_error = Some(format!(
"repeated hover retained extra copies of existing semantic nodes: {details}"
));
}
}
if open_error.is_none()
&& check.expect == qa::Expect::PixelsChange
&& check.after_prepare_hover.is_none()
&& let Err(error) = scroll_hover_target_into_view(client, &check.subject).await
{
open_error = Some(format!("could not reveal pixel subject: {error}"));
}
let mut prepared_click = None;
if open_error.is_none()
&& !check.press
&& let Some(want) = check.click.as_deref()
{
let deadline = std::time::Instant::now()
+ std::time::Duration::from_millis(check.open_timeout_ms.max(500));
loop {
match locate_control(client, want, &[]).await {
Ok((node_id, _)) => {
prepared_click = Some(node_id);
break;
}
Err(error) if std::time::Instant::now() >= deadline => {
open_error = Some(format!("could not locate {want:?}: {error}"));
break;
}
Err(_) => {
nagoya::sleep(std::time::Duration::from_millis(250)).await;
}
}
}
}
let (before, _) = inspect(client).await?;
let before_font_size = if open_error.is_none() && check.expect == qa::Expect::FontSizeGrows
{
match font_size(client, &check.subject).await {
Ok(size) => Some(size),
Err(error) => {
pixel_outcome = Some(Err(error));
None
}
}
} else {
None
};
let before_action_pixels = if open_error.is_none()
&& check.expect == qa::Expect::PixelsChange
&& check.after_prepare_hover.is_none()
{
Some(match capture_node_id(&before.nodes, &check.subject) {
Ok(node_id) => match capture_node_region(client, node_id, &check.subject).await {
Ok(image) => match client.arm_paint_events().await {
Ok(event_driven) => Ok((node_id, image, event_driven)),
Err(error) => Err(error.to_string()),
},
Err(error) => Err(error),
},
Err(error) => Err(error),
})
} else {
None
};
{
let dead = before
.nodes
.iter()
.filter(|node| !node.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0))
.count();
let live = before.nodes.len().saturating_sub(dead);
const MIN_SUSPICIOUS_DEAD_NODES: usize = 5_000;
const MAX_DEAD_TO_LIVE_RATIO: usize = 5;
if live > 0
&& dead > MIN_SUSPICIOUS_DEAD_NODES
&& dead > live.saturating_mul(MAX_DEAD_TO_LIVE_RATIO)
{
open_error = Some(format!(
"the document holds {dead} node(s) with no box against {live} with one, \
out of {}; something is retaining subtrees rather than reusing them",
before.nodes.len()
));
}
}
let mut check_started = (check.click.is_none()
&& check.text.is_none()
&& check.key.is_none()
&& check.keys.is_empty()
&& check.pointer_drag.is_none()
&& check.scroll_over.is_none())
.then(Instant::now);
let live_paint_expect = matches!(
check.expect,
qa::Expect::PixelsHold
| qa::Expect::PixelsHoldAfterHover
| qa::Expect::PixelsChange
| qa::Expect::VisibleInk
| qa::Expect::InteriorInk
| qa::Expect::OpaqueBackground
| qa::Expect::TransparentBackground
| qa::Expect::FullOpacity
| qa::Expect::TransparentWindowTint
| qa::Expect::Contrast
| qa::Expect::FontSizeGrows
| qa::Expect::GpuNoErrors
| qa::Expect::GpuHasResource
| qa::Expect::GpuDraws(_)
| qa::Expect::GpuTextureInk
);
let drives_action = check.click.is_some()
|| check.text.is_some()
|| check.key.is_some()
|| !check.keys.is_empty()
|| check.pointer_drag.is_some()
|| check.scroll_over.is_some();
let action_paint_armed = if drives_action && !live_paint_expect {
client.arm_paint_events().await.unwrap_or(false)
} else {
false
};
client.set_request_timeout(check_timeout(check.outcome_timeout_ms.max(900)));
let mut action_error = open_error;
let mut action_target = None;
let mut action_node_id = None;
if action_error.is_none()
&& let Some(text) = check.text.as_deref()
{
if check.key.is_none() && check.keys.is_empty() && check.click.is_none() {
check_started = Some(Instant::now());
}
if let Some(field) = check.type_into.as_deref() {
match type_text(client, field, text).await {
Ok(node_id) => action_node_id = Some(node_id),
Err(error) => {
action_error = Some(format!("could not type into {field:?}: {error}"));
}
}
} else {
action_error = Some("text requires type_into".to_owned());
}
}
if action_error.is_none()
&& let Some(want) = check.click.as_deref()
{
if check.key.is_none() && check.keys.is_empty() {
check_started = Some(Instant::now());
}
if check.expect == qa::Expect::TargetPaints && !check.press {
let (tree, _) = inspect(client).await?;
action_target = resolved_action_target(&tree.nodes, want);
}
let driven = drive_check_action(
client,
want,
check.press,
prepared_click,
check.hover.as_ref(),
)
.await;
action_error = driven.as_ref().err().cloned();
if check.text.is_none() {
action_node_id = driven.ok().flatten();
}
}
if action_error.is_none()
&& let Some(drag) = &check.pointer_drag
{
check_started = Some(Instant::now());
if let Err(error) = pointer_drag(
client,
&drag.from,
drag.dx,
drag.dy,
drag.steps,
drag.cancel,
drag.to.as_deref(),
)
.await
{
action_error = Some(format!("could not drag {:?}: {error}", drag.from));
}
}
if action_error.is_none()
&& let Some(key) = check.key.as_deref()
{
check_started = Some(Instant::now());
let target = check
.key_on
.as_deref()
.or(check.type_into.as_deref())
.unwrap_or("");
if let Err(error) = press_key(client, key, 1, target, check.key_on.is_some()).await {
action_error = Some(format!("could not send {key:?}: {error}"));
}
}
if action_error.is_none() && !check.keys.is_empty() {
check_started = Some(Instant::now());
if let Err(error) = key_steps(client, &check.keys).await {
action_error = Some(format!("could not send key steps: {error}"));
}
}
if action_error.is_none()
&& let Some(target) = check.scroll_over.as_deref()
{
check_started = Some(Instant::now());
match hover_over(client, target).await {
Ok(true) => {
if let Err(error) =
scroll_events(client, check.scroll_ticks, check.scroll_delta).await
{
action_error = Some(format!("could not scroll over {target:?}: {error}"));
}
}
Ok(false) => action_error = Some(format!("no painted node matching {target:?}")),
Err(error) => {
action_error = Some(format!("could not aim at {target:?}: {error}"));
}
}
}
if action_error.is_none() && gpu_expectation(check.expect) {
pixel_outcome =
Some(wait_for_gpu(client, check, check_started.unwrap_or_else(Instant::now)).await);
} else if action_error.is_none() && check.expect == qa::Expect::GpuTextureInk {
pixel_outcome = Some(
wait_for_gpu_texture_ink(
client,
check,
check_started.unwrap_or_else(Instant::now),
)
.await,
);
} else if action_error.is_none() && check.expect == qa::Expect::OpaqueBackground {
pixel_outcome = Some(opaque_background(client, &check.subject).await);
} else if action_error.is_none() && check.expect == qa::Expect::TransparentBackground {
pixel_outcome = Some(transparent_background(client, &check.subject).await);
} else if action_error.is_none() && check.expect == qa::Expect::FullOpacity {
pixel_outcome = Some(full_opacity(client, &check.subject).await);
} else if action_error.is_none() && check.expect == qa::Expect::TransparentWindowTint {
pixel_outcome = Some(transparent_window_tint(client).await);
} else if action_error.is_none() && check.expect == qa::Expect::VisibleInk {
pixel_outcome = Some(visible_ink(client, &check.subject).await);
} else if action_error.is_none() && check.expect == qa::Expect::InteriorInk {
pixel_outcome = Some(interior_ink(client, &check.subject).await);
} else if action_error.is_none() && check.expect == qa::Expect::Contrast {
pixel_outcome = Some(
wait_for_contrast(
client,
check,
check_started.unwrap_or_else(Instant::now),
action_paint_armed,
)
.await,
);
}
let transport_timed_out = action_error
.as_deref()
.is_some_and(|error| error.contains("inspector did not answer within"));
if pixel_outcome.is_none()
&& let Some(before_pixels) = before_action_pixels
{
pixel_outcome = Some(match before_pixels {
Ok((node_id, before_pixels, event_driven)) => {
let deadline = check_started.unwrap_or_else(Instant::now)
+ declared_outcome_timeout(check);
let comparison = wait_for_pixels_change(
client,
node_id,
&check.subject,
&before_pixels,
deadline,
event_driven,
)
.await;
if comparison.is_err()
&& let Ok(after) =
capture_node_region(client, node_id, &check.subject).await
&& let Err(error) = save_pixel_artifacts(&check.id, &before_pixels, &after)
{
eprintln!("could not save pixel artifacts for {}: {error}", check.id);
}
comparison
}
Err(error) => Err(error),
});
}
if pixel_outcome.is_none()
&& let Some((node_id, before_size)) = before_font_size
{
pixel_outcome = Some(
wait_for_larger_font(
client,
node_id,
&check.subject,
before_size,
check_timeout(if check.outcome_timeout_ms == 0 {
900
} else {
check.outcome_timeout_ms
}),
)
.await,
);
}
let fallback_after = || AgentSnapshot {
nodes: before.nodes.clone(),
..AgentSnapshot::default()
};
let (after, settle_error, settle_iterations) = if live_paint_expect {
match inspect(client).await {
Ok((snapshot, _)) => (snapshot, None, 0),
Err(error) => (
fallback_after(),
Some(format!("could not inspect after the action: {error}")),
0,
),
}
} else if action_error.is_none() || transport_timed_out {
match settle_for_outcome(
client,
check,
&before.nodes,
before.focused_node,
action_target.as_deref(),
action_node_id,
action_paint_armed,
)
.await
{
Ok(settled) => settled,
Err(error) => (
fallback_after(),
Some(format!("could not inspect the rendered outcome: {error}")),
0,
),
}
} else {
match inspect(client).await {
Ok((snapshot, _)) => (snapshot, None, 0),
Err(error) => (
fallback_after(),
Some(format!(
"could not inspect after the failed action: {error}"
)),
0,
),
}
};
let mut outcome = if live_paint_expect {
match action_error {
Some(error) => Err(error),
None => pixel_outcome.unwrap_or_else(|| {
Err("the live paint expectation was not measured".to_owned())
}),
}
} else {
match action_error {
Some(error) if transport_timed_out => outcome_verdict(
check,
&before.nodes,
&after.nodes,
action_target.as_deref(),
action_node_id,
before.focused_node,
after.focused_node,
)
.map_err(|outcome| format!("{error}; rendered outcome also failed: {outcome}")),
Some(error) => Err(error),
None => outcome_verdict(
check,
&before.nodes,
&after.nodes,
action_target.as_deref(),
action_node_id,
before.focused_node,
after.focused_node,
),
}
};
if let Some(error) = settle_error {
outcome = Err(match outcome {
Ok(()) => error,
Err(existing) => format!("{existing}; {error}"),
});
}
let elapsed = check_started.unwrap_or_else(Instant::now).elapsed();
let declared_outcome = if check.outcome_timeout_ms == 0 {
900
} else {
check.outcome_timeout_ms
};
let verdict_budget = check_timeout(1_250.max(declared_outcome.saturating_add(250)));
if elapsed > verdict_budget {
let timing = format!(
"check exceeded {}ms ({:.0}ms)",
verdict_budget.as_millis(),
elapsed.as_secs_f64() * 1000.0
);
outcome = Err(match outcome {
Ok(()) => timing,
Err(error) => format!("{error}; {timing}"),
});
}
if cli::trace() {
match &outcome {
Ok(()) => println!(
" verdict pass: {} ({:.0}ms)",
check.id,
elapsed.as_secs_f64() * 1000.0
),
Err(error) => println!(
" verdict fail: {} ({:.0}ms): {}",
check.id,
elapsed.as_secs_f64() * 1000.0,
redact_session_text(error, secrets)
),
}
}
Ok((outcome, settle_iterations, retries))
}
.await;
let gave_up = match &produced {
Ok((outcome, _, _)) => outcome
.as_ref()
.err()
.is_some_and(|error| host_gave_up(error)),
Err(error) => host_gave_up_report(error),
};
match produced {
Ok((outcome, settle_iterations, retries)) => results.push(CheckResult {
check,
outcome,
duration_ms: full_check_started.elapsed().as_millis() as u64,
settle_iterations,
retries,
}),
Err(error) if gave_up => results.push(CheckResult {
check,
outcome: Err(host_failure_text(&error)),
duration_ms: full_check_started.elapsed().as_millis() as u64,
settle_iterations: 0,
retries: 0,
}),
Err(error) => return Err(error),
}
if !gave_up && check.settle_after_ms > 0 {
nagoya::sleep(Duration::from_millis(check.settle_after_ms)).await;
}
let group_done = !gave_up
&& selected[index + 1..]
.iter()
.all(|later| later.group != check.group);
if group_done {
dump_qa_hosted_gpu(client, &check.group).await?;
}
if gave_up && index + 1 < selected.len() {
if let Some(session) = hosted.as_deref_mut() {
reconnect_hosted(client, session).await?;
} else {
let budget = check_recovery_budget(client, check);
if !inspector_answers_again(client, budget).await {
let id = &check.id;
let message = format!("host stopped answering after {id}");
for later in &selected[index + 1..] {
results.push(CheckResult {
check: later,
outcome: Err(message.clone()),
duration_ms: 0,
settle_iterations: 0,
retries: 0,
});
}
break;
}
}
}
}
let failed = results
.iter()
.filter(|result| result.outcome.is_err())
.count();
let tally_input: Vec<_> = results
.iter()
.map(|result| (result.check, result.outcome.clone()))
.collect();
let tally = qa::tally(&tally_input);
let mut groups: Vec<_> = tally.iter().collect();
groups.sort_by_key(|(name, _)| *name);
let report = Report {
passed: results.len() - failed,
failed,
groups: groups
.into_iter()
.map(|(name, (passed, total))| GroupRow {
name: name.to_string(),
passed: *passed,
total: *total,
})
.collect(),
checks: results.iter().map(CheckRow::from).collect(),
};
let encoded = toon_format::encode_default(&report).map_err(|e| eyre!(e.to_string()))?;
let printable = match secrets {
Some(secrets) => secrets.redact(&encoded),
None => encoded,
};
println!("{printable}");
Ok(failed)
}
static QA_HOSTED_GPU_DUMP: std::sync::OnceLock<Option<std::path::PathBuf>> =
std::sync::OnceLock::new();
fn note_qa_hosted_run() {
let dir = std::env::var_os("PS_QA_GPU_DUMP").and_then(|value| {
if value.is_empty() {
None
} else {
Some(std::path::PathBuf::from(value))
}
});
let _ = QA_HOSTED_GPU_DUMP.set(dir);
}
fn qa_hosted_gpu_dump_dir() -> Option<std::path::PathBuf> {
QA_HOSTED_GPU_DUMP.get().cloned().flatten()
}
fn gpu_glob_matches(pattern: &str, text: &str) -> bool {
let Some((prefix, rest)) = pattern.split_once('*') else {
return pattern == text;
};
if !text.starts_with(prefix) {
return false;
}
let text = &text[prefix.len()..];
if rest.is_empty() {
return true;
}
let mut skip = 0;
loop {
if gpu_glob_matches(rest, &text[skip..]) {
return true;
}
let Some(next) = text[skip..].chars().next() else {
return false;
};
skip += next.len_utf8();
}
}
fn gpu_draw_target_matches(subject: &str, target: &str) -> bool {
if target.starts_with("canvas:") {
subject == target
} else {
gpu_glob_matches(subject, target)
}
}
async fn dump_qa_hosted_gpu(client: &mut Client, group: &str) -> Result<()> {
let Some(dir) = qa_hosted_gpu_dump_dir() else {
return Ok(());
};
if group.is_empty() || group.contains('/') || group.contains('\\') || group.contains('\0') {
return Err(eyre!(
"GPU dump group {group:?} is not a single path segment"
));
}
let report = match gpu_report(client).await {
Ok(report) => report,
Err(_) => return Ok(()),
};
let encoded = toon_format::encode_default(&report).map_err(|error| eyre!(error.to_string()))?;
std::fs::create_dir_all(&dir)
.with_context(|| format!("creating GPU dump directory {}", dir.display()))?;
let path = dir.join(format!("{group}.gpu.toon"));
std::fs::write(&path, format!("{encoded}\n"))
.with_context(|| format!("writing GPU dump {}", path.display()))?;
Ok(())
}
fn gpu_expectation(expect: qa::Expect) -> bool {
matches!(
expect,
qa::Expect::GpuNoErrors | qa::Expect::GpuHasResource | qa::Expect::GpuDraws(_)
)
}
async fn gpu_report(client: &mut Client) -> Result<blitz_control_protocol::WebGpuReport> {
match client
.diagnostics(&DiagnosticsRequest::WebGpuInspect)
.await?
.response
{
DebugResponse::WebGpu(report) => Ok(report),
other => bail!("asked for WebGPU inspection, got {other:?}"),
}
}
fn gpu_verdict(
check: &qa::Check,
report: &blitz_control_protocol::WebGpuReport,
) -> std::result::Result<(), String> {
match check.expect {
qa::Expect::GpuNoErrors => {
if report.errors.validation_total == 0 && report.errors.uncaptured_total == 0 {
Ok(())
} else {
Err(format!(
"WebGPU has {} validation and {} uncaptured error(s) since load ({} total)",
report.errors.validation_total,
report.errors.uncaptured_total,
report.errors.total,
))
}
}
qa::Expect::GpuHasResource => {
let (kind, label) = check
.subject
.split_once(':')
.ok_or_else(|| "GpuHasResource subject must be kind:label".to_owned())?;
if report.resources.iter().any(|resource| {
resource.details.kind_name() == kind && gpu_glob_matches(label, &resource.label)
}) {
Ok(())
} else {
Err(format!(
"no live WebGPU resource matching {:?}",
check.subject
))
}
}
qa::Expect::GpuDraws(required) => {
let frame = report
.frames
.last()
.ok_or_else(|| "WebGPU has no submitted frame to inspect".to_owned())?;
if !frame.complete {
return Err(format!(
"WebGPU frame {} was not completely sampled",
frame.sequence
));
}
let count = frame
.passes
.iter()
.filter(|pass| {
pass.color_targets.iter().flatten().any(|target| {
gpu_draw_target_matches(&check.subject, &target.target)
|| target.resolve_target.as_deref().is_some_and(|resolve| {
gpu_draw_target_matches(&check.subject, resolve)
})
}) || pass.depth_target.as_ref().is_some_and(|target| {
gpu_draw_target_matches(&check.subject, &target.target)
})
})
.fold(0_u64, |count, pass| count.saturating_add(pass.draw_count()));
if count >= required {
Ok(())
} else {
Err(format!(
"WebGPU frame {} drew {} time(s) into {:?}; expected at least {}",
frame.sequence, count, check.subject, required,
))
}
}
_ => Err("not a GPU expectation".to_owned()),
}
}
async fn wait_for_gpu(
client: &mut Client,
check: &qa::Check,
started: Instant,
) -> std::result::Result<(), String> {
let deadline = started + declared_outcome_timeout(check);
let stable_for = Duration::from_millis(check.stable_for_ms);
let mut passing_since = None;
loop {
let report = gpu_report(client)
.await
.map_err(|error| error.to_string())?;
let verdict = gpu_verdict(check, &report);
let now = Instant::now();
if verdict.is_ok() {
let since = passing_since.get_or_insert(now);
if now.duration_since(*since) >= stable_for {
return Ok(());
}
} else {
passing_since = None;
if check.expect == qa::Expect::GpuNoErrors {
return verdict;
}
}
if now >= deadline {
return verdict.and_then(|()| {
Err(format!(
"GPU outcome did not remain complete for {}ms",
stable_for.as_millis(),
))
});
}
nagoya::sleep(
deadline
.saturating_duration_since(now)
.min(Duration::from_millis(25)),
)
.await;
}
}
fn outcome_verdict(
check: &qa::Check,
before: &[SemanticNode],
after: &[SemanticNode],
action_target: Option<&str>,
action_node_id: Option<u64>,
before_focused: Option<u64>,
after_focused: Option<u64>,
) -> std::result::Result<(), String> {
if check.expect == qa::Expect::TargetPaints {
let Some(subject) = action_target else {
return Err("could not resolve the exact click target".to_owned());
};
let mut targeted = check.clone();
targeted.subject = subject.to_owned();
targeted.expect = qa::Expect::Paints;
qa::verdict(&targeted, before, after)
} else if check.expect == qa::Expect::ValueChanges
&& action_target.is_some_and(|target| target == check.subject)
&& let Some(node_id) = action_node_id
{
qa::value_changed(node_id, before, after)
} else if check.expect == qa::Expect::SelectionChanges
&& action_target.is_some_and(|target| target == check.subject)
&& let Some(node_id) = action_node_id
{
qa::selection_changed(node_id, before, after)
} else if check.expect == qa::Expect::FocusMoves {
let destination = qa::matching(after, &check.subject)
.into_iter()
.find(|node| qa::paints(node))
.ok_or_else(|| format!("no painted focus destination matching {:?}", check.subject))?;
if after_focused != Some(destination.id) {
return Err(format!(
"focus ended on {:?}, expected {} for {:?}",
after_focused, destination.id, check.subject
));
}
if before_focused == after_focused {
return Err(format!("focus did not move from {:?}", before_focused));
}
Ok(())
} else {
qa::verdict(check, before, after)
}
}
#[derive(Default)]
struct OutcomeStability {
fingerprint: Option<u64>,
since: Option<std::time::Instant>,
}
impl OutcomeStability {
fn observe(
&mut self,
now: std::time::Instant,
fingerprint: u64,
passing: bool,
required: Duration,
) -> bool {
if !passing {
self.fingerprint = None;
self.since = None;
return false;
}
if required.is_zero() {
return true;
}
if self.fingerprint != Some(fingerprint) {
self.fingerprint = Some(fingerprint);
self.since = Some(now);
return false;
}
self.since
.is_some_and(|since| now.duration_since(since) >= required)
}
fn remaining(&self, now: std::time::Instant, required: Duration) -> Duration {
self.since
.map(|since| required.saturating_sub(now.saturating_duration_since(since)))
.unwrap_or_default()
}
}
fn semantic_fingerprint(nodes: &[SemanticNode]) -> u64 {
let mut fingerprint = DefaultHasher::new();
nodes.len().hash(&mut fingerprint);
for node in nodes {
node.dom_id.hash(&mut fingerprint);
node.id.hash(&mut fingerprint);
node.parent.hash(&mut fingerprint);
node.role.hash(&mut fingerprint);
node.name.hash(&mut fingerprint);
node.value.hash(&mut fingerprint);
node.enabled.hash(&mut fingerprint);
node.visible.hash(&mut fingerprint);
node.selected.hash(&mut fingerprint);
node.bounds
.map(|bounds| bounds.map(f64::to_bits))
.hash(&mut fingerprint);
node.slot.hash(&mut fingerprint);
}
fingerprint.finish()
}
struct OutcomePollScope {
root: u64,
baseline_ids: HashSet<u64>,
}
fn outcome_poll_scope(nodes: &[SemanticNode]) -> Option<OutcomePollScope> {
let surface = reach::surfaces()
.iter()
.find(|surface| reach::on_surface(nodes, surface))?;
let ids = reach::on_surface_subtree(nodes, surface);
let root = *ids.first()?;
(ids.len() < nodes.len()).then(|| OutcomePollScope {
root,
baseline_ids: ids.into_iter().collect(),
})
}
fn scope_for_passing_snapshot(nodes: &[SemanticNode], subject: &str) -> Option<OutcomePollScope> {
let candidate = outcome_poll_scope(nodes)?;
subject_belongs_to_scope(nodes, subject, &candidate.baseline_ids).then_some(candidate)
}
fn subject_belongs_to_scope(
nodes: &[SemanticNode],
subject: &str,
scope_ids: &HashSet<u64>,
) -> bool {
let mut subject_seen = false;
let mut subject_in_surface = false;
for node in nodes
.iter()
.filter(|node| selector_matches_node(node, subject))
{
subject_seen = true;
subject_in_surface |= scope_ids.contains(&node.id);
}
subject_seen && subject_in_surface
}
fn merge_outcome_snapshot(
before: &[SemanticNode],
mut scoped: AgentSnapshot,
baseline_ids: &HashSet<u64>,
) -> AgentSnapshot {
let mut nodes: Vec<_> = before
.iter()
.filter(|node| !baseline_ids.contains(&node.id))
.cloned()
.collect();
nodes.append(&mut scoped.nodes);
scoped.nodes = nodes;
scoped
}
const FULL_DOCUMENT_PROBE_INTERVAL: Duration = Duration::from_millis(250);
async fn wait_for_contrast(
client: &mut Client,
check: &qa::Check,
started: Instant,
event_driven: bool,
) -> std::result::Result<(), String> {
let deadline = started + declared_outcome_timeout(check);
let stable_for = Duration::from_millis(check.stable_for_ms);
let mut passing_since = None;
loop {
let verdict = paint_audit::contrast_verdict(client, &check.subject)
.await
.map_err(|error| error.to_string());
let now = Instant::now();
if verdict.is_ok() {
let since = *passing_since.get_or_insert(now);
if now.duration_since(since) >= stable_for {
return Ok(());
}
} else {
passing_since = None;
}
if now >= deadline {
return verdict.and_then(|()| {
Err(format!(
"contrast did not remain above its floor for {}ms within the outcome window",
stable_for.as_millis()
))
});
}
let remaining = deadline.saturating_duration_since(now);
let wait = passing_since.map_or(remaining, |since| {
stable_for
.saturating_sub(now.duration_since(since))
.min(remaining)
});
if event_driven {
client
.wait_for_paint(wait)
.await
.map_err(|error| error.to_string())?;
} else {
nagoya::sleep(wait.min(Duration::from_millis(25))).await;
}
}
}
async fn settle_for_outcome(
client: &mut Client,
check: &qa::Check,
before: &[SemanticNode],
before_focused: Option<u64>,
action_target: Option<&str>,
action_node_id: Option<u64>,
already_armed: bool,
) -> Result<(AgentSnapshot, Option<String>, u32)> {
let outcome_timeout = declared_outcome_timeout(check);
let deadline = std::time::Instant::now() + outcome_timeout;
let stable_for = Duration::from_millis(check.stable_for_ms);
let mut stability = OutcomeStability::default();
let mut iterations = 0;
let mut scope = outcome_poll_scope(before);
let mut last_full_probe: Option<std::time::Instant> = None;
let mut revealed_subject = false;
let event_driven = if already_armed {
true
} else {
client.arm_paint_events().await.unwrap_or(false)
};
loop {
let mut after = if let Some(scoped) = scope.as_ref() {
match inspect_subtree(client, scoped.root).await {
Ok((snapshot, _)) => merge_outcome_snapshot(before, snapshot, &scoped.baseline_ids),
Err(_) => {
let full = inspect(client).await?.0;
scope = outcome_poll_scope(&full.nodes);
full
}
}
} else {
inspect(client).await?.0
};
iterations += 1;
let now = std::time::Instant::now();
let mut passing = outcome_verdict(
check,
before,
&after.nodes,
action_target,
action_node_id,
before_focused,
after.focused_node,
)
.is_ok();
let due_for_full_probe =
last_full_probe.is_none_or(|at| now.duration_since(at) >= FULL_DOCUMENT_PROBE_INTERVAL);
if !passing && scope.is_some() && due_for_full_probe {
after = inspect(client).await?.0;
last_full_probe = Some(now);
passing = outcome_verdict(
check,
before,
&after.nodes,
action_target,
action_node_id,
before_focused,
after.focused_node,
)
.is_ok();
scope = if passing {
scope_for_passing_snapshot(&after.nodes, &check.subject)
} else {
outcome_poll_scope(&after.nodes)
};
}
if !passing
&& !revealed_subject
&& matches!(check.expect, qa::Expect::Paints | qa::Expect::PaintsNamed)
{
let offscreen = qa::matching(&after.nodes, &check.subject)
.into_iter()
.find(|node| {
node.bounds.is_some_and(|bounds| {
bounds[2] > 0.0
&& bounds[3] > 0.0
&& crate::target::offscreen(
bounds,
crate::target::viewport_for_node_in(&after.nodes, node.id),
)
})
})
.map(|node| node.id);
if let Some(node_id) = offscreen {
revealed_subject = true;
let _ = client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id,
}))
.await;
continue;
}
}
if stability.observe(now, semantic_fingerprint(&after.nodes), passing, stable_for) {
return Ok((after, None, iterations));
}
if now >= deadline {
let error = (passing && !stable_for.is_zero()).then(|| {
format!(
"rendered outcome did not remain complete and unchanged for {}ms within {}ms",
stable_for.as_millis(),
outcome_timeout.as_millis()
)
});
return Ok((after, error, iterations));
}
if event_driven {
let until_deadline = deadline.saturating_duration_since(now);
let wait = if passing {
stability.remaining(now, stable_for).min(until_deadline)
} else {
until_deadline
};
if !wait.is_zero() {
let _ = client.wait_for_paint(wait).await?;
}
} else {
nagoya::sleep(Duration::from_millis(25)).await;
}
}
}
fn declared_open_timeout(check: &qa::Check) -> Duration {
check_timeout(if check.open_timeout_ms == 0 {
900
} else {
check.open_timeout_ms
})
}
fn declared_outcome_timeout(check: &qa::Check) -> Duration {
check_timeout(if check.outcome_timeout_ms == 0 {
900
} else {
check.outcome_timeout_ms
})
}
fn surface_for_opener(want: &str) -> Option<&'static reach::Surface> {
reach::surfaces()
.iter()
.find(|surface| surface.opener == want)
.or_else(|| {
reach::profile()
.document_openers
.iter()
.any(|opener| opener.eq_ignore_ascii_case(want))
.then(|| {
reach::surfaces()
.iter()
.find(|surface| surface.opener == reach::DYNAMIC_DOCUMENT)
})
.flatten()
})
}
fn is_named_document_opener(want: &str) -> bool {
named_document_opener_for(reach::profile(), want)
}
fn named_document_opener_for(profile: &crate::app::AppProfile, want: &str) -> bool {
profile
.document_openers
.iter()
.any(|opener| opener.eq_ignore_ascii_case(want))
}
fn arrived_without_navigation(
nodes: &[SemanticNode],
destination: Option<&reach::Surface>,
want_here: &str,
named_document: bool,
active_document_matches: bool,
) -> bool {
if named_document {
active_document_matches
&& destination.is_some_and(|surface| reach::on_surface(nodes, surface))
} else {
destination.map_or_else(
|| painted_named(nodes, want_here),
|surface| reach::on_surface(nodes, surface),
)
}
}
fn named_document_is_active(nodes: &[SemanticNode], want: &str) -> bool {
named_document_is_active_with_permanent(nodes, want, &reach::profile().permanent_surfaces)
}
fn named_document_is_active_with_permanent(
nodes: &[SemanticNode],
want: &str,
permanent_surfaces: &[String],
) -> bool {
let tab_name = format!("{want}{want}");
let exact_document_selected = nodes.iter().any(|node| {
node.role == "button"
&& (node.name.eq_ignore_ascii_case(want) || node.name.eq_ignore_ascii_case(&tab_name))
&& node.selected
&& node.visible
&& painted_bounds(node).is_some()
});
let permanent_surface_selected = nodes.iter().any(|node| {
node.role == "button"
&& node.selected
&& node.visible
&& painted_bounds(node).is_some()
&& permanent_surfaces.iter().any(|surface| {
node.name.eq_ignore_ascii_case(surface)
|| node
.name
.eq_ignore_ascii_case(&format!("{surface}{surface}"))
})
});
exact_document_selected && !permanent_surface_selected
}
async fn click_opener_quiet(client: &mut Client, want: &str, named_document: bool) -> Result<u64> {
if named_document {
let tab = format!("button:{want}{want}");
match click_named_quiet(client, &tab).await {
Ok(node_id) => return Ok(node_id),
Err(error) if host_gave_up_report(&error) => return Err(error),
Err(_) => {}
}
}
click_named_quiet(client, want).await
}
async fn click_by_id(client: &mut Client, node_id: u64) -> Result<()> {
client
.agent(&AgentControlRequest::Act(AgentAction::Click { node_id }))
.await?;
Ok(())
}
#[derive(serde::Serialize)]
struct Report {
passed: usize,
failed: usize,
groups: Vec<GroupRow>,
checks: Vec<CheckRow>,
}
struct CheckResult<'a> {
check: &'a qa::Check,
outcome: std::result::Result<(), String>,
duration_ms: u64,
settle_iterations: u32,
retries: u32,
}
#[derive(serde::Serialize)]
struct GroupRow {
name: String,
passed: usize,
total: usize,
}
#[derive(serde::Serialize)]
struct CheckRow {
verdict: &'static str,
group: String,
id: String,
duration_ms: u64,
settle_iterations: u32,
retries: u32,
error: String,
what: String,
}
impl From<&CheckResult<'_>> for CheckRow {
fn from(result: &CheckResult<'_>) -> Self {
Self {
verdict: if result.outcome.is_ok() {
"pass"
} else {
"fail"
},
group: result.check.group.clone(),
id: result.check.id.clone(),
duration_ms: result.duration_ms,
settle_iterations: result.settle_iterations,
retries: result.retries,
error: result.outcome.clone().err().unwrap_or_default(),
what: result.check.what.clone(),
}
}
}
#[derive(serde::Serialize)]
struct FoundRow {
id: u64,
role: String,
name: String,
x: f64,
y: f64,
w: f64,
h: f64,
state: String,
}
#[expect(
clippy::too_many_arguments,
reason = "each one is a filter the caller asks for by name"
)]
async fn find(
client: &mut Client,
pattern: &str,
roles: &[String],
visible: bool,
hidden: bool,
painted: bool,
offscreen_only: bool,
disabled: bool,
count_only: bool,
limit: Option<usize>,
) -> Result<()> {
let (snapshot, elapsed) = inspect(client).await?;
let viewport = viewport_of(&snapshot);
let rows: Vec<FoundRow> = snapshot
.nodes
.iter()
.filter(|node| {
roles.is_empty()
|| roles
.iter()
.any(|role| role.eq_ignore_ascii_case(&node.role))
})
.filter(|node| match pattern.strip_prefix('@') {
Some(slot) => node.slot.as_deref().is_some_and(|have| have == slot),
None => name_matches(&node.name, pattern),
})
.filter(|node| !visible || node.visible)
.filter(|node| !hidden || !node.visible)
.filter(|node| !disabled || !node.enabled)
.filter(|node| {
!painted
|| node
.bounds
.is_some_and(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
})
.filter(|node| {
!offscreen_only
|| node
.bounds
.is_some_and(|bounds| offscreen(bounds, viewport))
})
.map(|node| {
let bounds = node.bounds.unwrap_or([0.0; 4]);
let mut state: Vec<&str> = Vec::new();
state.push(if node.visible { "visible" } else { "hidden" });
if !node.enabled {
state.push("disabled");
}
if node.selected {
state.push("selected");
}
if bounds[2] <= 0.0 || bounds[3] <= 0.0 {
state.push("0x0");
} else if offscreen(bounds, viewport) {
state.push("offscreen");
}
let round = |value: f64| (value * 10.0).round() / 10.0;
FoundRow {
id: node.id,
role: node.role.clone(),
name: node.name.clone(),
x: round(bounds[0]),
y: round(bounds[1]),
w: round(bounds[2]),
h: round(bounds[3]),
state: state.join(","),
}
})
.collect();
let matched = rows.len();
let shown: Vec<FoundRow> = match limit {
Some(limit) => rows.into_iter().take(limit).collect(),
None => rows,
};
#[derive(serde::Serialize)]
struct Found {
matched: usize,
of: usize,
inspect_ms: f64,
controls: Vec<FoundRow>,
}
let report = Found {
matched,
of: snapshot.nodes.len(),
inspect_ms: (elapsed * 10.0).round() / 10.0,
controls: if count_only { Vec::new() } else { shown },
};
println!(
"{}",
toon_format::encode_default(&report).map_err(|error| eyre!(error.to_string()))?
);
Ok(())
}
async fn locate_button(client: &mut Client, want: &str) -> Result<(u64, [f64; 4])> {
locate_control(client, want, &["button"]).await
}
async fn open_named(client: &mut Client, want: &str) -> Result<()> {
let (id, _) = locate_button(client, want).await?;
if cli::trace() {
println!(" opening {want:?} (id {id})");
}
client
.agent(&AgentControlRequest::Act(AgentAction::DoubleClick {
node_id: id,
}))
.await?;
Ok(())
}
async fn press_named(client: &mut Client, want: &str) -> Result<()> {
let (role, name) = want.split_once(':').unwrap_or(("", want));
let roles: &[&str] = if role.is_empty() { &[] } else { &[role] };
let (id, b) = locate_control(client, name, roles).await?;
let (x, y) = (b[0] + b[2] / 2.0, b[1] + b[3] / 2.0);
if cli::trace() {
println!(" pressing {want:?} (id {id}) at {x:.0},{y:.0}");
}
for phase in [PointerPhase::Move, PointerPhase::Down, PointerPhase::Up] {
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Pointer {
phase,
x,
y,
button: 0,
modifiers: Modifiers::default(),
},
)))
.await?;
}
Ok(())
}
async fn click_named_quiet(client: &mut Client, want: &str) -> Result<u64> {
let (target_id, _) = locate_control(client, want, &[]).await?;
if cli::trace() {
println!(" activating {want:?} (id {target_id})");
}
client
.agent(&AgentControlRequest::Act(AgentAction::Click {
node_id: target_id,
}))
.await?;
Ok(target_id)
}
async fn drive_check_action(
client: &mut Client,
want: &str,
coordinate_press: bool,
prepared_node_id: Option<u64>,
retry_hover: Option<&qa::Hover>,
) -> std::result::Result<Option<u64>, String> {
if coordinate_press {
press_named(client, want)
.await
.map(|()| None)
.map_err(|error| format!("could not press {want:?}: {error}"))
} else {
let node_id = prepared_node_id
.ok_or_else(|| format!("could not click {want:?}: target was not prepared"))?;
if cli::trace() {
println!(" activating {want:?} (id {node_id})");
}
match client
.agent(&AgentControlRequest::Act(AgentAction::Click { node_id }))
.await
{
Ok(_) => Ok(Some(node_id)),
Err(error) if error.to_string().contains("notInteractable") => {
if let Some(hover) = retry_hover {
let reacquire = qa::Hover::Once(hover.target().to_owned());
repeat_hover(client, &reacquire, false)
.await
.map_err(|error| {
format!("could not restore hover for {want:?}: {error}")
})?;
}
let (current_id, _) = locate_control(client, want, &[])
.await
.map_err(|error| format!("could not relocate {want:?}: {error}"))?;
if cli::trace() {
println!(
" target {node_id} reconciled; activating current id {current_id}"
);
}
client
.agent(&AgentControlRequest::Act(AgentAction::Click {
node_id: current_id,
}))
.await
.map(|_| Some(current_id))
.map_err(|error| format!("could not click {want:?}: {error}"))
}
Err(error) => Err(format!("could not click {want:?}: {error}")),
}
}
}
async fn capture(
client: &mut Client,
want: &str,
scale: f32,
output: Option<&std::path::Path>,
) -> Result<()> {
let node_id = if want.is_empty() {
None
} else {
let (snapshot, _) = inspect(client).await?;
let node = snapshot
.nodes
.iter()
.filter(|node| selector_matches_node(node, want) && node.visible)
.filter_map(|node| node.bounds.map(|bounds| (node, bounds)))
.filter(|(_, bounds)| bounds[2] > 0.0 && bounds[3] > 0.0)
.max_by(|a, b| {
(a.1[2] * a.1[3])
.partial_cmp(&(b.1[2] * b.1[3]))
.unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(node, _)| node);
let Some(node) = node else {
bail!("no visible node with a box matching {want:?}");
};
println!(
"capturing {} role={} name={}",
node.id,
node.role,
report::py_repr(&node.name.chars().take(50).collect::<String>())
);
Some(node.id)
};
let answer = client
.diagnostics(&DiagnosticsRequest::Capture(CaptureRequest {
node_id,
scale,
}))
.await?;
let image = match answer.response {
DebugResponse::Captured(image) => image,
other => bail!("asked for a capture, got {other:?}"),
};
report_captured(&image, scale, output)
}
fn report_captured(
image: &CapturedImage,
scale: f32,
output: Option<&std::path::Path>,
) -> Result<()> {
let ink = measure_ink(image)?;
let fingerprint = {
use std::hash::{DefaultHasher, Hash as _, Hasher as _};
let mut hasher = DefaultHasher::new();
image.width.hash(&mut hasher);
image.height.hash(&mut hasher);
image.rgba_base64.hash(&mut hasher);
hasher.finish()
};
println!(
"{}x{} at {scale}x, background #{:02x}{:02x}{:02x}",
image.width, image.height, ink.background.0, ink.background.1, ink.background.2
);
println!("rgba fingerprint: {fingerprint:016x}");
println!(
"visible ink: {} of {} pixels ({:.2}%)",
ink.visible,
ink.total,
ink.fraction() * 100.0
);
if ink.visible == 0 {
println!("nothing was drawn: every pixel is the background colour");
}
if let Some(output) = output {
write_capture_ppm(image, output)?;
}
Ok(())
}
async fn read_gpu_capture(client: &mut Client, target: &str) -> Result<CapturedImage> {
let previous_timeout = client.request_timeout();
client.set_request_timeout(Duration::from_secs(15));
let answer = client
.diagnostics(&DiagnosticsRequest::WebGpuCapture {
target: target.to_owned(),
})
.await;
client.set_request_timeout(previous_timeout);
let answer = answer?;
match answer.response {
DebugResponse::Captured(image) => Ok(image),
DebugResponse::Error(error) => bail!("{}: {}", error.code, error.message),
other => bail!("asked for a WebGPU capture, got {other:?}"),
}
}
async fn gpu_capture_command(
client: &mut Client,
target: &str,
output: Option<&std::path::Path>,
) -> Result<()> {
let image = read_gpu_capture(client, target).await?;
report_captured(&image, 1.0, output)
}
async fn gpu_texture_ink(client: &mut Client, target: &str) -> std::result::Result<(), String> {
let image = read_gpu_capture(client, target)
.await
.map_err(|error| error.to_string())?;
let ink = measure_ink(&image).map_err(|error| error.to_string())?;
if ink.visible == 0 {
Err(format!(
"{target:?} occupies {}x{} but draws no pixels distinguishable from its background",
image.width, image.height
))
} else {
Ok(())
}
}
async fn wait_for_gpu_texture_ink(
client: &mut Client,
check: &qa::Check,
started: Instant,
) -> std::result::Result<(), String> {
let deadline = started + declared_outcome_timeout(check);
loop {
let verdict = gpu_texture_ink(client, &check.subject).await;
if verdict.is_ok() {
return Ok(());
}
let now = Instant::now();
if now >= deadline {
return verdict;
}
nagoya::sleep(
deadline
.saturating_duration_since(now)
.min(Duration::from_millis(25)),
)
.await;
}
}
async fn run_audit(client: &mut Client, family: Option<&str>) -> Result<usize> {
use audit::{Audited, Verdict};
let (snapshot, _) = inspect(client).await?;
let viewport = snapshot
.nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| (b[1], b[1] + b[3]))
.max_by(|a, b| {
(a.1 - a.0)
.partial_cmp(&(b.1 - b.0))
.unwrap_or(std::cmp::Ordering::Equal)
})
.unwrap_or((0.0, f64::MAX));
let painted = painted_nodes(client).await?;
let mut rows: Vec<Audited> = Vec::new();
for node in audit::buttons(&snapshot.nodes) {
let family_name = audit::family_of(&node.name);
if family.is_some_and(|want| family_name != want) {
continue;
}
let (width, height) = node.bounds.map(|b| (b[2], b[3])).unwrap_or((0.0, 0.0));
let verdict = if !node.visible {
Verdict::Hidden
} else if width <= 0.0 || height <= 0.0 {
Verdict::NoBox
} else if node
.bounds
.is_some_and(|b| b[1] + b[3] < viewport.0 || b[0] + b[2] < 0.0 || b[1] > viewport.1)
{
Verdict::Offscreen
} else if painted.contains(&node.id) {
Verdict::Drawn
} else {
Verdict::Blank
};
rows.push(Audited {
name: node.name.clone(),
family: family_name,
width,
height,
verdict,
});
}
if rows.is_empty() {
bail!("no buttons matched {family:?}");
}
println!("auditing {} buttons in the running app\n", rows.len());
let faults: Vec<&Audited> = rows.iter().filter(|row| row.verdict.is_fault()).collect();
if faults.is_empty() {
println!("no faults: every visible button was painted");
} else {
println!("{} button(s) nobody can see:\n", faults.len());
for row in &faults {
println!(
" {:<8} {:<52} {:.0}x{:.0}",
row.verdict.label(),
row.name.chars().take(52).collect::<String>(),
row.width,
row.height
);
}
println!();
}
let mut families: Vec<_> = audit::by_family(&rows).into_iter().collect();
families.sort_by_key(|(name, _)| *name);
for (name, (passed, total)) in families {
let mark = if passed == total { " " } else { "!" };
println!("{mark} {name:<12} {passed}/{total}");
}
println!("\n{} audited, {} faults", rows.len(), faults.len());
Ok(faults.len())
}
async fn painted_nodes(client: &mut Client) -> Result<HashSet<u64>> {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: false,
include_layout: false,
include_computed_style: true,
node_ids: Vec::new(),
}))
.await?;
let DebugResponse::Snapshot(snapshot) = answer.response else {
bail!("asked for a paint snapshot, got {:?}", answer.response);
};
Ok(snapshot
.computed_style
.as_ref()
.and_then(|value| value.as_array())
.map(|rows| {
rows.iter()
.filter_map(|row| row.get("nodeId")?.as_u64())
.collect()
})
.unwrap_or_default())
}
async fn run_sweep(client: &mut Client, family: Option<&str>) -> Result<usize> {
let (snapshot, _) = inspect(client).await?;
let viewport = snapshot
.nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| (b[1], b[1] + b[3]))
.max_by(|a, b| {
(a.1 - a.0)
.partial_cmp(&(b.1 - b.0))
.unwrap_or(std::cmp::Ordering::Equal)
})
.unwrap_or((0.0, f64::MAX));
let planned = sweep::cases(
&snapshot.nodes,
family,
audit::family_of,
reach::is_inert_control,
);
if planned.is_empty() {
bail!("no clickable buttons matched {family:?}");
}
println!("clicking {} buttons\n", planned.len());
let mut occurrences = HashMap::<String, usize>::new();
let planned: Vec<(sweep::Case, usize)> = planned
.into_iter()
.map(|case| {
let key = case.name.to_lowercase();
let occurrence = occurrences.entry(key).or_insert(0);
let planned_occurrence = *occurrence;
*occurrence += 1;
(case, planned_occurrence)
})
.collect();
let mut outcomes: Vec<sweep::Outcome> = Vec::new();
for (case, occurrence) in planned {
let (before, _) = inspect(client).await?;
let Some(node) = sweep::resolve_case(&case, occurrence, &before.nodes) else {
continue;
};
let node_id = node.id;
let node_bounds = node.bounds;
if node_bounds
.is_some_and(|b| b[1] + b[3] < viewport.0 || b[0] + b[2] < 0.0 || b[1] > viewport.1)
{
continue;
}
if let Err(error) = click_by_id(client, node_id).await {
outcomes.push(sweep::Outcome {
case,
failure: Some(format!("could not be clicked: {error}")),
});
continue;
}
let active_case = sweep::Case {
id: node_id,
..case.clone()
};
let after = settle_sweep_case(client, &active_case, &before.nodes).await?;
let failure = sweep::judge(&active_case, &before.nodes, &after.nodes);
outcomes.push(sweep::Outcome { case, failure });
}
let failures: Vec<&sweep::Outcome> = outcomes.iter().filter(|o| o.failure.is_some()).collect();
if failures.is_empty() {
println!("every button acted");
} else {
println!("{} button(s) did not act:\n", failures.len());
for outcome in &failures {
println!(
" {:<48} {}",
outcome.case.name.chars().take(48).collect::<String>(),
outcome.failure.as_deref().unwrap_or("")
);
}
println!();
}
let mut by_family: HashMap<&'static str, (usize, usize)> = HashMap::new();
for outcome in &outcomes {
let entry = by_family.entry(outcome.case.family).or_insert((0, 0));
entry.1 += 1;
if outcome.failure.is_none() {
entry.0 += 1;
}
}
let mut families: Vec<_> = by_family.into_iter().collect();
families.sort_by_key(|(name, _)| *name);
for (name, (passed, total)) in families {
let mark = if passed == total { " " } else { "!" };
println!("{mark} {name:<12} {passed}/{total}");
}
println!(
"\n{} clicked, {} did not act",
outcomes.len(),
failures.len()
);
Ok(failures.len())
}
async fn expand_everything(client: &mut Client, surface: &reach::Surface) -> Result<usize> {
let mut opened = 0;
for _ in 0..6 {
let (tree, _) = inspect(client).await?;
let mine: std::collections::HashSet<u64> = reach::on_surface_subtree(&tree.nodes, surface)
.into_iter()
.collect();
let mut todo: Vec<String> =
reach::expanders(&tree.nodes, &reach::profile().expand_prefixes)
.into_iter()
.filter(|(id, _)| mine.contains(id))
.map(|(_, name)| name)
.collect();
todo.sort();
todo.dedup();
if todo.is_empty() {
break;
}
for name in todo {
if click_named_quiet(client, &name).await.is_ok() {
opened += 1;
}
}
}
Ok(opened)
}
async fn reveal_deferred_content(
client: &mut Client,
surface: &reach::Surface,
want: &str,
) -> Result<usize> {
let materialized = materialize_deferred_content(client, surface, want).await?
+ materialize_paginated_content(client, surface).await?;
if materialized > 0 {
let (snapshot, _) = inspect(client).await?;
if painted_named(&snapshot.nodes, want) {
return Ok(materialized);
}
}
for step in 0..8 {
let (snapshot, _) = inspect(client).await?;
if painted_named(&snapshot.nodes, want) {
return Ok(step);
}
let scope: HashSet<u64> = reach::on_surface_subtree(&snapshot.nodes, surface)
.into_iter()
.collect();
let Some(target) = snapshot
.nodes
.iter()
.filter(|node| scope.contains(&node.id))
.filter_map(|node| {
node.bounds
.filter(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
.map(|bounds| (node.id, bounds[1] + bounds[3]))
})
.max_by(|left, right| {
left.1
.partial_cmp(&right.1)
.unwrap_or(std::cmp::Ordering::Equal)
})
else {
return Ok(step);
};
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id: target.0,
}))
.await?;
}
Ok(8)
}
async fn materialize_deferred_content(
client: &mut Client,
surface: &reach::Surface,
want: &str,
) -> Result<usize> {
let Some(field) = surface.reveal_with.as_deref() else {
return Ok(0);
};
let query = want.split_once(':').map_or(want, |(_, name)| name);
type_text(client, field, query).await?;
if wait_for_arrival(client, None, want, check_timeout(900)).await? {
type_text(client, field, "").await?;
if !wait_for_arrival(client, None, want, check_timeout(900)).await? {
type_text(client, field, query).await?;
let _ = wait_for_arrival(client, None, want, check_timeout(900)).await?;
}
} else {
type_text(client, field, "").await?;
}
Ok(1)
}
async fn materialize_scrolled_content(
client: &mut Client,
surface: &reach::Surface,
) -> Result<usize> {
let mut previous_count = None;
let mut stable_passes = 0usize;
let mut scrolls = 0usize;
let (initial, _) = inspect(client).await?;
let initial_scope: HashSet<u64> = reach::on_surface_subtree(&initial.nodes, surface)
.into_iter()
.collect();
if let Some(top) = initial
.nodes
.iter()
.filter(|node| initial_scope.contains(&node.id))
.filter(|node| node.role == "heading" || reach::interactive(node))
.filter_map(|node| {
node.bounds
.filter(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
.map(|bounds| (node.id, bounds[1]))
})
.min_by(|left, right| left.1.total_cmp(&right.1))
{
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id: top.0,
}))
.await?;
scrolls += 1;
}
for _ in 0..16 {
let (tree, _) = inspect(client).await?;
let scope: HashSet<u64> = reach::on_surface_subtree(&tree.nodes, surface)
.into_iter()
.collect();
let interactive = tree
.nodes
.iter()
.filter(|node| scope.contains(&node.id) && reach::interactive(node))
.count();
if cli::trace() {
println!(
" inventory materialize surface={} interactive={} stable={}",
surface.name, interactive, stable_passes
);
}
if previous_count == Some(interactive) {
stable_passes += 1;
if stable_passes >= 2 {
return Ok(scrolls);
}
} else {
previous_count = Some(interactive);
stable_passes = 0;
}
let Some(target) = tree
.nodes
.iter()
.filter(|node| scope.contains(&node.id))
.filter_map(|node| {
node.bounds
.filter(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
.map(|bounds| (node.id, bounds[1]))
})
.max_by(|left, right| left.1.total_cmp(&right.1))
else {
return Ok(scrolls);
};
if cli::trace() {
println!(
" inventory reveal deepest node={} y={:.1}",
target.0, target.1
);
}
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id: target.0,
}))
.await?;
scrolls += 1;
}
Ok(scrolls)
}
fn is_pagination_control(node: &SemanticNode, scope: &HashSet<u64>, patterns: &[String]) -> bool {
scope.contains(&node.id)
&& node.role == "button"
&& node.enabled
&& node.visible
&& node
.bounds
.is_some_and(|bounds| bounds[2] > 0.0 && bounds[3] > 0.0)
&& patterns
.iter()
.any(|pattern| name_matches(&node.name, pattern))
}
type SemanticShape = (String, String, Option<String>);
fn semantic_shapes(nodes: &[SemanticNode], scope: &HashSet<u64>) -> HashSet<SemanticShape> {
nodes
.iter()
.filter(|node| scope.contains(&node.id))
.map(|node| (node.role.clone(), node.name.clone(), node.value.clone()))
.collect()
}
fn pagination_advanced(
previous_shapes: &HashSet<SemanticShape>,
previous_name: &str,
current_shapes: &HashSet<SemanticShape>,
current: &SemanticNode,
) -> bool {
current.name != previous_name || current_shapes.len() > previous_shapes.len()
}
async fn materialize_paginated_content(
client: &mut Client,
surface: &reach::Surface,
) -> Result<usize> {
let patterns = &reach::profile().pagination_controls;
if patterns.is_empty() {
return Ok(0);
}
let mut revealed = 0;
let mut retired_pagers = HashSet::new();
for _ in 0..32 {
let (tree, _) = inspect(client).await?;
let scope: HashSet<u64> = reach::on_surface_subtree(&tree.nodes, surface)
.into_iter()
.collect();
let target = tree.nodes.iter().find(|node| {
!retired_pagers.contains(&node.id) && is_pagination_control(node, &scope, patterns)
});
let Some(target) = target else {
return Ok(revealed);
};
let node_id = target.id;
let mut pager_name = target.name.clone();
let mut pager_shapes = semantic_shapes(&tree.nodes, &scope);
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id,
}))
.await?;
let mut removed = false;
for _ in 0..128 {
if click_by_id(client, node_id).await.is_err() {
removed = true;
break;
}
revealed += 1;
let (after, _) = inspect(client).await?;
let after_scope: HashSet<u64> = reach::on_surface_subtree(&after.nodes, surface)
.into_iter()
.collect();
let after_shapes = semantic_shapes(&after.nodes, &after_scope);
let still_present = after.nodes.iter().find(|node| {
node.id == node_id && is_pagination_control(node, &after_scope, patterns)
});
match still_present {
Some(current)
if pagination_advanced(&pager_shapes, &pager_name, &after_shapes, current) =>
{
pager_name.clone_from(¤t.name);
pager_shapes = after_shapes;
}
_ => {
retired_pagers.insert(node_id);
removed = true;
break;
}
}
}
if !removed {
bail!("pagination node {node_id} did not disappear after 128 activations");
}
}
bail!("pagination controls did not terminate after 32 semantic identities")
}
async fn hover_all_rows(client: &mut Client) -> Result<usize> {
let (tree, _) = inspect(client).await?;
let window = tree
.nodes
.iter()
.filter(|node| node.role == "main")
.filter_map(|node| node.bounds)
.map(|b| (b[1], b[1] + b[3]))
.max_by(|a, b| {
(a.1 - a.0)
.partial_cmp(&(b.1 - b.0))
.unwrap_or(std::cmp::Ordering::Equal)
})
.unwrap_or((0.0, 4000.0));
let mut revealed = 0;
for node_id in reach::hover_row_ids(&tree.nodes, "listitem", window) {
if client
.agent(&AgentControlRequest::Act(AgentAction::Hover { node_id }))
.await
.is_ok()
{
revealed += 1;
}
}
Ok(revealed)
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum InventoryClass {
MissingId,
UnstableId,
DuplicateId,
Manual,
Isolated,
Anonymous,
Unreachable,
Disabled,
Reachable,
}
fn duplicate_dom_ids<'a>(
nodes: impl IntoIterator<Item = &'a SemanticNode>,
) -> std::collections::HashSet<String> {
let mut counts = std::collections::HashMap::<&str, usize>::new();
for dom_id in nodes
.into_iter()
.filter_map(|node| node.dom_id.as_deref())
.filter(|dom_id| !dom_id.trim().is_empty())
{
*counts.entry(dom_id).or_default() += 1;
}
counts
.into_iter()
.filter(|(_, count)| *count > 1)
.map(|(dom_id, _)| dom_id.to_owned())
.collect()
}
fn generated_dom_id(dom_id: &str) -> bool {
let mut pieces = dom_id.split('-');
match (pieces.next(), pieces.next()) {
(Some("cl"), Some(instance)) => instance.parse::<u64>().is_ok(),
(Some(slot), Some(instance)) => {
slot.parse::<u64>().is_ok() && instance.parse::<u64>().is_ok()
}
_ => false,
}
}
fn inventory_class(
node: &SemanticNode,
manual: bool,
isolated: bool,
duplicate_ids: &std::collections::HashSet<String>,
) -> InventoryClass {
if manual {
InventoryClass::Manual
} else if isolated {
InventoryClass::Isolated
} else if node.dom_id.as_deref().is_none_or(|id| id.trim().is_empty()) {
InventoryClass::MissingId
} else if node.dom_id.as_deref().is_some_and(generated_dom_id) {
InventoryClass::UnstableId
} else if node
.dom_id
.as_ref()
.is_some_and(|id| duplicate_ids.contains(id))
{
InventoryClass::DuplicateId
} else if node.name.trim().is_empty() {
InventoryClass::Anonymous
} else if !reach::onscreen(node) {
InventoryClass::Unreachable
} else if !node.enabled {
InventoryClass::Disabled
} else {
InventoryClass::Reachable
}
}
fn outcome_check_ids(node: &SemanticNode, checks: &[qa::Check]) -> Vec<String> {
checks
.iter()
.filter(|check| {
let driven = [
check.prepare.as_deref(),
check.hover.as_ref().map(qa::Hover::target),
check.after_prepare_hover.as_ref().map(qa::Hover::target),
check.click.as_deref(),
check.setup_type_into.as_deref(),
check.type_into.as_deref(),
check.key_on.as_deref(),
check.scroll_over.as_deref(),
check.pointer_drag.as_ref().map(|drag| drag.from.as_str()),
]
.into_iter()
.flatten()
.any(|selector| coverage_action_matches_node(node, selector));
let family = check
.covers
.iter()
.any(|selector| selector_matches_node(node, selector));
let focus_destination = check.expect == qa::Expect::FocusMoves
&& exact_selector_matches_node(node, &check.subject);
let measured_outcome = matches!(
check.expect,
qa::Expect::ValueChanges | qa::Expect::SelectionChanges | qa::Expect::NameChanges
) && exact_selector_matches_node(node, &check.subject);
let disabled_outcome = !node.enabled
&& matches!(check.expect, qa::Expect::Disabled)
&& coverage_action_matches_node(node, &check.subject);
driven || family || focus_destination || measured_outcome || disabled_outcome
})
.map(|check| check.id.clone())
.collect()
}
fn coverage_action_matches_node(node: &SemanticNode, selector: &str) -> bool {
if selector.contains('*') {
selector_matches_node(node, selector)
} else {
exact_selector_matches_node(node, selector)
}
}
#[derive(Debug, PartialEq, Eq, serde::Deserialize)]
struct SavedControl {
surface: String,
#[serde(default)]
dom_id: Option<String>,
#[serde(default)]
slot: Option<String>,
role: String,
name: String,
classification: String,
}
#[derive(serde::Deserialize)]
struct SavedInventory {
controls: Vec<SavedControl>,
}
fn saved_controls(report: &str) -> Result<Vec<SavedControl>, String> {
toon_format::decode_default::<SavedInventory>(report)
.map(|inventory| inventory.controls)
.map_err(|error| format!("inventory report is not valid TOON: {error}"))
}
fn saved_control_node(control: &SavedControl) -> SemanticNode {
SemanticNode {
dom_id: control.dom_id.clone(),
id: 0,
parent: None,
role: control.role.clone(),
name: control.name.clone(),
value: None,
enabled: !control.classification.contains("disabled"),
focusable: false,
viewport_fixed: false,
visible: !control.classification.contains("unreachable"),
selected: false,
bounds: Some([0.0, 0.0, 1.0, 1.0]),
slot: control.slot.clone(),
}
}
fn reconcile_inventory(
inventory: &std::path::Path,
checks_dir: Option<&std::path::Path>,
) -> Result<usize> {
#[derive(serde::Serialize)]
struct MissingRow {
surface: String,
dom_id: Option<String>,
slot: Option<String>,
role: String,
name: String,
classification: String,
checks: Vec<String>,
}
#[derive(serde::Serialize)]
struct ReconcileReport {
components: usize,
outcome_declared: usize,
excluded_manual: usize,
failed_existing: usize,
unverified: usize,
controls: Vec<MissingRow>,
}
let input = std::fs::read_to_string(inventory)?;
let controls = saved_controls(&input).map_err(eyre::Report::msg)?;
let checks = qa::checks(checks_dir).map_err(eyre::Report::msg)?;
let mut outcome_declared = 0;
let mut excluded_manual = 0;
let mut failed_existing = 0;
let mut missing = Vec::new();
for control in &controls {
let node = saved_control_node(control);
let matched = outcome_check_ids(&node, &checks);
if control.classification == "excluded-manual" {
excluded_manual += 1;
} else if control.classification.starts_with("failed-") {
failed_existing += 1;
missing.push(MissingRow {
surface: control.surface.clone(),
dom_id: control.dom_id.clone(),
slot: control.slot.clone(),
role: control.role.clone(),
name: control.name.clone(),
classification: control.classification.clone(),
checks: matched,
});
} else if control.classification.contains("isolated") || matched.is_empty() {
missing.push(MissingRow {
surface: control.surface.clone(),
dom_id: control.dom_id.clone(),
slot: control.slot.clone(),
role: control.role.clone(),
name: control.name.clone(),
classification: if control.classification.contains("isolated") {
"isolated-unverified".into()
} else if control.classification.contains("disabled") {
"state-disabled-unverified".into()
} else {
"outcome-unverified".into()
},
checks: matched,
});
} else {
outcome_declared += 1;
}
}
let unverified = missing
.iter()
.filter(|row| !row.classification.starts_with("failed-"))
.count();
let report = ReconcileReport {
components: controls.len(),
outcome_declared,
excluded_manual,
failed_existing,
unverified,
controls: missing,
};
println!(
"{}",
toon_format::encode_default(&report).map_err(|error| eyre!(error.to_string()))?
);
let blocking_unverified = report
.controls
.iter()
.filter(|row| reconciliation_gap_blocks(&row.classification))
.count();
Ok(report.failed_existing + blocking_unverified)
}
fn reconciliation_gap_blocks(classification: &str) -> bool {
!classification.starts_with("isolated-") && !classification.starts_with("failed-")
}
fn inventory_outcome_failures(unverified: usize, isolated: usize, required: bool) -> usize {
if required {
unverified.saturating_sub(isolated)
} else {
0
}
}
fn inventory_outcome_declared(class: InventoryClass, matched_checks: &[String]) -> bool {
class != InventoryClass::Manual
&& class != InventoryClass::Isolated
&& !matched_checks.is_empty()
}
fn inventory_control_unverified(class: InventoryClass, matched_checks: &[String]) -> bool {
match class {
InventoryClass::Manual => false,
InventoryClass::Isolated => true,
_ => matched_checks.is_empty(),
}
}
fn validate_surface_filter_against(only: Option<&str>, surfaces: &[reach::Surface]) -> Result<()> {
let Some(want) = only else {
return Ok(());
};
if surfaces
.iter()
.any(|surface| surface.name.eq_ignore_ascii_case(want))
{
return Ok(());
}
let available = surfaces
.iter()
.map(|surface| surface.name.as_str())
.collect::<Vec<_>>()
.join(", ");
bail!("unknown surface {want:?}; choose one of: {available}")
}
fn validate_surface_filter(only: Option<&str>) -> Result<()> {
validate_surface_filter_against(only, reach::surfaces())
}
fn surface_selected(surface: &reach::Surface, only: Option<&str>) -> bool {
only.is_none_or(|want| surface.name.eq_ignore_ascii_case(want))
}
async fn run_inventory(
client: &mut Client,
only: Option<&str>,
require_outcomes: bool,
checks_paths: Option<&[std::path::PathBuf]>,
assertion_values: Option<&HashMap<String, String>>,
) -> Result<usize> {
validate_surface_filter(only)?;
#[derive(serde::Serialize)]
struct SurfaceRow {
surface: String,
opened: bool,
#[serde(skip_serializing_if = "Option::is_none")]
error: Option<String>,
components: usize,
reachable: usize,
unreachable: usize,
state_hidden: usize,
anonymous: usize,
missing_id: usize,
unstable_id: usize,
duplicate_id: usize,
disabled: usize,
manual: usize,
isolated: usize,
outcome_declared: usize,
unverified: usize,
sections_opened: usize,
rows_hovered: usize,
}
#[derive(serde::Serialize)]
struct ControlRow {
surface: String,
id: u64,
dom_id: Option<String>,
slot: Option<String>,
role: String,
name: String,
classification: String,
reason: String,
checks: Vec<String>,
}
#[derive(serde::Serialize)]
struct RoleRow {
role: String,
components: usize,
reachable: usize,
unreachable: usize,
state_hidden: usize,
anonymous: usize,
missing_id: usize,
unstable_id: usize,
duplicate_id: usize,
disabled: usize,
manual: usize,
isolated: usize,
outcome_declared: usize,
unverified: usize,
}
#[derive(serde::Serialize)]
struct InventoryReport {
components: usize,
reachable: usize,
unreachable: usize,
state_hidden: usize,
anonymous: usize,
missing_id: usize,
unstable_id: usize,
duplicate_id: usize,
disabled: usize,
manual: usize,
isolated: usize,
outcome_declared: usize,
unverified: usize,
surfaces: Vec<SurfaceRow>,
roles: Vec<RoleRow>,
controls: Vec<ControlRow>,
}
let default_checks_exist = qa::default_checks_path().is_dir();
let mut checks = if let Some(paths) = checks_paths {
qa::checks_from_paths(paths).map_err(eyre::Report::msg)?
} else if default_checks_exist || require_outcomes {
qa::checks(None).map_err(eyre::Report::msg)?
} else {
eprintln!(
"warning: no --checks directory and {} does not exist; outcome coverage will be reported as zero",
qa::default_checks_path().display()
);
Vec::new()
};
if let Some(values) = assertion_values {
resolve_assertion_subjects(&mut checks, values).map_err(eyre::Report::msg)?;
}
let mut rows = Vec::new();
let mut controls = Vec::new();
let mut surface_failures = 0_usize;
let mut role_counts: std::collections::BTreeMap<String, [usize; 13]> =
std::collections::BTreeMap::new();
for surface in reach::surfaces() {
if !surface_selected(surface, only) {
continue;
}
if !open_surface(client, surface).await? {
rows.push(SurfaceRow {
surface: surface.name.clone(),
opened: false,
error: None,
components: 0,
reachable: 0,
unreachable: 0,
state_hidden: 0,
anonymous: 0,
missing_id: 0,
unstable_id: 0,
duplicate_id: 0,
disabled: 0,
manual: 0,
isolated: 0,
outcome_declared: 0,
unverified: 0,
sections_opened: 0,
rows_hovered: 0,
});
continue;
}
let sections_opened = expand_everything(client, surface).await?;
if !open_surface(client, surface).await? {
rows.push(SurfaceRow {
surface: surface.name.clone(),
opened: false,
error: None,
components: 0,
reachable: 0,
unreachable: 0,
state_hidden: 0,
anonymous: 0,
missing_id: 0,
unstable_id: 0,
duplicate_id: 0,
disabled: 0,
manual: 0,
isolated: 0,
outcome_declared: 0,
unverified: 0,
sections_opened,
rows_hovered: 0,
});
continue;
}
let held_filter = if let Some(field) = surface.reveal_with.as_deref() {
let (tree, _) = inspect(client).await?;
let previous = tree
.nodes
.iter()
.find(|node| selector_matches_node(node, field))
.and_then(|node| node.value.clone())
.unwrap_or_default();
if !previous.is_empty() {
type_text(client, field, "").await?;
}
Some((field.to_owned(), previous))
} else {
None
};
let inspected = async {
materialize_scrolled_content(client, surface).await?;
materialize_paginated_content(client, surface).await?;
let rows_hovered = hover_all_rows(client).await?;
let (tree, _) = inspect(client).await?;
Ok::<_, eyre::Report>((rows_hovered, tree))
}
.await;
if let Some((field, previous)) = &held_filter
&& !previous.is_empty()
{
type_text(client, field, previous).await?;
}
let (rows_hovered, tree) = match inspected {
Ok(inspected) => inspected,
Err(error) => {
rows.push(SurfaceRow {
surface: surface.name.clone(),
opened: true,
error: Some(error.to_string()),
components: 0,
reachable: 0,
unreachable: 0,
state_hidden: 0,
anonymous: 0,
missing_id: 0,
unstable_id: 0,
duplicate_id: 0,
disabled: 0,
manual: 0,
isolated: 0,
outcome_declared: 0,
unverified: 0,
sections_opened,
rows_hovered: 0,
});
surface_failures += 1;
continue;
}
};
let mine: std::collections::HashSet<u64> =
reach::inventory_surface_subtree(&tree.nodes, surface)
.map_err(eyre::Report::msg)?
.into_iter()
.collect();
let components: Vec<_> = tree
.nodes
.iter()
.filter(|node| reach::interactive(node) && mine.contains(&node.id))
.collect();
let duplicate_ids =
duplicate_dom_ids(tree.nodes.iter().filter(|node| mine.contains(&node.id)));
let classes: Vec<_> = components
.iter()
.map(|node| {
inventory_class(
node,
reach::requires_manual_release_check(&node.name),
reach::requires_isolated_outcome(&node.name),
&duplicate_ids,
)
})
.collect();
let declared: Vec<Vec<String>> = components
.iter()
.map(|node| outcome_check_ids(node, &checks))
.collect();
let count = |class| classes.iter().filter(|found| **found == class).count();
let reachable = count(InventoryClass::Reachable);
let unreachable = classes
.iter()
.zip(&declared)
.filter(|(class, matches)| **class == InventoryClass::Unreachable && matches.is_empty())
.count();
let state_hidden = classes
.iter()
.zip(&declared)
.filter(|(class, matches)| {
**class == InventoryClass::Unreachable && !matches.is_empty()
})
.count();
let anonymous = count(InventoryClass::Anonymous);
let missing_id = count(InventoryClass::MissingId);
let unstable_id = count(InventoryClass::UnstableId);
let duplicate_id = count(InventoryClass::DuplicateId);
let disabled = count(InventoryClass::Disabled);
let manual = count(InventoryClass::Manual);
let isolated = count(InventoryClass::Isolated);
let outcome_declared = classes
.iter()
.zip(&declared)
.filter(|(class, matches)| inventory_outcome_declared(**class, matches))
.count();
let unverified = classes
.iter()
.zip(&declared)
.filter(|(class, matches)| inventory_control_unverified(**class, matches))
.count();
for (node, matched_checks) in components.iter().zip(&declared) {
let manual = reach::requires_manual_release_check(&node.name);
let isolated = reach::requires_isolated_outcome(&node.name);
let class = inventory_class(node, manual, isolated, &duplicate_ids);
let counts = role_counts.entry(node.role.clone()).or_default();
counts[0] += 1;
match class {
InventoryClass::Reachable => counts[1] += 1,
InventoryClass::Unreachable if matched_checks.is_empty() => counts[2] += 1,
InventoryClass::Unreachable => counts[12] += 1,
InventoryClass::Anonymous => counts[3] += 1,
InventoryClass::MissingId => counts[4] += 1,
InventoryClass::UnstableId => counts[5] += 1,
InventoryClass::DuplicateId => counts[6] += 1,
InventoryClass::Disabled => counts[7] += 1,
InventoryClass::Manual => counts[8] += 1,
InventoryClass::Isolated => counts[9] += 1,
}
if !matched_checks.is_empty() {
counts[10] += 1;
}
let is_unverified = inventory_control_unverified(class, matched_checks);
if is_unverified {
counts[11] += 1;
}
let (classification, reason) = match class {
InventoryClass::MissingId => ("failed-missing-id", "no stable DOM id"),
InventoryClass::UnstableId => (
"failed-unstable-id",
"framework-generated creation-order DOM id",
),
InventoryClass::DuplicateId => ("failed-duplicate-id", "DOM id is not unique"),
InventoryClass::Manual => ("excluded-manual", "native-dialog-or-external"),
InventoryClass::Isolated => (
"isolated-unverified",
"requires disposable-process outcome check",
),
InventoryClass::Anonymous => ("failed-anonymous", "no accessible name"),
InventoryClass::Unreachable if !matched_checks.is_empty() => (
"outcome-declared-hidden",
"matched named reveal/outcome check",
),
InventoryClass::Unreachable if !node.visible => ("failed-reachability", "hidden"),
InventoryClass::Unreachable => ("failed-reachability", "no-box"),
InventoryClass::Disabled if matched_checks.is_empty() => {
("state-disabled-unverified", "no outcome check matched")
}
InventoryClass::Disabled => {
("outcome-declared-disabled", "matched named outcome check")
}
InventoryClass::Reachable if matched_checks.is_empty() => {
("reachable-unverified", "no outcome check matched")
}
InventoryClass::Reachable => ("outcome-declared", "matched named outcome check"),
};
controls.push(ControlRow {
surface: surface.name.clone(),
id: node.id,
dom_id: node.dom_id.clone(),
slot: node.slot.clone(),
role: node.role.clone(),
name: node.name.clone(),
classification: classification.to_owned(),
reason: reason.to_owned(),
checks: matched_checks.clone(),
});
}
rows.push(SurfaceRow {
surface: surface.name.clone(),
opened: true,
error: None,
components: components.len(),
reachable,
unreachable,
state_hidden,
anonymous,
missing_id,
unstable_id,
duplicate_id,
disabled,
manual,
isolated,
outcome_declared,
unverified,
sections_opened,
rows_hovered,
});
}
let report = InventoryReport {
components: rows.iter().map(|row| row.components).sum(),
reachable: rows.iter().map(|row| row.reachable).sum(),
unreachable: rows.iter().map(|row| row.unreachable).sum(),
state_hidden: rows.iter().map(|row| row.state_hidden).sum(),
anonymous: rows.iter().map(|row| row.anonymous).sum(),
missing_id: rows.iter().map(|row| row.missing_id).sum(),
unstable_id: rows.iter().map(|row| row.unstable_id).sum(),
duplicate_id: rows.iter().map(|row| row.duplicate_id).sum(),
disabled: rows.iter().map(|row| row.disabled).sum(),
manual: rows.iter().map(|row| row.manual).sum(),
isolated: rows.iter().map(|row| row.isolated).sum(),
outcome_declared: rows.iter().map(|row| row.outcome_declared).sum(),
unverified: rows.iter().map(|row| row.unverified).sum(),
surfaces: rows,
roles: role_counts
.into_iter()
.map(|(role, counts)| RoleRow {
role,
components: counts[0],
reachable: counts[1],
unreachable: counts[2],
state_hidden: counts[12],
anonymous: counts[3],
missing_id: counts[4],
unstable_id: counts[5],
duplicate_id: counts[6],
disabled: counts[7],
manual: counts[8],
isolated: counts[9],
outcome_declared: counts[10],
unverified: counts[11],
})
.collect(),
controls,
};
let failures = report.unreachable
+ report.anonymous
+ report.missing_id
+ report.unstable_id
+ report.duplicate_id
+ inventory_outcome_failures(report.unverified, report.isolated, require_outcomes)
+ surface_failures;
println!(
"{}",
toon_format::encode_default(&report).map_err(|error| eyre!(error.to_string()))?
);
Ok(failures)
}
async fn click_surface_path(
client: &mut Client,
surface: &reach::Surface,
opener: &str,
) -> Result<()> {
for step in surface.via.iter().map(String::as_str).chain([opener]) {
click_named_quiet(client, step).await?;
}
Ok(())
}
async fn open_surface(client: &mut Client, surface: &reach::Surface) -> Result<bool> {
if surface.opener.is_empty() {
return Ok(true);
}
let (current, _) = inspect(client).await?;
if reach::on_surface(¤t.nodes, surface) {
return Ok(true);
}
let opener = if surface.opener == reach::DYNAMIC_DOCUMENT {
let (here, _) = inspect(client).await?;
if reach::document_opener(&here.nodes).is_none() {
if let Some(home) = reach::profile().home_opener.as_deref() {
let _ = click_named_quiet(client, home).await;
}
let _ = wait_for_semantic_condition(client, check_timeout(900), |nodes| {
reach::document_opener(nodes).is_some()
})
.await?;
}
let (tree, _) = inspect(client).await?;
match reach::document_opener(&tree.nodes) {
Some(name) => name,
None => return Ok(false),
}
} else {
surface.opener.to_owned()
};
if surface.opener == reach::DYNAMIC_DOCUMENT {
let (tree, _) = inspect(client).await?;
let Some(id) = tree
.nodes
.iter()
.find(|n| n.name == opener && reach::onscreen(n))
.map(|n| n.id)
else {
return Ok(false);
};
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id: id,
}))
.await?;
let (tree, _) = inspect(client).await?;
if !tree.nodes.iter().any(|n| n.id == id && reach::onscreen(n)) {
return Ok(false);
}
client
.agent(&AgentControlRequest::Act(AgentAction::DoubleClick {
node_id: id,
}))
.await?;
} else if click_surface_path(client, surface, &opener).await.is_err() {
return Ok(false);
}
if settle_on(client, surface).await? {
return Ok(true);
}
if surface.opener == reach::DYNAMIC_DOCUMENT {
return Ok(false);
}
if let Some(home) = reach::profile().home_opener.as_deref() {
let _ = click_named_quiet(client, home).await;
}
if click_surface_path(client, surface, &opener).await.is_err() {
return Ok(false);
}
settle_on(client, surface).await
}
async fn sweep_modal(
client: &mut Client,
opener: &str,
here: &mut reach::Coverage,
failures: &mut Vec<(String, String, String)>,
surface: &str,
) -> Result<bool> {
let (tree, _) = inspect(client).await?;
let dismiss_ids: Vec<u64> = reach::dismissers(&tree.nodes)
.into_iter()
.map(|(id, _)| id)
.collect();
let scope = dismiss_ids
.first()
.map(|id| reach::enclosing_dialog(&tree.nodes, *id))
.unwrap_or_default();
if cli::trace() {
for node in tree.nodes.iter().filter(|node| {
scope.contains(&node.id)
&& (reach::profile()
.deferred_controls
.iter()
.any(|name| node.name.eq_ignore_ascii_case(name))
|| reach::requires_isolated_outcome(&node.name))
}) {
println!(
" [modal] deferred to an outcome check: {:?}",
node.name
);
}
}
let inner: Vec<(u64, String)> = tree
.nodes
.iter()
.filter(|n| n.role == "button" && reach::onscreen(n) && n.enabled)
.filter(|n| !n.name.trim().is_empty())
.filter(|n| !dismiss_ids.contains(&n.id))
.filter(|n| scope.contains(&n.id))
.filter(|n| !reach::requires_manual_release_check(&n.name))
.filter(|n| !reach::requires_isolated_outcome(&n.name))
.filter(|n| {
!reach::profile()
.deferred_controls
.iter()
.any(|name| n.name.eq_ignore_ascii_case(name))
})
.map(|n| (n.id, n.name.clone()))
.collect();
for (id, name) in inner {
let (before, _) = inspect(client).await?;
if !before
.nodes
.iter()
.any(|n| n.id == id && reach::onscreen(n))
{
continue;
}
if click_by_id(client, id).await.is_err() {
continue;
}
here.revealed += 1;
if cli::trace() {
println!(" [modal] clicked: {name:?}");
}
let case = sweep::Case {
id,
name: name.clone(),
family: audit::family_of(&name),
expect: sweep::expectation_for(&name, reach::is_inert_control(&name)),
};
let after = settle_sweep_case(client, &case, &before.nodes).await?;
if let Some(why) = sweep::judge(&case, &before.nodes, &after.nodes) {
failures.push((
surface.to_owned(),
format!("{name} (in {opener} dialog)"),
why,
));
}
let (now, _) = inspect(client).await?;
if !reach::modal_open(&now.nodes) {
return Ok(false);
}
}
let (tree, _) = inspect(client).await?;
for (id, name) in reach::dismissers(&tree.nodes) {
if click_by_id(client, id).await.is_err() {
continue;
}
here.revealed += 1;
let after = wait_for_semantic_condition(client, check_timeout(900), |nodes| {
!reach::modal_open(nodes)
})
.await?;
if !reach::modal_open(&after.nodes) {
return Ok(false);
}
failures.push((
surface.to_owned(),
format!("{name} (in {opener} dialog)"),
"the dialog is still open; it did not dismiss".to_owned(),
));
}
let _ = press_key(client, "escape", 1, "", false).await;
let after = wait_for_semantic_condition(client, check_timeout(900), |nodes| {
!reach::modal_open(nodes)
})
.await?;
if !reach::modal_open(&after.nodes) {
return Ok(false);
}
failures.push((
surface.to_owned(),
format!("{opener} dialog"),
"TRAPPED: no dismiss control and no Escape closes it".to_owned(),
));
Ok(true)
}
async fn settle_on(client: &mut Client, surface: &reach::Surface) -> Result<bool> {
let tree = wait_for_semantic_condition(client, check_timeout(4_800), |nodes| {
reach::on_surface(nodes, surface)
})
.await?;
Ok(reach::on_surface(&tree.nodes, surface))
}
async fn run_cover(
client: &mut Client,
only: Option<&str>,
unmapped_only: bool,
checks_dir: Option<&std::path::Path>,
) -> Result<usize> {
validate_surface_filter(only)?;
let mut total = reach::Coverage::default();
let mut failures: Vec<(String, String, String)> = Vec::new();
let mut skipped_manual: Vec<String> = Vec::new();
let mut skipped_isolated: Vec<String> = Vec::new();
let checks = if unmapped_only {
qa::checks(checks_dir).map_err(eyre::Report::msg)?
} else {
Vec::new()
};
for surface in reach::surfaces() {
if !surface_selected(surface, only) {
continue;
}
if !open_surface(client, surface).await? {
println!("- {:<10} could not be opened, skipping\n", surface.name);
continue;
}
let opened = expand_everything(client, surface).await?;
if !open_surface(client, surface).await? {
println!("- {:<10} left during expansion, skipping\n", surface.name);
continue;
}
materialize_deferred_content(client, surface, "").await?;
if let Err(error) = materialize_paginated_content(client, surface).await {
failures.push((
surface.name.clone(),
"pagination".to_owned(),
error.to_string(),
));
continue;
}
let hovered = hover_all_rows(client).await?;
let (tree, _) = inspect(client).await?;
let mine: std::collections::HashSet<u64> = reach::on_surface_subtree(&tree.nodes, surface)
.into_iter()
.collect();
let buttons: Vec<&blitz_control_protocol::SemanticNode> = tree
.nodes
.iter()
.filter(|n| n.role == "button" && !n.name.trim().is_empty())
.filter(|n| n.visible)
.filter(|n| mine.contains(&n.id))
.collect();
let retained = tree
.nodes
.iter()
.filter(|n| n.role == "button" && !n.name.trim().is_empty() && !n.visible)
.count();
let mut here = reach::Coverage {
in_tree: buttons.len(),
..Default::default()
};
let mut plan: Vec<(u64, String)> = Vec::new();
let mut collapsers: Vec<(u64, String)> = Vec::new();
let mut closers: Vec<(u64, String)> = Vec::new();
for node in &buttons {
if !reach::onscreen(node) {
here.unreachable += 1;
} else if reach::requires_isolated_outcome(&node.name) {
here.isolated += 1;
skipped_isolated.push(node.name.clone());
} else if reach::requires_manual_release_check(&node.name) {
here.manual += 1;
skipped_manual.push(node.name.clone());
} else if unmapped_only && !outcome_check_ids(node, &checks).is_empty() {
here.outcome_declared += 1;
} else if reach::profile()
.fold_prefixes
.iter()
.any(|p| node.name.to_lowercase().starts_with(&p.to_lowercase()))
|| reach::profile()
.deferred_controls
.iter()
.any(|c| node.name.eq_ignore_ascii_case(c))
|| reach::folds_a_section(&node.name)
{
collapsers.push((node.id, node.name.clone()));
} else if reach::navigates(&node.name)
|| reach::opens_document_row(&tree.nodes, node.id)
{
here.navigation += 1;
} else if reach::closes_a_surface(&node.name) {
closers.push((node.id, node.name.clone()));
} else {
plan.push((node.id, node.name.clone()));
}
}
plan.sort_by_key(|(_, name)| {
let lower = name.to_lowercase();
u8::from(
lower.starts_with("delete ")
|| lower.starts_with("close ")
|| lower.starts_with("remove ")
|| lower.starts_with("retire "),
)
});
plan.extend(collapsers);
plan.extend(closers);
let mut done: HashMap<String, usize> = HashMap::new();
let planned_total = plan.len();
for (index, (planned_id, name)) in plan.into_iter().enumerate() {
let remaining = planned_total.saturating_sub(index + 1);
let (mut before, _) = inspect(client).await?;
if !reach::on_surface(&before.nodes, surface) {
if !open_surface(client, surface).await? {
here.vanished += 1;
if cli::trace() {
println!(" left surface, could not return: {name:?}");
}
continue;
}
let (fresh, _) = inspect(client).await?;
before = fresh;
}
let seen = done.entry(name.clone()).or_insert(0);
let found = before
.nodes
.iter()
.find(|n| n.id == planned_id && n.name == name && reach::onscreen(n))
.or_else(|| {
before
.nodes
.iter()
.filter(|n| n.role == "button" && n.name == name && reach::onscreen(n))
.nth(*seen)
});
let Some(node) = found else {
here.vanished += 1;
if cli::trace() {
println!(" vanished: {name:?} (id {planned_id})");
}
continue;
};
*seen += 1;
let id = node.id;
if !reach::onscreen(node) || !node.enabled {
here.vanished += 1;
if cli::trace() {
let visible_buttons = before
.nodes
.iter()
.filter(|n| n.role == "button" && reach::onscreen(n))
.count();
println!(
" offscreen/disabled: {name:?} visible={} bounds={:?} \
[on_surface={}, {visible_buttons} buttons on screen]",
node.visible,
node.bounds,
reach::on_surface(&before.nodes, surface),
);
}
continue;
}
let case = sweep::Case {
id,
name: name.clone(),
family: audit::family_of(&name),
expect: sweep::expectation_for(&name, reach::is_inert_control(&name)),
};
let activation = nagoya::timeout(check_timeout(900), click_by_id(client, id))
.await
.map_err(|_| {
eyre!(
"activation exceeded {}ms for {name:?} (id {id}) on {:?}",
check_timeout(900).as_millis(),
surface.name
)
})?;
if let Err(error) = activation {
bail!(
"could not activate {name:?} (id {id}) on {:?}; stopping the sweep: {error}",
surface.name
);
}
here.swept += 1;
if cli::trace() {
println!(" clicked: {name:?}");
}
let after_click = settle_sweep_case(client, &case, &before.nodes).await?;
let after = if reach::modal_open(&after_click.nodes) {
let trapped =
sweep_modal(client, &name, &mut here, &mut failures, &surface.name).await?;
if trapped {
println!(
" ! {:?} opened a dialog that will not dismiss - \
the rest of this surface is unreachable behind it",
name
);
here.blocked += remaining;
break;
}
inspect(client).await?.0
} else {
after_click
};
if let Some(why) = sweep::judge(&case, &before.nodes, &after.nodes) {
failures.push((surface.name.to_owned(), name, why));
}
}
println!(
"= {:<10} {} ({} sections opened, {} rows hovered, {} retained elsewhere)",
surface.name,
here.line(),
opened,
hovered,
retained
);
total.in_tree += here.in_tree;
total.swept += here.swept;
total.outcome_declared += here.outcome_declared;
total.unreachable += here.unreachable;
total.vanished += here.vanished;
total.navigation += here.navigation;
total.manual += here.manual;
total.isolated += here.isolated;
total.blocked += here.blocked;
total.revealed += here.revealed;
}
println!("\n{}", total.line());
if total.manual > 0 {
println!(
"\n{} control(s) need the manual release pass:",
total.manual
);
let mut seen: Vec<&str> = skipped_manual.iter().map(String::as_str).collect();
seen.sort_unstable();
seen.dedup();
for label in seen {
let command = reach::profile()
.manual_controls
.iter()
.find(|exception| label.starts_with(exception.label.as_str()))
.map(|exception| exception.command.as_str())
.unwrap_or("(unmapped manual control)");
println!(" {label:<38} {command}");
}
}
if total.isolated > 0 {
println!(
"\n{} control(s) require an isolated outcome check:",
total.isolated
);
skipped_isolated.sort_unstable();
skipped_isolated.dedup();
for label in skipped_isolated {
println!(" {label}");
}
}
if failures.is_empty() {
if total.isolated > 0 {
println!("every broadly swept button acted; isolated controls remain listed above");
} else {
println!("every reached button acted");
}
} else {
println!("\n{} did not act:\n", failures.len());
for (surface, name, why) in &failures {
println!(
" [{surface}] {:<40} {why}",
name.chars().take(40).collect::<String>()
);
}
}
Ok(failures.len())
}
fn preflight_plan(
plan: &plan::ResolvedPlan,
session_profile: Option<&std::path::Path>,
) -> Result<Option<std::path::PathBuf>> {
let load_suite = |suite: &plan::CheckSuite| -> Result<Vec<qa::Check>> {
let all = qa::checks_from_paths(&suite.checks).map_err(eyre::Report::msg)?;
let selected = ordered_checks(&all, Some(&suite.selector));
if selected.is_empty() {
bail!("run-plan selector {:?} matched no check", suite.selector);
}
Ok(selected.into_iter().cloned().collect())
};
let mut used_assertions = HashSet::new();
let validate_assertions =
|checks: &mut [qa::Check], used: &mut HashSet<String>| -> Result<()> {
used.extend(
resolve_assertion_subjects(checks, &plan.assertion_values)
.map_err(eyre::Report::msg)?,
);
Ok(())
};
for guest in &plan.guests {
app::AppProfile::load(Some(&guest.app_profile)).map_err(|error| eyre!(error))?;
if let Some(inventory) = &guest.inventory {
let mut checks = qa::checks_from_paths(&inventory.checks).map_err(eyre::Report::msg)?;
validate_assertions(&mut checks, &mut used_assertions)?;
if checks
.iter()
.any(|check| check.setup_secret.is_some() || check.subject_secret.is_some())
{
bail!("runtime credentials cannot be referenced by an inventory manifest");
}
}
}
if let Some(login) = &plan.login {
let mut login_checks = load_suite(login)?;
validate_assertions(&mut login_checks, &mut used_assertions)?;
let mut username_inputs = 0;
let mut password_inputs = 0;
for check in &login_checks {
match check.setup_secret {
Some(qa::SecretInput::Username) => username_inputs += 1,
Some(qa::SecretInput::Password) => password_inputs += 1,
None => {}
}
if check.setup_text.is_some() || check.setup_text_env.is_some() || check.text.is_some()
{
bail!("run-plan login checks must use runtime credential references");
}
}
if username_inputs != 1 || password_inputs != 1 {
bail!("run-plan login selector must establish one username and one password");
}
}
let validate_non_login_suites = |suites: &[plan::CheckSuite],
credentials: Option<&plan::SessionSecrets>,
used: &mut HashSet<String>|
-> Result<()> {
for suite in suites {
let mut checks = load_suite(suite)?;
validate_assertions(&mut checks, used)?;
for check in checks {
if check.setup_secret.is_some() {
bail!("runtime credentials may be used only by the run-plan login suite");
}
if check.subject_secret.is_some() && credentials.is_none() {
bail!("runtime credential subjects require a signed-in session");
}
if credentials.is_some_and(|credentials| {
let subject = check
.subject_secret
.is_none()
.then_some(check.subject.as_str());
[check.setup_text.as_deref(), check.text.as_deref(), subject]
.into_iter()
.flatten()
.chain(
check
.any_of
.iter()
.map(|alternative| alternative.subject.as_str()),
)
.any(|literal| {
literal == credentials.value(qa::SecretInput::Username)
|| literal == credentials.value(qa::SecretInput::Password)
})
}) {
bail!("a check contains a literal matching a runtime credential");
}
}
}
Ok(())
};
for guest in &plan.guests {
validate_non_login_suites(&guest.suites, None, &mut used_assertions)?;
}
for session in &plan.sessions {
validate_non_login_suites(
&session.suites,
session.credentials.as_ref(),
&mut used_assertions,
)?;
}
if used_assertions.len() != plan.assertion_values.len() {
return Err(eyre!(
"run plan declares an unused assertion environment binding"
));
}
let session_profile = if plan.sessions.is_empty() {
None
} else {
let path = session_profile.ok_or_else(|| {
eyre!("run plan with signed-in sessions requires the command-line --app profile")
})?;
app::AppProfile::load(Some(path)).map_err(|error| eyre!(error))?;
Some(
path.canonicalize()
.map_err(|_| eyre!("could not resolve --app profile"))?,
)
};
Ok(session_profile)
}
async fn run_plan_host(
plan: &plan::ResolvedPlan,
app_profile: &std::path::Path,
suites: &[plan::CheckSuite],
inventory: Option<&plan::InventoryPlan>,
login: Option<(&plan::CheckSuite, &str, &plan::SessionSecrets)>,
) -> Result<usize> {
cli::set_app_profile(Some(app_profile.to_path_buf()));
let mut hosted = HostedSession::start(
&plan.host,
std::path::Path::new(&plan.page),
Duration::from_secs(plan.startup_timeout_secs),
true,
true,
)
.map_err(eyre::Report::msg)?;
let descriptor = inspector::discover(hosted.descriptor_path.to_str())?;
let mut client = Client::connect(&descriptor.socket_path()).await?;
client.initialize().await?;
let mut failed = 0;
if let Some((login_suite, landing, credentials)) = login {
let login_failed = run_qa_with_plan_values(
&mut client,
Some(&login_suite.selector),
Some(&login_suite.checks),
Some(credentials),
Some(&plan.assertion_values),
Some(&mut hosted),
)
.await?;
failed += login_failed;
if login_failed > 0 {
println!("SKIP signed-in suites: login checks failed");
drop(hosted);
return Ok(failed);
}
let arrived = wait_for_arrival(&mut client, None, landing, Duration::from_secs(15))
.await
.map_err(|error| eyre!(credentials.redact(&error.to_string())))?;
if !arrived {
println!("FAIL signed-in session: expected landing marker did not paint");
drop(hosted);
return Ok(failed + 1);
}
}
if let Some(inventory) = inventory {
failed += run_inventory(
&mut client,
None,
inventory.require_outcomes,
Some(&inventory.checks),
Some(&plan.assertion_values),
)
.await?;
} else {
for suite in suites {
let credentials = login.map(|(_, _, credentials)| credentials);
failed += run_qa_with_plan_values(
&mut client,
Some(&suite.selector),
Some(&suite.checks),
credentials,
Some(&plan.assertion_values),
Some(&mut hosted),
)
.await?;
}
}
drop(hosted);
Ok(failed)
}
async fn run_plan(
path: &std::path::Path,
cli_app_profile: Option<&std::path::Path>,
) -> Result<usize> {
let parsed = plan::RunPlan::load(path).map_err(eyre::Report::msg)?;
let resolved = parsed.resolve().map_err(eyre::Report::msg)?;
let session_profile = preflight_plan(&resolved, cli_app_profile)?;
let mut failed = 0;
for guest in &resolved.guests {
println!("== {} invocation ==", guest.id);
match run_plan_host(
&resolved,
&guest.app_profile,
&guest.suites,
guest.inventory.as_ref(),
None,
)
.await
{
Ok(session_failed) => failed += session_failed,
Err(error) => {
eprintln!("FAIL {} invocation: {error}", guest.id);
failed += 1;
}
}
}
for session in &resolved.sessions {
if let Some(reason) = session.skip_reason {
println!("SKIP {}: {}", session.id, reason);
continue;
}
let Some(credentials) = session.credentials.as_ref() else {
eprintln!("FAIL {} session has no configured credentials", session.id);
failed += 1;
continue;
};
println!("== {} session ==", session.id);
let login = resolved
.login
.as_ref()
.ok_or_else(|| eyre!("run plan with signed-in sessions must declare a login suite"))?;
match run_plan_host(
&resolved,
session_profile
.as_deref()
.ok_or_else(|| eyre!("signed-in session profile was not preflighted"))?,
&session.suites,
None,
Some((login, &session.landing, credentials)),
)
.await
{
Ok(session_failed) => failed += session_failed,
Err(error) => {
eprintln!(
"FAIL {} session: {}",
session.id,
credentials.redact(&error.to_string())
);
failed += 1;
}
}
}
Ok(failed)
}
pub async fn run() -> Result<()> {
let cli = <cli::Cli as clap::Parser>::parse();
cli::set_trace(cli.trace);
cli::set_headless(cli.headless);
cli::set_pace(cli.pace);
cli::set_timeout_scale(cli.timeout_scale);
cli::set_capture_options(
cli.trace_capture,
cli.require_paint_events,
cli.pixel_artifact_dir.clone(),
);
cli::set_app_profile(cli.app.clone());
if let cli::Command::List { checks } = &cli.command {
print!(
"{}",
qa::manifest(checks.as_deref()).map_err(|error| eyre!(error))?
);
return Ok(());
}
if let cli::Command::Reconcile { inventory, checks } = &cli.command {
let failures = reconcile_inventory(inventory, checks.as_deref())?;
if failures > 0 {
std::process::exit(1);
}
return Ok(());
}
if cli.command.requires_app_profile() && !matches!(&cli.command, cli::Command::RunPlan { .. }) {
app::AppProfile::load(cli.app.as_deref()).map_err(|error| eyre!(error))?;
}
if let cli::Command::RunPlan { plan } = &cli.command {
if cli.trace
|| cli.trace_capture
|| cli.require_paint_events
|| cli.pixel_artifact_dir.is_some()
{
bail!("`run-plan` does not accept tracing or pixel capture options");
}
if cli.descriptor.is_some() {
bail!("`run-plan` starts its own host and does not accept `--descriptor`");
}
let failures = run_plan(plan, cli.app.as_deref()).await?;
if failures > 0 {
std::process::exit(1);
}
return Ok(());
}
if let cli::Command::SweepComponents {
ids,
host,
dists,
checks,
startup_timeout,
mode,
} = &cli.command
{
let failures = sweep_components(
ids,
host,
dists,
checks.as_deref(),
std::time::Duration::from_secs(*startup_timeout),
*mode,
)
.await?;
if failures > 0 {
std::process::exit(1);
}
return Ok(());
}
if let cli::Command::QaHosted {
selector,
host,
page,
checks,
startup_timeout,
} = &cli.command
{
note_qa_hosted_run();
let mut hosted = HostedSession::start(
host,
page,
std::time::Duration::from_secs(*startup_timeout),
false,
false,
)
.map_err(|error| eyre!(error))?;
hosted.seeds_storage = true;
let descriptor_path = hosted.descriptor_path.clone();
let result = run_component(
&descriptor_path,
selector.as_deref(),
checks.as_deref(),
Some(&mut hosted),
)
.await;
drop(hosted);
let failures = result?;
if failures > 0 {
std::process::exit(1);
}
return Ok(());
}
let descriptor = inspector::discover(cli.descriptor.as_deref().and_then(|p| p.to_str()))?;
descriptor.warn_if_stale();
let verbose = cli.command.is_dump();
if verbose {
println!("descriptor: {}", descriptor.path.display());
println!("{}", report::dump(&descriptor.raw, usize::MAX));
}
let mut client = Client::connect(&descriptor.socket_path()).await?;
let initialize = client.initialize().await?;
if verbose {
println!("\n== initialize ==");
println!("{}", report::dump(&initialize, 800));
println!("\n== tools ==");
let tools = client.tools_list().await?;
println!("{}", report::dump(&tools, 1200));
}
match cli.command {
cli::Command::Gpu => {
let report = gpu_report(&mut client).await?;
println!(
"{}",
toon_format::encode_default(&report).map_err(|error| eyre!(error.to_string()))?
);
}
cli::Command::Metrics => {
println!("\n== metrics ==");
let envelope = client
.diagnostics_envelope(&DiagnosticsRequest::Metrics)
.await?;
println!("{}", report::dump(&envelope, 2000));
}
cli::Command::Watch { seconds } => {
println!("\n== observing paint/metrics/console/runtimeErrors for {seconds}s ==");
let answer = client
.diagnostics_envelope(&DiagnosticsRequest::Observe {
streams: vec![
DebugStream::Paint,
DebugStream::Metrics,
DebugStream::Console,
DebugStream::RuntimeErrors,
],
})
.await?;
println!("{}", report::dump(&answer, 400));
for message in client.drain(seconds).await? {
println!("{}", report::dump(&message, 400));
}
}
cli::Command::Frames => report::show_frames(&metrics(&mut client).await?),
cli::Command::Tree => {
println!("\n== semantic tree ==");
let envelope = client
.agent_envelope(&AgentControlRequest::Inspect {
root: None,
max_depth: 3,
})
.await?;
println!("{}", report::dump(&envelope, 3000));
}
cli::Command::Find {
pattern,
role,
visible,
hidden,
painted,
offscreen,
disabled,
count,
limit,
} => {
find(
&mut client,
&pattern,
&role,
visible,
hidden,
painted,
offscreen,
disabled,
count,
limit,
)
.await?;
}
cli::Command::Layout { name } => {
layout(&mut client, &name).await?;
}
cli::Command::Transcript => transcript(&mut client).await?,
cli::Command::Paint { name, min_area } => {
paint_audit::report(&mut client, &name, min_area).await?;
}
cli::Command::Contrast {
name,
text_ratio,
control_ratio,
} => {
paint_audit::contrast(&mut client, &name, text_ratio, control_ratio).await?;
}
cli::Command::Nodes => {
nodes(&mut client).await?;
}
cli::Command::Panes => panes(&mut client).await?,
cli::Command::Dom { name, depth } => {
dom(&mut client, &name, depth).await?;
}
cli::Command::Spill { axis, tolerance } => {
spill(&mut client, &axis, tolerance).await?;
}
cli::Command::Idle => report::show("idle", &metrics(&mut client).await?),
cli::Command::Drift { seconds } => {
let before = metrics(&mut client).await?;
println!("== holding still for {seconds}s, nothing driven ==");
nagoya::sleep(Duration::from_secs_f64(seconds)).await;
let after = metrics(&mut client).await?;
let frames_of =
|m: &RendererMetrics| m.frame_window.as_ref().map(|w| w.frames_total).unwrap_or(0);
let frames = frames_of(&after).saturating_sub(frames_of(&before));
println!(
"frames={frames} over {seconds}s = {:.1}fps with no input",
frames as f64 / seconds
);
report::show_delta(&before, &after, 0);
}
cli::Command::Ghost { min_area, max } => {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: true,
include_layout: true,
include_computed_style: false,
node_ids: Vec::new(),
}))
.await?;
let DebugResponse::Snapshot(snapshot) = answer.response else {
bail!("asked for a layout snapshot, got {:?}", answer.response);
};
let mut boxes: HashMap<u64, (f64, f64, f64, f64)> = HashMap::new();
if let Some(rows) = snapshot.layout {
for row in rows {
boxes.insert(
row.node_id,
(
row.bounds.x,
row.bounds.y,
row.bounds.width,
row.bounds.height,
),
);
}
}
let nodes = snapshot
.dom
.as_ref()
.and_then(|v| v.as_array())
.cloned()
.unwrap_or_default();
let mut ghosts = Vec::new();
for node in &nodes {
let visible = node
.get("visible")
.and_then(|v| v.as_bool())
.unwrap_or(true);
if visible {
continue;
}
let Some(id) = node.get("id").and_then(|v| v.as_u64()) else {
continue;
};
let Some(&(x, y, w, h)) = boxes.get(&id) else {
continue;
};
if w * h >= min_area {
let name = node
.get("name")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
ghosts.push((id, name, x, y, w, h));
}
}
if !ghosts.is_empty() {
let answer = client
.diagnostics(&DiagnosticsRequest::Snapshot(SnapshotRequest {
include_dom: false,
include_layout: false,
include_computed_style: true,
node_ids: ghosts.iter().map(|ghost| ghost.0).collect(),
}))
.await?;
let DebugResponse::Snapshot(styled) = answer.response else {
bail!("asked for computed styles, got {:?}", answer.response);
};
let painted: HashSet<u64> = styled
.computed_style
.as_ref()
.and_then(|v| v.as_array())
.into_iter()
.flatten()
.filter(|row| row.get("visibility").and_then(|v| v.as_str()) == Some("Visible"))
.filter_map(|row| row.get("nodeId").and_then(|v| v.as_u64()))
.collect();
ghosts.retain(|ghost| !painted.contains(&ghost.0));
}
ghosts.sort_by(|a, b| (b.4 * b.5).total_cmp(&(a.4 * a.5)));
println!(
"{} nodes inspected, reporting hidden boxes of {min_area}px2 or more",
nodes.len()
);
for (id, name, x, y, w, h) in &ghosts {
println!(" {id:>10} {w:>7.1}x{h:<7.1} at {x:.0},{y:.0} {name}");
}
if ghosts.is_empty() {
println!("\nno ghosts: nothing hidden is holding a painted box");
} else {
println!(
"\nGHOSTS PRESENT: {} hidden node(s) still occupy layout",
ghosts.len()
);
if ghosts.len() > max {
println!(
"over budget: {} ghosts, limit {max}. Hidden subtrees are not \
being unmounted.",
ghosts.len()
);
std::process::exit(1);
}
println!("within budget: limit {max}");
}
}
cli::Command::Blink { allowed_missed } => {
let reading = metrics(&mut client).await?;
report::show("blink", &reading);
let Some(window) = reading.frame_window.as_ref() else {
eyre::bail!(
"the app published no frame window, so there is nothing to assert; \
launch it with --blitz-deep-profiling"
);
};
let period_ms = 1000.0
/ window
.display_refresh_hz
.filter(|hz| *hz > 0.0)
.unwrap_or(60.0);
let interval_budget = period_ms * 2.0;
let worst_interval = window.interval.max_ms;
println!();
println!("== the reported repro: a project with 0 items, item list expanded ==");
println!(
" missed refreshes : {} over {} frames (allowed {allowed_missed})",
window.missed_refreshes, window.window_frames
);
println!(
" worst interval : {worst_interval:.1}ms against a {period_ms:.1}ms refresh \
(budget {interval_budget:.1}ms)"
);
let mut faults = Vec::new();
if window.missed_refreshes > allowed_missed {
faults.push(format!(
"{} missed refreshes over {} frames",
window.missed_refreshes, window.window_frames
));
}
if worst_interval > interval_budget {
faults.push(format!(
"a {worst_interval:.1}ms frame interval, {:.1}x the refresh period",
worst_interval / period_ms
));
}
if faults.is_empty() {
println!("\nno blink: the window is quiet by both measures");
} else {
println!("\nBLINK PRESENT: {}", faults.join(", "));
std::process::exit(1);
}
}
cli::Command::Click { name, id } => {
nodes(&mut client).await?;
match (name, id) {
(Some(name), None) => click_named(&mut client, &name).await?,
(None, Some(node_id)) => {
click_by_id(&mut client, node_id).await?;
println!("activated node {node_id}");
}
_ => unreachable!("clap requires exactly one click selector"),
}
}
cli::Command::Capture {
name,
scale,
output,
} => {
capture(&mut client, &name, scale as f32, output.as_deref()).await?;
}
cli::Command::GpuCapture { target, output } => {
gpu_capture_command(&mut client, &target, output.as_deref()).await?;
}
cli::Command::Audit { family } => {
let faults = run_audit(&mut client, family.as_deref()).await?;
if faults > 0 {
std::process::exit(1);
}
}
cli::Command::Sweep { family } => {
let failures = run_sweep(&mut client, family.as_deref()).await?;
if failures > 0 {
std::process::exit(1);
}
}
cli::Command::Press { name } => {
let (snapshot, _) = inspect(&mut client).await?;
let (role, bare) = name.split_once(':').unwrap_or(("", name.as_str()));
let wanted = bare.to_lowercase();
let Some(node) = snapshot
.nodes
.iter()
.filter(|n| role.is_empty() || n.role.eq_ignore_ascii_case(role))
.filter(|n| n.name.to_lowercase().contains(&wanted))
.filter(|n| n.visible && n.enabled)
.find(|n| n.bounds.is_some_and(|b| b[2] > 0.0 && b[3] > 0.0))
else {
bail!("no visible, enabled, sized node matching {name:?}");
};
let b = node.bounds.unwrap();
let (x, y) = (b[0] + b[2] / 2.0, b[1] + b[3] / 2.0);
println!("pressing {:?} at {x:.0},{y:.0}", node.name);
for phase in [PointerPhase::Move, PointerPhase::Down, PointerPhase::Up] {
client
.agent(&AgentControlRequest::Act(AgentAction::Input(
InputCommand::Pointer {
phase,
x,
y,
button: 0,
modifiers: Modifiers::default(),
},
)))
.await?;
}
println!("pressed");
}
cli::Command::Cover {
surface,
unmapped_only,
checks,
max_seconds,
} => {
let result = nagoya::timeout(
Duration::from_secs(max_seconds),
run_cover(
&mut client,
surface.as_deref(),
unmapped_only,
checks.as_deref(),
),
)
.await
.map_err(|_| eyre!("coverage sweep exceeded {max_seconds}s"))?;
let failures = result?;
if failures > 0 {
std::process::exit(1);
}
}
cli::Command::Inventory {
surface,
require_outcomes,
checks,
} => {
let checks_paths = checks.as_ref().map(std::slice::from_ref);
let failures = run_inventory(
&mut client,
surface.as_deref(),
require_outcomes,
checks_paths,
None,
)
.await?;
if failures > 0 {
std::process::exit(1);
}
}
cli::Command::Qa { selector, checks } => {
let failed = run_qa(&mut client, selector.as_deref(), checks.as_deref(), None).await?;
if failed > 0 {
std::process::exit(1);
}
}
cli::Command::Reveal { name } => {
let (snapshot, _) = inspect(&mut client).await?;
let Some(target) = snapshot
.nodes
.iter()
.find(|node| {
(selector_matches_node(node, &name) || node.name.contains(&name))
&& node.bounds.is_some()
})
else {
bail!("no node named {name:?}");
};
let before = target.bounds.unwrap();
let id = target.id;
println!("{id} {:?} y={:.1}", target.role, before[1]);
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollIntoView {
node_id: id,
}))
.await?;
let (after_snapshot, _) = inspect(&mut client).await?;
let after = after_snapshot
.nodes
.iter()
.find(|node| node.id == id)
.and_then(|node| node.bounds);
match after {
Some(b) => println!("after: y={:.1} (moved {:.1})", b[1], b[1] - before[1]),
None => println!("after: the node is gone from the tree"),
}
}
cli::Command::Key { name, count, over } => {
let fallback = if over.is_empty() {
app::AppProfile::load(None)
.ok()
.and_then(|profile| profile.transcript_region)
.unwrap_or_default()
} else {
String::new()
};
let over = if over.is_empty() { &fallback } else { &over };
press_key(&mut client, &name, count as usize, over, false).await?;
}
cli::Command::Type { count, name } => {
nodes(&mut client).await?;
type_keys(&mut client, count as usize, &name).await?;
}
cli::Command::PointerDrag {
name,
dx,
dy,
steps,
cancel,
} => {
pointer_drag(&mut client, &name, dx, dy, steps, cancel, None).await?;
}
cli::Command::Drag { name, dy, steps } => {
let fallback = reach::profile()
.transcript_region
.clone()
.unwrap_or_default();
let name = if name.is_empty() { &fallback } else { &name };
let times = steps as usize;
let (snapshot, _) = inspect(&mut client).await?;
let Some(target) = snapshot
.nodes
.iter()
.filter(|node| node.visible && node.name.contains(name))
.filter_map(|node| node.bounds.map(|b| (node, b)))
.max_by(|a, b| {
(a.1[2] * a.1[3])
.partial_cmp(&(b.1[2] * b.1[3]))
.unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(node, _)| node.id)
else {
bail!("no visible node named {name:?}");
};
println!("scrolling node {target} by {dy} x{times}");
for _ in 0..times {
client
.agent(&AgentControlRequest::Act(AgentAction::ScrollBy {
node_id: target,
delta_x: 0.0,
delta_y: dy,
}))
.await?;
sleep_pace().await;
}
}
cli::Command::Scroll { ticks, delta, over } => {
let fallback = reach::profile()
.transcript_region
.clone()
.unwrap_or_default();
let over = if over.is_empty() { &fallback } else { &over };
let count = nodes(&mut client).await?;
hover_over(&mut client, over).await?;
report::show("before", &metrics(&mut client).await?);
scroll(&mut client, ticks as usize, delta).await?;
report::show("after", &metrics(&mut client).await?);
println!("tree size during run: {count} nodes");
}
cli::Command::QaHosted { .. }
| cli::Command::RunPlan { .. }
| cli::Command::SweepComponents { .. }
| cli::Command::List { .. }
| cli::Command::Reconcile { .. } => {
unreachable!("handled before the client connects")
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{
InventoryClass, OutcomeStability, accumulated_hover_signatures, arrival_sample_matches,
arrived_without_navigation, assess_pixel_change, away_target_in, capture_node_id,
declared_open_timeout, declared_outcome_timeout, duplicate_dom_ids, generated_dom_id,
hover_signature_counts, inventory_class, is_pagination_control, measure_ink, name_matches,
named_document_is_active, named_document_is_active_with_permanent,
named_document_opener_for, ordered_checks, outcome_check_ids, outcome_verdict,
pagination_advanced, painted_bounds, painted_named, pixels_change, pixels_hold,
require_transparent_window_tint, resolved_action_target, rgb_pixels_hold,
saved_control_node, saved_controls, selector_matches_node, settle_for_outcome,
setup_value_landed, stable_arrival, subject_belongs_to_scope,
validate_surface_filter_against,
};
use crate::app::{AppProfile, SurfaceSpec};
use crate::interaction::parse_key_chord;
use crate::qa::{Check, Expect, PointerDrag};
use crate::target::{
exact_selector_matches_node, offscreen, retain_exact_candidates, viewport_for_node,
};
use blitz_control_protocol::{AgentSnapshot, CapturedImage, SemanticNode, WindowComposition};
use std::collections::HashSet;
use std::time::Duration;
#[test]
fn outcome_stability_restarts_when_late_content_changes_the_document() {
let start = std::time::Instant::now();
let required = Duration::from_millis(150);
let mut stability = OutcomeStability::default();
assert!(!stability.observe(start, 7, true, required));
assert!(!stability.observe(start + Duration::from_millis(100), 7, true, required));
assert!(!stability.observe(start + Duration::from_millis(125), 8, true, required));
assert!(!stability.observe(start + Duration::from_millis(250), 8, true, required));
assert!(stability.observe(start + Duration::from_millis(275), 8, true, required));
}
#[test]
fn failed_outcome_clears_a_partial_stability_window() {
let start = std::time::Instant::now();
let required = Duration::from_millis(100);
let mut stability = OutcomeStability::default();
assert!(!stability.observe(start, 3, true, required));
assert!(!stability.observe(start + Duration::from_millis(75), 3, false, required));
assert!(!stability.observe(start + Duration::from_millis(100), 3, true, required));
assert!(stability.observe(start + Duration::from_millis(200), 3, true, required));
}
#[test]
fn outcome_stability_waits_only_for_the_unproven_part_of_its_window() {
let start = std::time::Instant::now();
let required = Duration::from_millis(100);
let mut stability = OutcomeStability::default();
assert!(!stability.observe(start, 3, true, required));
assert_eq!(stability.remaining(start, required), required);
assert_eq!(
stability.remaining(start + Duration::from_millis(75), required),
Duration::from_millis(25)
);
assert_eq!(
stability.remaining(start + Duration::from_millis(150), required),
Duration::ZERO
);
}
#[test]
fn a_new_surface_owns_its_declared_outcome_but_not_a_portal() {
let mut heading = component("Outcome per dollar", true, true);
heading.id = 20;
heading.role = "heading".into();
let mut dialog = component("Welcome", true, true);
dialog.id = 30;
dialog.role = "dialog".into();
let nodes = [heading, dialog];
let analytics = HashSet::from([20]);
assert!(subject_belongs_to_scope(
&nodes,
"heading:Outcome per dollar",
&analytics
));
assert!(!subject_belongs_to_scope(
&nodes,
"dialog:Welcome",
&analytics
));
assert!(!subject_belongs_to_scope(
&nodes,
"generic:Already vanished",
&analytics
));
}
fn component(name: &str, enabled: bool, visible: bool) -> SemanticNode {
SemanticNode {
dom_id: Some(if name.is_empty() {
"anonymous-component".into()
} else {
name.to_lowercase().replace(' ', "-")
}),
id: 1,
parent: None,
role: "button".into(),
name: name.into(),
value: None,
enabled,
focusable: true,
viewport_fixed: false,
visible,
selected: false,
bounds: Some([0.0, 0.0, 20.0, 20.0]),
slot: None,
}
}
#[test]
fn surface_content_uses_main_viewport_while_chrome_uses_the_window() {
let mut root = component("", true, true);
root.id = 9;
root.role = "generic".into();
root.bounds = Some([0.0, 0.0, 1344.0, 960.0]);
let mut main = component("", true, true);
main.id = 10;
main.parent = Some(root.id);
main.role = "main".into();
main.bounds = Some([0.0, 58.0, 1344.0, 842.0]);
let mut content = component("Row action", true, true);
content.id = 11;
content.parent = Some(main.id);
content.bounds = Some([962.0, -3.0, 28.0, 28.0]);
let mut chrome = component("Project tab", true, true);
chrome.id = 12;
chrome.parent = Some(root.id);
chrome.bounds = Some([20.0, 15.0, 120.0, 35.0]);
let snapshot = AgentSnapshot {
nodes: vec![root, main, content, chrome],
..AgentSnapshot::default()
};
assert_eq!(viewport_for_node(&snapshot, 11), (58.0, 900.0));
assert_eq!(viewport_for_node(&snapshot, 12), (0.0, 960.0));
}
#[test]
fn document_height_does_not_expand_the_pointer_viewport() {
let mut root = component("", true, true);
root.id = 20;
root.role = "generic".into();
root.bounds = Some([0.0, 0.0, 1344.0, 2200.0]);
let mut main = component("", true, true);
main.id = 21;
main.parent = Some(root.id);
main.role = "main".into();
main.bounds = Some([0.0, 64.0, 1344.0, 2200.0]);
let mut content = component("Below the fold", true, true);
content.id = 22;
content.parent = Some(main.id);
content.bounds = Some([40.0, 1200.0, 200.0, 40.0]);
let snapshot = AgentSnapshot {
nodes: vec![root, main, content],
viewport: Some([0.0, 0.0, 1344.0, 960.0]),
..AgentSnapshot::default()
};
assert_eq!(viewport_for_node(&snapshot, 22), (64.0, 960.0));
}
#[test]
fn fixed_content_uses_the_window_viewport_after_its_dom_parent_scrolls() {
let mut root = component("", true, true);
root.id = 30;
root.role = "generic".into();
root.bounds = Some([0.0, -11000.0, 1344.0, 12000.0]);
let mut main = component("", true, true);
main.id = 31;
main.parent = Some(root.id);
main.role = "main".into();
main.bounds = Some([0.0, -10942.0, 1344.0, 11942.0]);
let mut fixed = component("Return to coverage", true, true);
fixed.id = 32;
fixed.parent = Some(main.id);
fixed.viewport_fixed = true;
fixed.bounds = Some([16.0, 16.0, 180.0, 40.0]);
let snapshot = AgentSnapshot {
nodes: vec![root, main, fixed],
viewport: Some([0.0, 0.0, 1344.0, 960.0]),
..AgentSnapshot::default()
};
assert_eq!(viewport_for_node(&snapshot, 32), (0.0, 960.0));
assert!(!offscreen(
snapshot.nodes[2].bounds.expect("fixed bounds"),
viewport_for_node(&snapshot, 32)
));
}
#[test]
fn pixel_stability_reports_a_changed_rendered_pixel() {
use base64::Engine as _;
let capture = |rgba: &[u8]| CapturedImage {
width: 1,
height: 1,
rgba_base64: base64::engine::general_purpose::STANDARD.encode(rgba),
node_id: Some(7),
};
let before = capture(&[20, 20, 20, 255]);
assert!(pixels_hold(&before, &before).is_ok());
assert_eq!(
pixels_change(&before, &before).unwrap_err(),
"hover left every rendered pixel unchanged"
);
let jitter = capture(&[19, 20, 20, 255]);
assert!(pixels_hold(&before, &jitter).is_ok());
assert_eq!(
pixels_change(&before, &jitter).unwrap_err(),
"hover left every rendered pixel unchanged"
);
let after = capture(&[16, 20, 20, 255]);
assert_eq!(
pixels_hold(&before, &after).unwrap_err(),
"1 rendered pixel(s) changed after the pointer returned to the same state"
);
assert!(pixels_change(&before, &after).is_ok());
let resized = CapturedImage {
width: 2,
height: 1,
rgba_base64: base64::engine::general_purpose::STANDARD
.encode([20, 20, 20, 255, 20, 20, 20, 255]),
node_id: Some(7),
};
assert!(pixels_change(&before, &resized).is_ok());
let transparent = capture(&[0, 0, 0, 0]);
let resized_transparent = CapturedImage {
width: 2,
height: 1,
rgba_base64: base64::engine::general_purpose::STANDARD.encode([0, 0, 0, 0, 0, 0, 0, 0]),
node_id: Some(7),
};
assert_eq!(
pixels_change(&transparent, &resized_transparent).unwrap_err(),
"hover left every rendered pixel unchanged"
);
let alpha_only = capture(&[20, 20, 20, 80]);
assert!(rgb_pixels_hold(&before, &alpha_only).is_ok());
let visible_colour = capture(&[29, 20, 20, 255]);
assert_eq!(
rgb_pixels_hold(&before, &visible_colour).unwrap_err(),
"1 visibly coloured pixel(s) changed after the first hover"
);
}
#[test]
fn an_unchanged_early_paint_stays_pending_until_the_hover_frame_arrives() {
use base64::Engine as _;
let capture = |rgba: [u8; 4]| CapturedImage {
width: 1,
height: 1,
rgba_base64: base64::engine::general_purpose::STANDARD.encode(rgba),
node_id: Some(7),
};
let before = capture([20, 20, 20, 255]);
let unrelated_paint = capture([20, 20, 20, 255]);
let hover_paint = capture([12, 20, 20, 255]);
assert_eq!(
assess_pixel_change(&before, &unrelated_paint, false).unwrap(),
None,
"an earlier reveal/scroll paint must not become the hover verdict"
);
assert_eq!(
assess_pixel_change(&before, &hover_paint, false).unwrap(),
Some(()),
"the later hover paint completes the verdict"
);
assert_eq!(
assess_pixel_change(&before, &unrelated_paint, true).unwrap_err(),
"hover left every rendered pixel unchanged"
);
}
#[test]
fn raster_ink_rejects_an_empty_box_and_accepts_a_visible_mark() {
use base64::Engine as _;
let capture = |rgba: &[u8]| CapturedImage {
width: 2,
height: 2,
rgba_base64: base64::engine::general_purpose::STANDARD.encode(rgba),
node_id: Some(7),
};
let background = [20, 20, 20, 255];
let empty = capture(&background.repeat(4));
assert_eq!(measure_ink(&empty).unwrap().visible, 0);
let mut marked = background.repeat(4);
marked[0..4].copy_from_slice(&[240, 240, 240, 255]);
assert_eq!(measure_ink(&capture(&marked)).unwrap().visible, 1);
let transparent = capture(&[255, 255, 255, 0].repeat(4));
assert_eq!(measure_ink(&transparent).unwrap().visible, 0);
}
#[test]
fn zero_opacity_requires_the_applied_native_tint_to_be_clear() {
let clear = WindowComposition {
supported: true,
surface_transparent: true,
glass_enabled: true,
glass_backend: Some("nativeGlass".into()),
tint_rgba: Some([174, 50, 112, 0]),
radius: Some(12.0),
};
assert!(require_transparent_window_tint(&clear).is_ok());
let no_tint_backend = WindowComposition {
glass_backend: Some("vibrancy".into()),
tint_rgba: None,
radius: None,
..clear.clone()
};
assert!(require_transparent_window_tint(&no_tint_backend).is_ok());
let unreported_tint = WindowComposition {
tint_rgba: None,
..clear.clone()
};
assert!(
require_transparent_window_tint(&unreported_tint)
.unwrap_err()
.contains("did not expose")
);
let accent_sheet = WindowComposition {
tint_rgba: Some([174, 50, 112, 255]),
..clear.clone()
};
assert!(
require_transparent_window_tint(&accent_sheet)
.unwrap_err()
.contains("retains tint")
);
let opaque_window = WindowComposition {
surface_transparent: false,
..clear
};
assert!(
require_transparent_window_tint(&opaque_window)
.unwrap_err()
.contains("surface is opaque")
);
}
#[test]
fn capture_accepts_only_explicitly_requested_decorative_art() {
let mut icon = component("", true, false);
icon.id = 17;
icon.role = "presentation".into();
icon.bounds = Some([10.0, 10.0, 16.0, 16.0]);
assert!(capture_node_id(&[icon.clone()], "*").is_err());
assert_eq!(capture_node_id(&[icon], "presentation:*").unwrap(), 17);
}
#[test]
fn arrival_requires_three_consecutive_painted_snapshots() {
let mut streak = 0;
assert!(!stable_arrival(&mut streak, true));
assert!(!stable_arrival(&mut streak, false));
assert!(!stable_arrival(&mut streak, true));
assert!(!stable_arrival(&mut streak, true));
assert!(stable_arrival(&mut streak, true));
}
#[test]
fn scoped_arrival_needs_only_its_painted_marker() {
let marker = component("Search settings", true, true);
let settings = SurfaceSpec {
name: "settings".into(),
opener: "Settings".into(),
via: vec![],
marker: Some("Search settings".into()),
inventory_root: None,
reveal_with: Some("Search settings".into()),
};
assert!(arrival_sample_matches(
&[marker],
Some(&settings),
"Glass opacity",
true
));
}
#[test]
fn key_chords_preserve_dom_key_code_and_modifiers() {
let (key, code, modifiers) = parse_key_chord("Cmd+2").unwrap();
assert_eq!(key, "2");
assert_eq!(code, "Digit2");
assert!(modifiers.meta);
assert!(!modifiers.control);
let (key, code, modifiers) = parse_key_chord("Ctrl+Shift+Tab").unwrap();
assert_eq!(key, "Tab");
assert_eq!(code, "Tab");
assert!(modifiers.control);
assert!(modifiers.shift);
assert!(!modifiers.meta);
}
#[test]
fn inventory_categories_are_mutually_exclusive() {
let duplicates = HashSet::new();
assert_eq!(
inventory_class(
&component("Import data", true, true),
true,
false,
&duplicates
),
InventoryClass::Manual
);
assert_eq!(
inventory_class(
&component("Restart application", true, true),
false,
true,
&duplicates
),
InventoryClass::Isolated
);
assert_eq!(
inventory_class(&component("", false, false), false, false, &duplicates),
InventoryClass::Anonymous
);
assert_eq!(
inventory_class(&component("Save", false, true), false, false, &duplicates),
InventoryClass::Disabled
);
assert_eq!(
inventory_class(&component("Hidden", true, false), false, false, &duplicates),
InventoryClass::Unreachable
);
assert_eq!(
inventory_class(
&component("Synchronize", true, true),
false,
false,
&duplicates
),
InventoryClass::Reachable
);
}
#[test]
fn inventory_rejects_missing_and_duplicate_dom_ids() {
let mut missing = component("Import data", true, true);
missing.dom_id = None;
assert_eq!(
inventory_class(&missing, false, false, &HashSet::new()),
InventoryClass::MissingId
);
let duplicate = component("Save", true, true);
let duplicates = HashSet::from([duplicate.dom_id.clone().unwrap()]);
assert_eq!(
inventory_class(&duplicate, false, false, &duplicates),
InventoryClass::DuplicateId
);
}
#[test]
fn inventory_counts_an_excluded_control_as_excluded() {
let mut missing = component("Import data", true, true);
missing.dom_id = None;
assert_eq!(
inventory_class(&missing, true, false, &HashSet::new()),
InventoryClass::Manual
);
assert_eq!(
inventory_class(&missing, false, true, &HashSet::new()),
InventoryClass::Isolated
);
}
#[test]
fn inventory_rejects_framework_creation_order_ids() {
assert!(generated_dom_id("cl-0-trigger"));
assert!(generated_dom_id("7-31"));
assert!(generated_dom_id("7-31-trigger"));
assert!(!generated_dom_id("composer-effort-trigger"));
let mut generated = component("Save", true, true);
generated.dom_id = Some("cl-22-trigger".into());
assert_eq!(
inventory_class(&generated, false, false, &HashSet::new()),
InventoryClass::UnstableId
);
}
#[test]
fn duplicate_dom_ids_are_counted_from_the_live_tree() {
let first = component("Save", true, true);
let mut second = component("Discard", true, true);
second.dom_id = first.dom_id.clone();
assert_eq!(
duplicate_dom_ids(&[first.clone(), second]),
HashSet::from([first.dom_id.unwrap()])
);
}
#[test]
fn duplicate_dom_ids_are_scoped_to_the_active_surface() {
let first = component("Save", true, true);
let mut retained = component("Save retained", true, true);
retained.id = first.id + 1;
retained.dom_id.clone_from(&first.dom_id);
let mine = HashSet::from([first.id]);
assert!(
duplicate_dom_ids(
[&first, &retained]
.into_iter()
.filter(|node| mine.contains(&node.id))
)
.is_empty()
);
}
#[test]
fn hover_accumulation_ignores_one_new_identity_but_rejects_extra_copies() {
let existing = component("Hover action", true, true);
let baseline = hover_signature_counts(std::slice::from_ref(&existing));
let mut legitimate = component("Status mounted later", true, true);
legitimate.id = existing.id + 1;
let after_unique = hover_signature_counts(&[existing.clone(), legitimate]);
assert!(accumulated_hover_signatures(&baseline, &after_unique).is_empty());
let mut leaked = existing.clone();
leaked.id = existing.id + 2;
let after_leak = hover_signature_counts(&[existing, leaked]);
assert_eq!(
accumulated_hover_signatures(&baseline, &after_leak).len(),
1
);
}
#[test]
fn pointer_parking_ignores_a_larger_retained_hidden_surface() {
let mut visible = component("Active page", true, true);
visible.id = 41;
visible.bounds = Some([0.0, 0.0, 800.0, 600.0]);
let mut retained = component("Previous page", true, false);
retained.id = 42;
retained.bounds = Some([0.0, 0.0, 1600.0, 1200.0]);
assert_eq!(away_target_in(&[retained, visible]), Some(41));
}
#[test]
fn a_retained_hidden_pager_is_not_activated() {
let scope = HashSet::from([1]);
let patterns = vec![" more projects".to_owned()];
assert!(is_pagination_control(
&component("Show 5 more projects", true, true),
&scope,
&patterns
));
assert!(!is_pagination_control(
&component("Show 5 more projects", true, false),
&scope,
&patterns
));
}
#[test]
fn an_unchanged_pager_requires_real_tree_progress() {
let before = HashSet::from([
("button".to_owned(), "Show 5 more projects".to_owned(), None),
("button".to_owned(), "Open project one".to_owned(), None),
]);
assert!(!pagination_advanced(
&before,
"Show 5 more projects",
&before,
&component("Show 5 more projects", true, true)
));
assert!(pagination_advanced(
&before,
"Show 5 more projects",
&HashSet::from([
("button".to_owned(), "Show 5 more projects".to_owned(), None),
("button".to_owned(), "Open project one".to_owned(), None),
("button".to_owned(), "Open project two".to_owned(), None),
]),
&component("Show 5 more projects", true, true)
));
assert!(!pagination_advanced(
&before,
"Show 5 more projects",
&HashSet::from([
("button".to_owned(), "Show 5 more projects".to_owned(), None),
(
"button".to_owned(),
"Open project replacement".to_owned(),
None
),
]),
&component("Show 5 more projects", true, true)
));
assert!(pagination_advanced(
&before,
"Show 5 more projects",
&before,
&component("Show 3 more projects", true, true)
));
}
async fn serve_silent_host(socket: std::path::PathBuf, nodes: Vec<SemanticNode>) {
use blitz_control_protocol::{
DebugResponse, IncomingRequest, MessageStream, TransportStream, decode_incoming,
encode_initialize_response, encode_response,
};
use blitz_control_protocol::{NagoyaStream, framed_json_neutral};
use nagoya::reactor::{Addr, Reactor, TcpListener};
use std::os::unix::ffi::OsStrExt;
let reactor = Reactor::start().expect("fixture reactor");
let addr = Addr::path(socket.as_os_str().as_bytes()).expect("socket path");
let listener = TcpListener::bind(addr, &reactor.handle()).expect("bind test socket");
let (stream, _) = listener.accept().await.expect("accept test client");
let mut stream = TransportStream::new(framed_json_neutral(NagoyaStream::new(stream)));
while let Some(Ok(message)) = stream.recv().await {
let answer = match decode_incoming(message) {
Ok(IncomingRequest::Initialize { id }) => {
encode_initialize_response(id, "silent-host-fixture", "0.0.0")
.expect("encode initialize response")
}
Ok(IncomingRequest::Agent { id, .. }) => encode_response(
id,
&DebugResponse::AgentSnapshot(AgentSnapshot {
nodes: nodes.clone(),
..AgentSnapshot::default()
}),
)
.expect("encode snapshot"),
Ok(IncomingRequest::Diagnostics { id, .. }) => {
encode_response(id, &DebugResponse::Ack).expect("encode ack")
}
_ => continue,
};
if stream.send(answer).await.is_err() {
return;
}
}
}
#[test]
fn the_outcome_path_does_not_spend_the_declared_budget_waiting_for_a_frame() {
nagoya::block_on(the_outcome_path_does_not_spend_the_declared_budget());
}
async fn the_outcome_path_does_not_spend_the_declared_budget() {
let socket = std::env::temp_dir().join(format!(
"ps-qa-silent-host-{}-{}.sock",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system clock is after the epoch")
.as_nanos()
));
let saved = SemanticNode {
dom_id: None,
id: 3,
parent: None,
role: "button".into(),
name: "Saved".into(),
value: None,
enabled: true,
focusable: false,
viewport_fixed: false,
visible: true,
selected: false,
bounds: Some([0.0, 0.0, 90.0, 24.0]),
slot: None,
};
let host = serve_silent_host(socket.clone(), vec![saved.clone()]);
let client_socket = socket.clone();
let driver = async move {
let mut check = check("saved-paints", Some("Save"), "Saved");
check.outcome_timeout_ms = 200;
let budget = declared_outcome_timeout(&check);
let mut client = crate::inspector::Client::connect(&client_socket)
.await
.expect("connect to the silent host");
client.initialize().await.expect("initialize completes");
assert!(
client
.arm_paint_events()
.await
.expect("streaming is served"),
"the host serves the paint stream; it just never commits a frame"
);
let waited = std::time::Instant::now();
assert!(
!client
.wait_for_paint(budget)
.await
.expect("waiting for a frame is not itself an error"),
"no frame is ever committed"
);
assert!(
waited.elapsed() >= budget,
"the removed boundary spent the whole declared budget: {:?}",
waited.elapsed()
);
let settled = std::time::Instant::now();
let (_, error, _) =
settle_for_outcome(&mut client, &check, &[], None, None, None, true)
.await
.expect("the outcome is judged from the tree");
assert!(error.is_none(), "{error:?}");
assert!(
settled.elapsed() < budget / 2,
"the outcome is judged when it arrives, not when the budget expires: {:?}",
settled.elapsed()
);
};
futures::join!(host, driver);
let _ = std::fs::remove_file(socket);
}
#[test]
fn a_baseline_that_predates_the_setup_makes_its_outcome_vacuous() {
let field = |value: &str| SemanticNode {
dom_id: None,
id: 7,
parent: None,
role: "textbox".into(),
name: "Filter".into(),
value: Some(value.into()),
enabled: true,
focusable: false,
viewport_fixed: false,
visible: true,
selected: false,
bounds: Some([0.0, 0.0, 240.0, 28.0]),
slot: None,
};
let mut check = check("filter-value", Some("Apply"), "Filter");
check.setup_type_into = Some("Filter".into());
check.setup_text = Some("acme".into());
check.expect = Expect::ValueChanges;
let raced = [field("")];
let settled = [field("acme")];
assert!(
crate::qa::verdict(&check, &raced, &settled).is_ok(),
"the setup's own effect satisfies the outcome when the baseline predates it"
);
assert!(!setup_value_landed(&raced, 7, "acme"));
assert!(setup_value_landed(&settled, 7, "acme"));
assert!(!setup_value_landed(&settled, 8, "acme"));
assert!(crate::qa::verdict(&check, &settled, &settled).is_err());
}
fn check(id: &str, click: Option<&str>, subject: &str) -> Check {
Check {
id: id.into(),
group: "coverage".into(),
what: "a rendered outcome".into(),
open: None,
navigate: None,
arrival_subject: None,
prepare: None,
prepare_unless: None,
prepare_press: false,
prepare_key: None,
prepare_wait_ms: 0,
prepare_until: None,
hover: None,
hover_unless: None,
after_prepare_hover: None,
reveal_before_capture: None,
setup_type_into: None,
setup_text: None,
setup_text_env: None,
setup_secret: None,
click: click.map(str::to_owned),
storage: Default::default(),
type_into: None,
text: None,
key: None,
key_on: None,
keys: Vec::new(),
scroll_over: None,
scroll_ticks: 0,
scroll_delta: 0.0,
compare: None,
expect_size: None,
expect_count: None,
expect_value: None,
covers: Vec::new(),
press: false,
pointer_drag: None,
settle_after_ms: 0,
open_timeout_ms: 0,
outcome_timeout_ms: 0,
stable_for_ms: 0,
require_visible: false,
text_sized: false,
destructive: false,
subject: subject.into(),
subject_env: None,
subject_secret: None,
expect: Expect::Paints,
any_of: Vec::new(),
}
}
#[test]
fn every_outcome_layer_uses_the_check_deadline() {
let mut check = check("deadline", Some("Save"), "Saved");
assert_eq!(declared_outcome_timeout(&check), Duration::from_millis(900));
check.outcome_timeout_ms = 1_700;
assert_eq!(
declared_outcome_timeout(&check),
Duration::from_millis(1_700)
);
}
#[test]
fn a_declared_sequence_is_not_reordered_by_the_surface_it_names() {
fn old_order<'a>(all: &'a [Check], surfaces: &[&str]) -> Vec<&'a str> {
let mut affinity = 0;
let mut selected: Vec<(usize, &Check)> = Vec::new();
for check in all {
if let Some(opener) = check.open.as_deref()
&& let Some(index) = surfaces.iter().position(|surface| *surface == opener)
{
affinity = index;
}
selected.push((affinity, check));
}
selected.sort_by_key(|(surface, check)| (check.destructive, *surface));
selected
.into_iter()
.map(|(_, check)| check.id.as_str())
.collect()
}
let step = |id: &str, open: Option<&str>| {
let mut check = check(id, Some("Act"), "Result");
check.open = open.map(str::to_owned);
check
};
let mut teardown = step("reset-fixtures", Some("Settings"));
teardown.destructive = true;
let all = [
step("settings-open", Some("Settings")),
step("settings-edit", None),
step("registry-open", Some("Registry")),
step("registry-publish", None),
step("settings-save", Some("Settings")),
teardown,
];
assert_eq!(
old_order(&all, &["Settings", "Registry"]),
[
"settings-open",
"settings-edit",
"settings-save",
"registry-open",
"registry-publish",
"reset-fixtures",
],
"the bucketing ran a Settings step between the two Registry steps"
);
assert_eq!(
ordered_checks(&all, None)
.into_iter()
.map(|check| check.id.as_str())
.collect::<Vec<_>>(),
[
"settings-open",
"settings-edit",
"registry-open",
"registry-publish",
"settings-save",
"reset-fixtures",
],
"declaration order, with the destructive tail last"
);
assert_eq!(
ordered_checks(&all, Some("registry-publish"))
.into_iter()
.map(|check| check.id.as_str())
.collect::<Vec<_>>(),
["registry-publish"],
"one check by id still runs alone"
);
}
#[test]
fn the_open_step_takes_its_own_declared_deadline() {
let mut check = check("crates-tab", Some("Publish"), "Published");
assert_eq!(declared_open_timeout(&check), Duration::from_millis(900));
check.open_timeout_ms = 4_000;
assert_eq!(declared_open_timeout(&check), Duration::from_millis(4_000));
assert_eq!(
declared_outcome_timeout(&check),
Duration::from_millis(900),
"a slow route must not weaken the interaction contract"
);
check.outcome_timeout_ms = 1_500;
assert_eq!(declared_open_timeout(&check), Duration::from_millis(4_000));
assert_eq!(
declared_outcome_timeout(&check),
Duration::from_millis(1_500)
);
}
#[test]
fn a_surface_filter_is_case_insensitive_and_never_succeeds_empty() {
let surfaces = [SurfaceSpec {
name: "settings".into(),
opener: "Settings".into(),
via: vec![],
marker: Some("Search settings".into()),
inventory_root: None,
reveal_with: None,
}];
assert!(validate_surface_filter_against(Some("Settings"), &surfaces).is_ok());
let error = validate_surface_filter_against(Some("missing"), &surfaces)
.expect_err("an unknown surface must not produce a zero-row success");
assert!(error.to_string().contains("unknown surface \"missing\""));
assert!(error.to_string().contains("settings"));
}
#[test]
fn role_qualified_coverage_does_not_credit_a_same_named_wrong_role() {
let button = component("Rename project", true, true);
assert!(!selector_matches_node(&button, "textbox:Rename project"));
let mut textbox = button.clone();
textbox.role = "textbox".into();
assert!(selector_matches_node(&textbox, "textbox:Rename project"));
}
#[test]
fn exact_name_priority_does_not_choose_a_longer_substring() {
let restart = component("Restart", false, true);
let proxy = component("Restart AgencyProxy", true, true);
assert!(exact_selector_matches_node(&restart, "Restart"));
assert!(!exact_selector_matches_node(&proxy, "Restart"));
assert!(exact_selector_matches_node(&restart, "button:Restart"));
assert!(!exact_selector_matches_node(&restart, "switch:Restart"));
let mut candidates = vec![
(&proxy, [0.0, 0.0, 20.0, 20.0]),
(&restart, [0.0, 900.0, 20.0, 20.0]),
];
retain_exact_candidates(&mut candidates, "button:Restart");
assert_eq!(candidates.len(), 1);
assert_eq!(candidates[0].0.name, "Restart");
assert!(!candidates[0].0.enabled);
}
#[test]
fn check_preconditions_require_a_painted_matching_role() {
let button = component("Rename project", true, true);
assert!(painted_named(std::slice::from_ref(&button), "Rename"));
assert!(!painted_named(
std::slice::from_ref(&button),
"textbox:Rename project"
));
let mut textbox = button;
textbox.role = "textbox".into();
assert!(painted_named(
std::slice::from_ref(&textbox),
"textbox:Rename project"
));
textbox.bounds = Some([0.0, 0.0, 0.0, 0.0]);
assert!(!painted_named(
std::slice::from_ref(&textbox),
"textbox:Rename project"
));
}
#[test]
fn actionability_rejects_hidden_retained_menu_items_with_stale_boxes() {
let mut retained_menu_item = component("Opus", true, false);
assert_eq!(
painted_bounds(&retained_menu_item),
retained_menu_item.bounds
);
assert_eq!(
resolved_action_target(std::slice::from_ref(&retained_menu_item), "menuitem:Opus"),
None,
"the fixture role has not yet been made a menuitem",
);
retained_menu_item.role = "menuitem".into();
assert_eq!(
resolved_action_target(std::slice::from_ref(&retained_menu_item), "menuitem:Opus"),
None,
"a stale box does not make a hidden retained menu item actionable",
);
let mut mounted_menu_item = retained_menu_item.clone();
mounted_menu_item.visible = true;
assert_eq!(
resolved_action_target(std::slice::from_ref(&mounted_menu_item), "menuitem:Opus"),
Some("Opus".into()),
);
mounted_menu_item.bounds = Some([0.0, 0.0, 0.0, 0.0]);
assert!(painted_bounds(&mounted_menu_item).is_none());
}
#[test]
fn only_profile_declared_documents_require_exact_activation() {
let profile = AppProfile {
document_openers: vec!["Fixture project".into()],
..AppProfile::default()
};
assert!(named_document_opener_for(&profile, "fixture PROJECT"));
assert!(!named_document_opener_for(&profile, "Settings"));
}
#[test]
fn a_generic_document_marker_never_skips_named_document_activation() {
let project = SurfaceSpec {
name: "project".into(),
opener: crate::reach::DYNAMIC_DOCUMENT.into(),
via: vec![],
marker: Some("Send".into()),
inventory_root: None,
reveal_with: None,
};
let nodes = [component("Send", true, true)];
assert!(!arrived_without_navigation(
&nodes,
Some(&project),
"fixture row",
true,
false,
));
assert!(arrived_without_navigation(
&nodes,
Some(&project),
"fixture row",
true,
true,
));
assert!(arrived_without_navigation(
&nodes,
Some(&project),
"fixture row",
false,
false,
));
}
#[test]
fn a_named_document_is_active_only_when_its_exact_tab_is_selected() {
let mut tab = component("Fixture projectFixture project", true, true);
assert!(!named_document_is_active(&[tab.clone()], "Fixture project"));
tab.selected = true;
assert!(named_document_is_active(&[tab.clone()], "Fixture project"));
let mut undoubled_tab = component("Fixture project", true, true);
undoubled_tab.selected = true;
assert!(named_document_is_active(
&[undoubled_tab],
"Fixture project"
));
let permanent_name = "Settings".to_owned();
let mut permanent = component(&permanent_name, true, true);
permanent.selected = true;
assert!(
!named_document_is_active_with_permanent(
&[tab, permanent],
"Fixture project",
&[permanent_name],
),
"a selected retained document is not in front of a selected permanent surface"
);
}
#[test]
fn target_paints_resolves_a_role_qualified_click_selector() {
let status = component("Change the status of fixture item", true, true);
assert_eq!(
resolved_action_target(&[status], "button:Change the status of fixture item"),
Some("Change the status of fixture item".into()),
);
}
#[test]
fn outcome_coverage_names_only_checks_that_drive_an_enabled_control() {
let checks = [
check(
"rename",
Some("button:Rename project"),
"textbox:Rename project",
),
check("save", Some("Save"), "Saved"),
];
assert_eq!(
outcome_check_ids(&component("Rename project", true, true), &checks),
vec!["rename"]
);
assert!(outcome_check_ids(&component("Delete project", true, true), &checks).is_empty());
let mut editor = component("Rename project", true, true);
editor.role = "textbox".into();
assert!(outcome_check_ids(&editor, &checks).is_empty());
}
#[test]
fn navigation_never_credits_a_destination_control_but_prepare_does() {
let mut navigation = check("visit-settings", None, "heading:Settings");
navigation.open = Some("Settings".into());
assert!(
outcome_check_ids(
&component("Allow agents to update app settings", true, true),
&[navigation]
)
.is_empty(),
"opening Settings is not an outcome for an unrelated Settings control",
);
let mut selection = check(
"select-permission",
Some("menuitem:Edit"),
"Default permission",
);
selection.prepare = Some("button:Default permission: Auto".into());
assert_eq!(
outcome_check_ids(
&component("Default permission: Auto", true, true),
&[selection]
),
vec!["select-permission"],
"a dropdown trigger is genuinely driven by the check that selects one of its options",
);
let broad = check("click-settings", Some("Settings"), "heading:Settings");
assert!(
outcome_check_ids(
&component("Allow agents to update app settings", true, true),
&[broad]
)
.is_empty(),
"a convenient live substring must not become broad coverage credit",
);
}
#[test]
fn observing_disabled_is_complete_coverage_without_activation() {
let mut disabled = component("Use the default", false, true);
disabled.role = "button".into();
let mut observed = check("disabled-default", None, "Use the default");
observed.expect = Expect::Disabled;
assert_eq!(
outcome_check_ids(&disabled, &[observed]),
vec!["disabled-default"]
);
}
#[test]
fn a_control_measured_by_a_state_outcome_receives_coverage_credit() {
let mut selection = check(
"select-option",
Some("option:Beta"),
"combobox:Coverage combo box",
);
selection.expect = Expect::ValueChanges;
let mut input = component("Coverage combo box", true, true);
input.role = "combobox".into();
assert_eq!(
outcome_check_ids(&input, &[selection]),
vec!["select-option"]
);
}
#[test]
fn pointer_drag_targets_receive_coverage_credit() {
let mut drag = check("drag-hue", None, "slider:Hue");
drag.expect = Expect::ValueChanges;
drag.pointer_drag = Some(PointerDrag {
from: "slider:Hue".into(),
to: None,
dx: 20.0,
dy: 0.0,
steps: 2,
cancel: false,
});
let mut slider = component("Hue", true, true);
slider.role = "slider".into();
assert_eq!(outcome_check_ids(&slider, &[drag]), vec!["drag-hue"]);
}
#[test]
fn an_explicit_family_selector_credits_repeated_component_rows() {
let mut check = check("offer-model", Some("Offer Default"), "Offer Default");
check.covers.push("checkbox:Offer ".into());
let mut sibling = component("Offer Sonnet", true, true);
sibling.role = "checkbox".into();
assert_eq!(outcome_check_ids(&sibling, &[check]), vec!["offer-model"]);
}
#[test]
fn outcome_waiting_rejects_an_unchanged_refresh_indicator() {
let mut check = check("refresh", Some("Refresh"), "Refresh generation");
check.expect = Expect::NameChanges;
let before = component("Refresh generation 1", true, true);
let after = before.clone();
assert!(outcome_verdict(&check, &[before], &[after], None, None, None, None).is_err());
}
#[test]
fn outcome_waiting_accepts_the_completed_refresh_indicator() {
let mut check = check("refresh", Some("Refresh"), "Refresh generation");
check.expect = Expect::NameChanges;
let before = component("Refresh generation 1", true, true);
let after = component("Refresh generation 2", true, true);
assert!(outcome_verdict(&check, &[before], &[after], None, None, None, None).is_ok());
}
#[test]
fn focus_outcome_requires_the_named_destination_and_a_real_move() {
let mut check = check("toolbar-next", None, "button:Second tool");
check.expect = Expect::FocusMoves;
let mut first = component("First tool", true, true);
first.role = "button".into();
let mut second = component("Second tool", true, true);
second.id = 2;
second.role = "button".into();
assert!(
outcome_verdict(
&check,
&[first.clone(), second.clone()],
&[first.clone(), second.clone()],
None,
None,
Some(first.id),
Some(second.id),
)
.is_ok()
);
assert!(
outcome_verdict(
&check,
&[first.clone(), second.clone()],
&[first.clone(), second.clone()],
None,
None,
Some(first.id),
Some(first.id),
)
.is_err()
);
assert!(
outcome_verdict(
&check,
&[first, second.clone()],
&[second.clone()],
None,
None,
Some(second.id),
Some(second.id),
)
.is_err()
);
}
#[test]
fn value_outcome_can_belong_to_a_different_node_than_the_clicked_action() {
let mut check = check("reset-opacity", Some("Reset to default"), "Glass opacity");
check.expect = Expect::ValueChanges;
let reset = component("Reset to default", true, true);
let mut before_slider = component("Glass opacity", true, true);
before_slider.id = 2;
before_slider.role = "slider".into();
before_slider.value = Some("100".into());
let mut after_slider = before_slider.clone();
after_slider.value = Some("55".into());
assert!(
outcome_verdict(
&check,
&[reset.clone(), before_slider],
&[reset, after_slider],
Some("Reset to default"),
Some(1),
None,
None,
)
.is_ok()
);
}
#[test]
fn saved_inventory_rows_keep_quoted_control_names_with_commas() {
let rows = saved_controls(
"controls[1]{surface,id,role,name,classification,reason}:\n \
home,7,button,\"Delete alpha, beta\",reachable-unverified,none",
)
.expect("controls");
let row = &rows[0];
assert_eq!(row.surface, "home");
assert_eq!(row.role, "button");
assert_eq!(row.name, "Delete alpha, beta");
assert_eq!(row.classification, "reachable-unverified");
}
#[test]
fn a_saved_report_must_have_an_inventory_table() {
assert!(saved_controls("components: 3").is_err());
assert_eq!(
saved_controls(
"controls[1]{surface,id,role,name,classification,reason}:\n home,7,button,Save,reachable-unverified,none"
)
.expect("controls")
.len(),
1
);
}
#[test]
fn nested_inventory_rows_round_trip_through_the_toon_decoder() {
let rows = saved_controls(
"components: 1\ncontrols[1]:\n - surface: settings\n id: 7\n \
role: switch\n name: Enable inspection\n \
classification: \"isolated-unverified\"\n reason: separate process\n \
checks[0]:",
)
.expect("nested controls");
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].surface, "settings");
assert_eq!(rows[0].role, "switch");
assert_eq!(rows[0].name, "Enable inspection");
assert_eq!(rows[0].classification, "isolated-unverified");
}
#[test]
fn offline_inventory_keeps_slot_selector_credit() {
let rows = saved_controls(
"controls[1]{surface,id,dom_id,slot,role,name,classification,reason}:\n \
settings,7,theme-accent,complex-color-wheel,button,Accent,outcome-declared,matched",
)
.expect("controls");
let mut check = check("accent-wheel", None, "@complex-color-wheel");
check.covers.push("@complex-color-wheel".into());
assert_eq!(rows[0].slot.as_deref(), Some("complex-color-wheel"));
assert_eq!(
outcome_check_ids(&saved_control_node(&rows[0]), &[check]),
vec!["accent-wheel"]
);
}
#[test]
fn isolated_inventory_rows_remain_reported_without_blocking_reconciliation() {
assert!(!super::reconciliation_gap_blocks("isolated-unverified"));
assert!(super::reconciliation_gap_blocks("outcome-unverified"));
assert_eq!(super::inventory_outcome_failures(1, 1, true), 0);
assert_eq!(super::inventory_outcome_failures(3, 1, true), 2);
assert_eq!(super::inventory_outcome_failures(3, 1, false), 0);
}
#[test]
fn identity_failures_do_not_hide_missing_outcomes() {
let none: Vec<String> = Vec::new();
let declared = vec!["save-works".into()];
for class in [
InventoryClass::MissingId,
InventoryClass::UnstableId,
InventoryClass::DuplicateId,
InventoryClass::Anonymous,
InventoryClass::Unreachable,
] {
assert!(super::inventory_control_unverified(class, &none));
assert!(!super::inventory_outcome_declared(class, &none));
assert!(!super::inventory_control_unverified(class, &declared));
assert!(super::inventory_outcome_declared(class, &declared));
}
assert!(!super::inventory_control_unverified(
InventoryClass::Manual,
&none
));
assert!(!super::inventory_outcome_declared(
InventoryClass::Manual,
&declared
));
assert!(super::inventory_control_unverified(
InventoryClass::Isolated,
&declared
));
assert!(!super::inventory_outcome_declared(
InventoryClass::Isolated,
&declared
));
}
#[test]
fn a_bare_pattern_is_a_substring() {
assert!(name_matches("Rename project", "rename"));
assert!(name_matches("Rename project", "project"));
assert!(!name_matches("Rename project", "delete"));
}
#[test]
fn a_trailing_star_anchors_the_front() {
assert!(name_matches("chat with agent", "chat*"));
assert!(!name_matches("open chat", "chat*"));
}
#[test]
fn a_leading_star_anchors_the_end() {
assert!(name_matches("open chat", "*chat"));
assert!(!name_matches("chat with agent", "*chat"));
}
#[test]
fn stars_at_both_ends_match_anywhere() {
assert!(name_matches("the chat panel", "*chat*"));
assert!(!name_matches("Rename project", "*chat*"));
}
#[test]
fn matching_ignores_case() {
assert!(name_matches("Rename Project", "rename project"));
assert!(name_matches("CHAT", "chat*"));
}
#[test]
fn a_lone_star_matches_everything() {
assert!(name_matches("anything at all", "*"));
assert!(name_matches("", "*"));
}
}