use std::collections::HashMap;
#[cfg(all(feature = "agent-control", unix))]
use blitz_control_protocol::{
AgentSnapshot, DebugError, DebugResponse, KeyPhase, Modifiers as ControlModifiers, SemanticNode,
};
#[cfg(all(feature = "diagnostics", unix))]
use blitz_control_protocol::{
DebugSnapshot, FrameMetrics, FrameWindowMetrics, LayoutBounds, LayoutDiagnosticRow,
LayoutEdges, LayoutOffset, LayoutSize, RendererMetrics, RevisionSet, ScriptMetrics,
ScriptSource, SnapshotCost, SnapshotRequest, TimingStats,
};
#[cfg(all(feature = "agent-control", unix))]
use blitz_dom::Document;
use blitz_script::ScriptDocument;
#[cfg(all(feature = "agent-control", unix))]
use blitz_traits::events::{
BlitzKeyEvent, BlitzPointerEvent, BlitzPointerId, DomEvent, DomEventData, KeyState,
MouseEventButton, MouseEventButtons, Point, PointerCoords, PointerDetails, UiEvent,
};
#[cfg(all(feature = "diagnostics", unix))]
use blitz_traits::node_id::NodeId;
#[cfg(all(feature = "agent-control", unix))]
use keyboard_types::{Code, Key, Location, Modifiers as KeyboardModifiers};
#[cfg(all(feature = "diagnostics", unix))]
pub(crate) struct CaptureSurface {
pub(crate) width: u32,
pub(crate) height: u32,
pub(crate) renderer: anyrender_vello_cpu::VelloCpuImageRenderer,
pub(crate) rgba: Vec<u8>,
}
#[cfg(all(feature = "diagnostics", unix))]
pub struct DocumentCapture {
surface: Option<CaptureSurface>,
}
#[cfg(all(feature = "diagnostics", unix))]
impl DocumentCapture {
pub fn new() -> Self {
Self { surface: None }
}
pub fn capture(
&mut self,
document: &mut ScriptDocument,
request: blitz_control_protocol::CaptureRequest,
) -> Result<blitz_control_protocol::CapturedImage, DebugError> {
capture_document_with_surface(document, request, &mut self.surface)
}
}
#[cfg(all(feature = "diagnostics", unix))]
impl Default for DocumentCapture {
fn default() -> Self {
Self::new()
}
}
#[cfg(all(feature = "diagnostics", unix))]
impl CaptureSurface {
pub(crate) fn new(width: u32, height: u32) -> Self {
use anyrender::ImageRenderer as _;
Self {
width,
height,
renderer: anyrender_vello_cpu::VelloCpuImageRenderer::new(width, height),
rgba: Vec::with_capacity((width as usize) * (height as usize) * 4),
}
}
pub(crate) fn size_to(&mut self, width: u32, height: u32) {
use anyrender::ImageRenderer as _;
if self.width == width && self.height == height {
return;
}
self.renderer.resize(width, height);
self.width = width;
self.height = height;
}
}
#[cfg(all(feature = "diagnostics", unix))]
pub fn capture_document(
script_document: &mut ScriptDocument,
request: blitz_control_protocol::CaptureRequest,
) -> Result<blitz_control_protocol::CapturedImage, DebugError> {
DocumentCapture::new().capture(script_document, request)
}
#[cfg(all(feature = "diagnostics", unix))]
pub(crate) fn capture_document_with_surface(
script_document: &mut ScriptDocument,
request: blitz_control_protocol::CaptureRequest,
surface: &mut Option<CaptureSurface>,
) -> Result<blitz_control_protocol::CapturedImage, DebugError> {
use anyrender::ImageRenderer;
use base64::Engine as _;
let scale = if request.scale.is_finite() && request.scale > 0.0 {
request.scale.clamp(0.1, 8.0)
} else {
1.0
};
let node_id = request.node_id;
script_document.inner_mut().resolve(0.0);
let (full_width, full_height) = {
let inner = script_document.inner();
let viewport = inner.viewport();
(viewport.window_size.0, viewport.window_size.1)
};
if full_width == 0 || full_height == 0 {
return Err(debug_error(
"captureUnavailable",
"the document has no viewport to draw",
));
}
let (crop_x, crop_y, crop_width, crop_height) = match node_id {
None => (
0.0_f64,
0.0_f64,
f64::from(full_width),
f64::from(full_height),
),
Some(id) => {
let inner = script_document.inner();
let node = inner
.get_node(NodeId::from_u64(id))
.ok_or_else(|| debug_error("unknownNode", &format!("no node {id}")))?;
let layout = node.final_layout();
let position = node.absolute_position(0.0, 0.0);
if layout.size.width <= 0.0 || layout.size.height <= 0.0 {
return Err(debug_error(
"captureEmpty",
&format!("node {id} has a zero-sized box, so there is nothing to capture"),
));
}
let box_ = (
f64::from(position.x),
f64::from(position.y),
f64::from(layout.size.width),
f64::from(layout.size.height),
);
drop(inner);
box_
}
};
let full_pixel_width = ((f64::from(full_width) * f64::from(scale)).round() as u32).max(1);
let full_pixel_height = ((f64::from(full_height) * f64::from(scale)).round() as u32).max(1);
let left = ((crop_x * f64::from(scale)).round().max(0.0) as u32).min(full_pixel_width);
let top = ((crop_y * f64::from(scale)).round().max(0.0) as u32).min(full_pixel_height);
let width = ((crop_width * f64::from(scale)).round() as u32)
.min(full_pixel_width.saturating_sub(left))
.max(1);
let height = ((crop_height * f64::from(scale)).round() as u32)
.min(full_pixel_height.saturating_sub(top))
.max(1);
const FRAME_ENVELOPE_RESERVE: usize = 64 * 1024;
const MAX_BASE64_BYTES: usize =
blitz_control_protocol::MAX_DEBUG_FRAME_BYTES - FRAME_ENVELOPE_RESERVE;
const MAX_RAW_BYTES: usize = (MAX_BASE64_BYTES / 4) * 3;
const MAX_PIXELS: u64 = (MAX_RAW_BYTES / 4) as u64;
if u64::from(width) * u64::from(height) > MAX_PIXELS {
return Err(debug_error(
"captureTooLarge",
&format!(
"{width}x{height} cannot fit in one diagnostic frame; capture a node or lower the scale"
),
));
}
let surface = surface.get_or_insert_with(|| CaptureSurface::new(width, height));
surface.size_to(width, height);
surface.renderer.reset();
let mut document = script_document.inner_mut();
surface.renderer.render_to_vec(
|scene| {
if node_id.is_some() {
blitz_paint::paint_scene_region(
scene,
&mut document,
blitz_paint::PaintRegion::crop(
f64::from(scale),
f64::from(left) / f64::from(scale),
f64::from(top) / f64::from(scale),
width,
height,
),
);
} else {
blitz_paint::paint_scene(
scene,
&mut document,
f64::from(scale),
width,
height,
0,
0,
);
}
},
&mut surface.rgba,
);
Ok(blitz_control_protocol::CapturedImage {
width,
height,
rgba_base64: base64::engine::general_purpose::STANDARD.encode(&surface.rgba),
node_id,
})
}
#[cfg(all(feature = "diagnostics", unix))]
pub fn snapshot_document(
document: &mut ScriptDocument,
request: SnapshotRequest,
revision: u64,
) -> Result<DebugSnapshot, DebugError> {
let started = std::time::Instant::now();
let poll_started = std::time::Instant::now();
let mut polls = 0u64;
for _ in 0..100 {
polls += 1;
if !document.poll(None) {
break;
}
}
let poll_ms = poll_started.elapsed().as_secs_f64() * 1_000.0;
let resolve_started = std::time::Instant::now();
document.inner_mut().resolve(0.0);
let snapshot_resolve_ms = resolve_started.elapsed().as_secs_f64() * 1_000.0;
let inner = document.inner();
let layout_node_limit = inner.tree().iter().count();
let active_element = inner.get_focussed_node_id().map(|id| id.as_u64());
let nodes: Vec<SemanticNode> = inner
.tree()
.iter()
.filter_map(|(id, node)| {
if !request.node_ids.is_empty() && !request.node_ids.contains(&id.as_u64()) {
return None;
}
let element = node.element_data()?;
if !dom_chain_is_attached(&inner, id, layout_node_limit)
|| !layout_chain_is_valid(&inner, id, layout_node_limit)
{
return None;
}
let rect = inner.get_client_bounding_rect(id);
let visible = node_is_visible(&inner, id)
&& rect
.as_ref()
.is_some_and(|rect| rect.width > 0.0 && rect.height > 0.0);
let role = semantic_role(element);
let value = if role == "generic" {
Some(
element
.attrs()
.iter()
.map(|attribute| format!("{}={}", attribute.name.local, attribute.value))
.collect::<Vec<_>>()
.join(" "),
)
} else {
semantic_value(element)
};
Some(SemanticNode {
dom_id: element_attr(element, "id").map(str::to_owned),
id: id.as_u64(),
parent: semantic_parent(&inner, id, None).map(|id| id.as_u64()),
name: semantic_name(element, node, &role),
role,
value,
enabled: element_attr(element, "disabled").is_none()
&& element_attr(element, "aria-disabled") != Some("true"),
visible,
selected: semantic_selected(element),
bounds: rect.and_then(|rect| {
let bounds = [rect.x, rect.y, rect.width, rect.height];
bounds
.iter()
.all(|value| value.is_finite())
.then_some(bounds)
}),
slot: element_attr(element, "data-slot").map(str::to_owned),
})
})
.collect();
let total_ms = started.elapsed().as_secs_f64() * 1_000.0;
let revisions = RevisionSet {
document: revision,
style: 0,
layout: 0,
paint: 0,
};
let frame_stats = blitz_shell::latest_frame_stats();
let metrics = RendererMetrics {
revisions: revisions.clone(),
queue_depth: None,
invalidations_coalesced: polls.saturating_sub(1),
frame: frame_stats.as_ref().map(|stats| FrameMetrics {
input_to_present_ms: None,
style_ms: None,
layout_ms: None,
resolve_ms: stats.latest.resolve_ms,
scene_ms: stats.latest.paint_ms,
submit_ms: None,
present_ms: None,
renderer_ms: stats.latest.renderer_ms,
total_ms: stats.latest.total_ms,
age_ms: stats.latest.age_ms,
}),
frame_window: frame_stats.as_ref().map(|stats| FrameWindowMetrics {
frames_total: stats.frames_total,
window_frames: stats.window_frames,
resolve: timing_stats(stats.resolve),
scene: timing_stats(stats.paint),
renderer: timing_stats(stats.renderer),
total: timing_stats(stats.frame_total),
interval: timing_stats(stats.interval),
active_fps: stats.active_fps,
missed_refreshes: stats.missed_refreshes,
display_refresh_hz: stats.display_refresh_hz,
}),
snapshot: Some(SnapshotCost {
poll_ms,
resolve_ms: snapshot_resolve_ms,
total_ms,
}),
script: blitz_script::script_stats::latest_script_stats().map(|stats| ScriptMetrics {
mean_ms: stats.mean_ms,
p95_ms: stats.p95_ms,
max_ms: stats.max_ms,
window_polls: stats.window_polls,
total_polls: stats.total_polls,
productive_polls: stats.productive_polls,
spent_ms: stats.spent_ms,
breakdown: blitz_script::script_stats::work_breakdown()
.into_iter()
.take(12)
.map(|(label, calls, total_ms, worst_ms)| ScriptSource {
label,
calls,
total_ms,
worst_ms,
})
.collect(),
}),
resident_bytes: resident_bytes(),
};
let dom = request
.include_dom
.then(|| serde_json::to_value(&nodes).unwrap_or(serde_json::Value::Null));
let layout = request.include_layout.then(|| {
nodes
.iter()
.filter_map(|node| diagnostic_layout_row(&inner, node))
.collect()
});
let computed_style = request.include_computed_style.then(|| {
serde_json::Value::Array(
nodes
.iter()
.filter_map(|node| diagnostic_style_row(&inner, node))
.collect(),
)
});
Ok(DebugSnapshot {
revisions,
active_window: Some("blitz-main".into()),
active_element,
dom,
layout,
computed_style,
metrics,
})
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn element_attr<'a>(element: &'a blitz_dom::ElementData, name: &str) -> Option<&'a str> {
element
.attrs()
.iter()
.find(|attribute| attribute.name.local.as_ref() == name)
.map(|attribute| attribute.value.as_ref())
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn semantic_role(element: &blitz_dom::ElementData) -> String {
if let Some(role) = element_attr(element, "role") {
return role.into();
}
let tag = element.name.local.as_ref();
match tag {
"a" if element_attr(element, "href").is_some() => "link",
"button" => "button",
"textarea" => "textbox",
"select" => "combobox",
"option" => "option",
"img" => "img",
"nav" => "navigation",
"main" => "main",
"form" => "form",
"ul" | "ol" => "list",
"li" => "listitem",
"table" => "table",
"tr" => "row",
"td" | "th" => "cell",
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => "heading",
"input" => match element_attr(element, "type").unwrap_or("text") {
"checkbox" => "checkbox",
"radio" => "radio",
"button" | "submit" | "reset" => "button",
"range" => "slider",
_ => "textbox",
},
_ => "generic",
}
.into()
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn semantic_name(
element: &blitz_dom::ElementData,
node: &blitz_dom::Node,
role: &str,
) -> String {
let name = element_attr(element, "aria-label")
.or_else(|| element_attr(element, "alt"))
.or_else(|| element_attr(element, "title"))
.map(std::borrow::Cow::Borrowed)
.or_else(|| {
matches!(role, "button" | "link" | "heading" | "option")
.then(|| std::borrow::Cow::Owned(node.text_content()))
})
.unwrap_or_default();
let mut normalized = String::with_capacity(name.len().min(512));
let mut characters = 0;
for word in name.split_whitespace() {
if !normalized.is_empty() && characters < 512 {
normalized.push(' ');
characters += 1;
}
for character in word.chars() {
if characters == 512 {
return normalized;
}
normalized.push(character);
characters += 1;
}
}
normalized
}
#[cfg(all(feature = "agent-control", unix))]
fn semantic_value(element: &blitz_dom::ElementData) -> Option<String> {
element
.text_input_data()
.map(|input| input.editor.text().to_string())
.or_else(|| {
element
.checkbox_input_checked()
.map(|checked| checked.to_string())
})
.or_else(|| element_attr(element, "aria-valuenow").map(str::to_string))
.or_else(|| element_attr(element, "value").map(str::to_string))
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn semantic_selected(element: &blitz_dom::ElementData) -> bool {
if let Some(checked) = element.checkbox_input_checked() {
return checked;
}
let attribute_on = |name| match element_attr(element, name) {
Some("false") => false,
Some(_) => true,
None => false,
};
element_attr(element, "aria-selected") == Some("true")
|| element_attr(element, "aria-pressed") == Some("true")
|| element_attr(element, "aria-checked") == Some("true")
|| element_attr(element, "aria-current").is_some_and(|value| value != "false")
|| attribute_on("checked")
|| attribute_on("selected")
}
#[cfg(all(feature = "agent-control", unix))]
fn semantic_parent(
document: &blitz_dom::BaseDocument,
node_id: blitz_dom::NodeId,
root: Option<blitz_dom::NodeId>,
) -> Option<blitz_dom::NodeId> {
if Some(node_id) == root {
return None;
}
let mut current = document.get_node(node_id)?.parent;
while let Some(id) = current {
let node = document.get_node(id)?;
if node.element_data().is_some() {
return Some(id);
}
current = node.parent;
}
None
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn node_is_visible(
document: &blitz_dom::BaseDocument,
node_id: blitz_dom::NodeId,
) -> bool {
let mut current = Some(node_id);
while let Some(id) = current {
let Some(node) = document.get_node(id) else {
return false;
};
if !node_is_individually_visible(node) {
return false;
}
current = node.parent;
}
true
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn dom_chain_is_attached(
document: &blitz_dom::BaseDocument,
node_id: blitz_dom::NodeId,
node_limit: usize,
) -> bool {
let root = document.root_node().id;
let mut current = Some(node_id);
for _ in 0..=node_limit {
let Some(id) = current else {
return false;
};
if id == root {
return true;
}
let Some(node) = document.get_node(id) else {
return false;
};
current = node.parent;
}
false
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn layout_chain_is_valid(
document: &blitz_dom::BaseDocument,
node_id: blitz_dom::NodeId,
node_limit: usize,
) -> bool {
let mut current = Some(node_id);
for _ in 0..=node_limit {
let Some(id) = current else {
return true;
};
let Some(node) = document.get_node(id) else {
return false;
};
current = node.layout_parent.get();
}
false
}
#[cfg(all(feature = "agent-control", unix))]
#[cfg(all(feature = "agent-control", unix))]
#[cfg(all(feature = "agent-control", unix))]
#[cfg(all(feature = "agent-control", unix))]
pub fn hover_agent_node(
document: &mut ScriptDocument,
node_id: u64,
) -> Result<(f32, f32), DebugError> {
let position = resolve_agent_node(document, node_id)?.1;
document.handle_ui_event(UiEvent::PointerMove(pointer_event(
position,
MouseEventButton::Main,
MouseEventButtons::default(),
KeyboardModifiers::empty(),
)));
Ok(position)
}
pub fn press_agent_key(
document: &mut ScriptDocument,
key: &str,
code: &str,
) -> Result<(), DebugError> {
let parsed_key = key
.parse::<Key>()
.unwrap_or_else(|_| Key::Character(key.to_owned()));
let parsed_code = code.parse::<Code>().unwrap_or(Code::Unidentified);
for phase in [KeyPhase::Down, KeyPhase::Up] {
let event = key_event(
phase,
parsed_key.clone(),
parsed_code,
keyboard_modifiers(Default::default()),
);
document.handle_ui_event(match phase {
KeyPhase::Down => UiEvent::KeyDown(event),
KeyPhase::Up => UiEvent::KeyUp(event),
});
}
Ok(())
}
pub fn click_agent_node(
document: &mut ScriptDocument,
node_id: u64,
count: u8,
) -> Result<(f32, f32), DebugError> {
activate_agent_node(document, node_id, count)
}
#[cfg(all(feature = "agent-control", unix))]
pub fn focus_agent_node(
document: &mut ScriptDocument,
node_id: blitz_dom::NodeId,
) -> Result<(), DebugError> {
let focusable = document
.inner()
.get_node(node_id)
.and_then(|node| node.element_data())
.is_some_and(focuses_on_click);
if !focusable {
return Err(debug_error(
"notFocusable",
"node does not accept keyboard focus",
));
}
document.inner_mut().set_focus_to(node_id);
Ok(())
}
#[cfg(all(feature = "agent-control", unix))]
pub fn inspect_document(
document: &mut ScriptDocument,
root: Option<u64>,
max_depth: u32,
revision: u64,
) -> DebugResponse {
let mut ran_script = false;
for _ in 0..100 {
if !document.poll(None) {
break;
}
ran_script = true;
}
if ran_script {
document.inner_mut().resolve(0.0);
}
let inner = document.inner();
let root = root.map(blitz_dom::NodeId::from_u64);
if root.is_some_and(|id| inner.get_node(id).is_none()) {
return control_error("unknownNode", "the requested root node does not exist");
}
let focused_node = inner.get_focussed_node_id().map(|id| id.as_u64());
let node_limit = inner.tree().iter().count();
let candidates = if let Some(root) = root {
semantic_subtree_ids(&inner, root, max_depth)
.into_iter()
.filter_map(|id| {
inner.get_node(id)?;
dom_chain_is_attached(&inner, id, node_limit).then(|| SemanticCandidate {
id,
parent: semantic_parent(&inner, id, Some(root)),
visible: node_is_visible(&inner, id),
})
})
.collect()
} else {
attached_semantic_candidates(&inner, max_depth)
};
let layout_validity = layout_chain_validities(&inner, &candidates, node_limit);
let nodes = candidates
.into_iter()
.filter_map(|candidate| {
let id = candidate.id;
let node = inner.get_node(id)?;
let element = node.element_data()?;
if layout_validity.get(&id) != Some(&true) {
return None;
}
let rect = inner.get_client_bounding_rect(id);
let visible = candidate.visible
&& rect
.as_ref()
.is_some_and(|rect| rect.width > 0.0 && rect.height > 0.0);
let role = semantic_role(element);
let name = semantic_name(element, node, &role);
let value = semantic_value(element);
Some(SemanticNode {
dom_id: element_attr(element, "id").map(str::to_owned),
id: id.as_u64(),
parent: candidate.parent.map(|id| id.as_u64()),
role,
name,
value,
enabled: element_attr(element, "disabled").is_none()
&& element_attr(element, "aria-disabled") != Some("true"),
visible,
selected: semantic_selected(element),
bounds: rect.and_then(|rect| {
let bounds = [rect.x, rect.y, rect.width, rect.height];
bounds
.iter()
.all(|value| value.is_finite())
.then_some(bounds)
}),
slot: element_attr(element, "data-slot").map(str::to_owned),
})
})
.collect();
DebugResponse::AgentSnapshot(AgentSnapshot {
revision,
active_window: Some("blitz-main".into()),
focused_node,
nodes,
})
}
#[cfg(all(feature = "diagnostics", unix))]
pub(crate) fn timing_stats(stats: blitz_shell::TimingStats) -> TimingStats {
TimingStats {
mean_ms: stats.mean_ms,
p95_ms: stats.p95_ms,
max_ms: stats.max_ms,
}
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn debug_error(code: &str, message: &str) -> DebugError {
DebugError {
code: code.into(),
message: message.into(),
}
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn focuses_on_click(element: &blitz_dom::ElementData) -> bool {
let tag = element.name.local.as_ref();
matches!(tag, "button" | "input" | "select" | "textarea")
|| tag == "a" && element_attr(element, "href").is_some()
|| element_attr(element, "tabindex")
.and_then(|value| value.parse::<i32>().ok())
.is_some_and(|value| value >= 0)
|| element_attr(element, "contenteditable").is_some_and(|value| value != "false")
}
#[cfg(all(feature = "diagnostics", unix))]
pub(crate) fn resident_bytes() -> Option<u64> {
let output = std::process::Command::new("ps")
.args(["-o", "rss=", "-p", &std::process::id().to_string()])
.output()
.ok()?;
if !output.status.success() {
return None;
}
std::str::from_utf8(&output.stdout)
.ok()?
.trim()
.parse::<u64>()
.ok()
.and_then(|kilobytes| kilobytes.checked_mul(1_024))
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn node_is_individually_visible(node: &blitz_dom::Node) -> bool {
if !node.flags.is_in_document() || node.is_display_none() {
return false;
}
if node.primary_styles().is_some_and(|style| {
use style::computed_values::visibility::T as Visibility;
matches!(
style.clone_visibility(),
Visibility::Hidden | Visibility::Collapse
)
}) {
return false;
}
!node.element_data().is_some_and(|element| {
element_attr(element, "hidden").is_some()
|| element_attr(element, "aria-hidden") == Some("true")
})
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn resolve_agent_node(
document: &mut ScriptDocument,
raw_node_id: u64,
) -> Result<(blitz_dom::NodeId, (f32, f32)), DebugError> {
let node_id = blitz_dom::NodeId::from_u64(raw_node_id);
document.inner_mut().resolve(0.0);
let inner = document.inner();
let node = inner
.get_node(node_id)
.ok_or_else(|| debug_error("unknownNode", "node does not exist"))?;
if !node_is_visible(&inner, node_id) {
return Err(debug_error("notInteractable", "node is not visible"));
}
let node_limit = inner.tree().iter().count();
if !dom_chain_is_attached(&inner, node_id, node_limit)
|| !layout_chain_is_valid(&inner, node_id, node_limit)
{
return Err(debug_error(
"notInteractable",
"node has a detached layout ancestor",
));
}
if node
.element_data()
.is_some_and(|element| element_attr(element, "disabled").is_some())
{
return Err(debug_error("notInteractable", "node is disabled"));
}
let rect = inner
.get_client_bounding_rect(node_id)
.filter(|rect| rect.width > 0.0 && rect.height > 0.0)
.ok_or_else(|| debug_error("notInteractable", "node has no layout box"))?;
Ok((
node_id,
(
(rect.x + rect.width / 2.0) as f32,
(rect.y + rect.height / 2.0) as f32,
),
))
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn pointer_event(
position: (f32, f32),
button: MouseEventButton,
buttons: MouseEventButtons,
modifiers: KeyboardModifiers,
) -> BlitzPointerEvent {
BlitzPointerEvent {
id: BlitzPointerId::Mouse,
is_primary: true,
coords: pointer_coords(position),
button,
buttons,
mods: modifiers,
details: PointerDetails::default(),
element: Point::default(),
active_pointers: Default::default(),
}
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) struct SemanticCandidate {
pub(crate) id: blitz_dom::NodeId,
pub(crate) parent: Option<blitz_dom::NodeId>,
pub(crate) visible: bool,
}
#[cfg(all(feature = "agent-control", unix))]
#[derive(Clone, Copy)]
enum LayoutChainState {
Visiting,
Valid,
Invalid,
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn layout_chain_validities(
document: &blitz_dom::BaseDocument,
candidates: &[SemanticCandidate],
node_limit: usize,
) -> HashMap<blitz_dom::NodeId, bool> {
let mut states = HashMap::with_capacity(candidates.len());
for candidate in candidates {
if matches!(
states.get(&candidate.id),
Some(LayoutChainState::Valid | LayoutChainState::Invalid)
) {
continue;
}
let mut chain = Vec::new();
let mut current = Some(candidate.id);
let valid = loop {
let Some(id) = current else {
break true;
};
match states.get(&id) {
Some(LayoutChainState::Valid) => break true,
Some(LayoutChainState::Invalid | LayoutChainState::Visiting) => break false,
None => {}
}
if chain.len() > node_limit {
break false;
}
let Some(node) = document.get_node(id) else {
break false;
};
states.insert(id, LayoutChainState::Visiting);
chain.push(id);
current = node.layout_parent.get();
};
let resolved = if valid {
LayoutChainState::Valid
} else {
LayoutChainState::Invalid
};
for id in chain {
states.insert(id, resolved);
}
}
states
.into_iter()
.filter_map(|(id, state)| match state {
LayoutChainState::Valid => Some((id, true)),
LayoutChainState::Invalid => Some((id, false)),
LayoutChainState::Visiting => None,
})
.collect()
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn semantic_subtree_ids(
document: &blitz_dom::BaseDocument,
root: blitz_dom::NodeId,
max_depth: u32,
) -> Vec<blitz_dom::NodeId> {
let mut out = Vec::new();
let mut stack = vec![(root, 0_u32)];
while let Some((node_id, depth)) = stack.pop() {
let Some(node) = document.get_node(node_id) else {
continue;
};
if node.element_data().is_some() {
out.push(node_id);
}
for &child_id in node.children.iter().rev() {
let child_depth = depth.saturating_add(
document
.get_node(child_id)
.is_some_and(|child| child.element_data().is_some()) as u32,
);
if max_depth == 0 || child_depth <= max_depth {
stack.push((child_id, child_depth));
}
}
}
out
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn pointer_coords((x, y): (f32, f32)) -> PointerCoords {
PointerCoords {
page_x: x,
page_y: y,
screen_x: x,
screen_y: y,
client_x: x,
client_y: y,
}
}
#[cfg(all(feature = "diagnostics", unix))]
pub(crate) fn diagnostic_style_row(
document: &blitz_dom::BaseDocument,
node: &SemanticNode,
) -> Option<serde_json::Value> {
let dom_node = document.get_node(NodeId::from_u64(node.id))?;
let styles = dom_node.primary_styles()?;
let current = styles.clone_color();
let hex = |absolute: style::color::AbsoluteColor| {
let [r, g, b, a] = *absolute
.to_color_space(style::color::ColorSpace::Srgb)
.raw_components();
let channel = |value: f32| (value.clamp(0.0, 1.0) * 255.0).round() as u8;
format!(
"#{:02x}{:02x}{:02x}{:02x}",
channel(r),
channel(g),
channel(b),
channel(a),
)
};
let radius = format!("{:?}", styles.get_border().border_top_left_radius.0.width);
let border = styles.get_border();
let border_width = border.border_top_width.0.to_f64_px();
let font_size = styles.clone_font_size().computed_size().px();
let has_text_content = !dom_node.text_content().trim().is_empty();
Some(serde_json::json!({
"nodeId": node.id,
"color": hex(current),
"backgroundColor": hex(
styles.clone_background_color().resolve_to_absolute(¤t),
),
"borderColor": hex(border.border_top_color.resolve_to_absolute(¤t)),
"borderWidth": format!("{border_width}px"),
"fontSize": format!("{font_size}px"),
"hasTextContent": has_text_content,
"opacity": styles.clone_opacity(),
"borderTopLeftRadius": radius,
"visibility": format!("{:?}", styles.clone_visibility()),
}))
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn keyboard_modifiers(modifiers: ControlModifiers) -> KeyboardModifiers {
let mut output = KeyboardModifiers::empty();
output.set(KeyboardModifiers::SHIFT, modifiers.shift);
output.set(KeyboardModifiers::CONTROL, modifiers.control);
output.set(KeyboardModifiers::ALT, modifiers.alt);
output.set(KeyboardModifiers::META, modifiers.meta);
output
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn key_event(
phase: KeyPhase,
key: Key,
code: Code,
modifiers: KeyboardModifiers,
) -> BlitzKeyEvent {
let text = match (&key, phase) {
(Key::Character(value), KeyPhase::Down)
if !modifiers.intersects(
KeyboardModifiers::CONTROL | KeyboardModifiers::ALT | KeyboardModifiers::META,
) =>
{
Some(value.clone().into())
}
_ => None,
};
BlitzKeyEvent {
key,
code,
modifiers,
location: Location::Standard,
is_auto_repeating: false,
is_composing: false,
state: match phase {
KeyPhase::Down => KeyState::Pressed,
KeyPhase::Up => KeyState::Released,
},
text,
}
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn attached_semantic_candidates(
document: &blitz_dom::BaseDocument,
max_depth: u32,
) -> Vec<SemanticCandidate> {
let root = document.root_node().id;
let mut candidates = Vec::new();
let mut stack = vec![(root, 0_u32, None, true)];
while let Some((id, depth, semantic_parent, ancestors_visible)) = stack.pop() {
let Some(node) = document.get_node(id) else {
continue;
};
let visible = ancestors_visible && node_is_individually_visible(node);
let is_element = node.element_data().is_some();
if is_element && max_depth != 0 && depth > max_depth {
continue;
}
if is_element {
candidates.push(SemanticCandidate {
id,
parent: semantic_parent,
visible,
});
}
let child_depth = depth.saturating_add(is_element as u32);
let child_parent = if is_element {
Some(id)
} else {
semantic_parent
};
for &child in node.children.iter().rev() {
stack.push((child, child_depth, child_parent, visible));
}
}
candidates
}
#[cfg(all(feature = "agent-control", unix))]
pub(crate) fn control_error(code: &str, message: &str) -> DebugResponse {
DebugResponse::Error(debug_error(code, message))
}
#[cfg(all(feature = "diagnostics", unix))]
pub(crate) fn diagnostic_layout_row(
document: &blitz_dom::BaseDocument,
node: &SemanticNode,
) -> Option<LayoutDiagnosticRow> {
let bounds = node.bounds?;
let dom_node = document.get_node(NodeId::from_u64(node.id))?;
let layout = dom_node.final_layout();
let unzoom = |value: f32| match dom_node.primary_styles() {
Some(styles) => styles.effective_zoom.unzoom(value),
None => value,
};
let scroll_offset = dom_node.scroll_offset();
Some(LayoutDiagnosticRow {
node_id: node.id,
bounds: LayoutBounds::from(bounds),
scroll_offset: LayoutOffset {
x: f64::from(unzoom(scroll_offset.x as f32)),
y: f64::from(unzoom(scroll_offset.y as f32)),
},
client_size: LayoutSize {
width: f64::from(unzoom(layout.size.width)),
height: f64::from(unzoom(layout.size.height)),
},
scroll_size: LayoutSize {
width: f64::from(unzoom(layout.size.width + layout.scroll_width())),
height: f64::from(unzoom(layout.size.height + layout.scroll_height())),
},
scroll_range: LayoutSize {
width: f64::from(unzoom(layout.scroll_width())),
height: f64::from(unzoom(layout.scroll_height())),
},
border: LayoutEdges {
top: f64::from(unzoom(layout.border.top)),
right: f64::from(unzoom(layout.border.right)),
bottom: f64::from(unzoom(layout.border.bottom)),
left: f64::from(unzoom(layout.border.left)),
},
padding: LayoutEdges {
top: f64::from(unzoom(layout.padding.top)),
right: f64::from(unzoom(layout.padding.right)),
bottom: f64::from(unzoom(layout.padding.bottom)),
left: f64::from(unzoom(layout.padding.left)),
},
content_size: LayoutSize {
width: f64::from(unzoom(
layout.size.width
- layout.border.left
- layout.border.right
- layout.padding.left
- layout.padding.right,
)),
height: f64::from(unzoom(
layout.size.height
- layout.border.top
- layout.border.bottom
- layout.padding.top
- layout.padding.bottom,
)),
},
})
}
pub(crate) fn activate_agent_node(
document: &mut ScriptDocument,
raw_node_id: u64,
count: u8,
) -> Result<(f32, f32), DebugError> {
let (node_id, position) = resolve_agent_node(document, raw_node_id)?;
let focusable = document
.inner()
.get_node(node_id)
.and_then(|node| node.element_data())
.is_some_and(focuses_on_click);
for _ in 0..count {
let down = pointer_event(
position,
MouseEventButton::Main,
MouseEventButtons::Primary,
KeyboardModifiers::empty(),
);
let up = pointer_event(
position,
MouseEventButton::Main,
MouseEventButtons::default(),
KeyboardModifiers::empty(),
);
for data in [
DomEventData::PointerDown(down.clone()),
DomEventData::MouseDown(down),
DomEventData::PointerUp(up.clone()),
DomEventData::MouseUp(up.clone()),
DomEventData::Click(up),
] {
if document.inner().get_node(node_id).is_none() {
break;
}
document.dispatch_dom_event(DomEvent::new(node_id, data));
}
if focusable && document.inner().get_node(node_id).is_some() {
document.inner_mut().set_focus_to(node_id);
}
}
Ok(position)
}