#![allow(clippy::type_complexity)]
use crate::element::{SerializableUIElement, UIElementAttributes};
use crate::types::{OmniparserItem, VisionElement};
use crate::OcrElement;
use crate::UINode;
use std::collections::HashMap;
#[derive(Debug, Clone)]
pub struct TreeFormattingResult {
pub formatted: String,
pub index_to_bounds: HashMap<u32, (String, String, (f64, f64, f64, f64), Option<String>)>,
pub element_count: u32,
}
#[derive(Debug, Clone)]
pub struct OcrFormattingResult {
pub formatted: String,
pub index_to_bounds: HashMap<u32, (String, (f64, f64, f64, f64))>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ElementSource {
Uia, Dom, Ocr, Omniparser, Gemini, }
impl ElementSource {
pub fn prefix(&self) -> char {
match self {
ElementSource::Uia => 'u',
ElementSource::Dom => 'd',
ElementSource::Ocr => 'o',
ElementSource::Omniparser => 'p',
ElementSource::Gemini => 'g',
}
}
pub fn parse_prefixed_index(s: &str) -> Option<(ElementSource, u32)> {
if s.is_empty() {
return None;
}
let prefix = s.chars().next()?;
let num_str = &s[1..];
let num: u32 = num_str.parse().ok()?;
let source = match prefix {
'u' => ElementSource::Uia,
'd' => ElementSource::Dom,
'o' => ElementSource::Ocr,
'p' => ElementSource::Omniparser,
'g' => ElementSource::Gemini,
_ => return None,
};
Some((source, num))
}
}
#[derive(Debug, Clone)]
pub struct UnifiedElement {
pub source: ElementSource,
pub index: u32,
pub display_type: String, pub text: Option<String>, pub description: Option<String>, pub bounds: (f64, f64, f64, f64), }
impl UnifiedElement {
pub fn prefixed_index(&self) -> String {
format!("{}{}", self.source.prefix(), self.index)
}
pub fn center(&self) -> (f64, f64) {
let (x, y, w, h) = self.bounds;
(x + w / 2.0, y + h / 2.0)
}
}
#[derive(Debug, Clone)]
pub struct ClusteredFormattingResult {
pub formatted: String,
pub index_to_source_and_bounds: HashMap<String, (ElementSource, u32, (f64, f64, f64, f64))>,
}
pub fn serializable_to_ui_node(element: &SerializableUIElement) -> UINode {
UINode {
id: element.id.clone(),
attributes: UIElementAttributes {
role: element.role.clone(),
name: element.name.clone(),
label: element.label.clone(),
text: element.text.clone(),
value: element.value.clone(),
description: element.description.clone(),
application_name: element.window_and_application_name.clone(),
properties: HashMap::new(),
is_keyboard_focusable: element.is_keyboard_focusable,
is_focused: element.is_focused,
is_toggled: element.is_toggled,
bounds: element.bounds,
enabled: element.enabled,
is_selected: element.is_selected,
child_count: element.child_count,
index_in_parent: element.index_in_parent,
},
children: element
.children
.as_ref()
.map(|c| c.iter().map(serializable_to_ui_node).collect())
.unwrap_or_default(),
selector: element.selector.clone(),
}
}
fn ui_node_to_serializable(node: &UINode) -> SerializableUIElement {
SerializableUIElement {
id: node.id.clone(),
role: node.attributes.role.clone(),
name: node.attributes.name.clone(),
bounds: node.attributes.bounds,
value: node.attributes.value.clone(),
description: node.attributes.description.clone(),
window_and_application_name: node.attributes.application_name.clone(),
window_title: None,
url: None,
process_id: None,
process_name: None,
children: if node.children.is_empty() {
None
} else {
Some(node.children.iter().map(ui_node_to_serializable).collect())
},
label: node.attributes.label.clone(),
text: None, is_keyboard_focusable: node.attributes.is_keyboard_focusable,
is_focused: node.attributes.is_focused,
is_toggled: node.attributes.is_toggled,
enabled: node.attributes.enabled,
is_selected: node.attributes.is_selected,
child_count: node.attributes.child_count,
index_in_parent: node.attributes.index_in_parent,
selector: node.selector.clone(),
}
}
pub fn format_tree_as_compact_yaml(
tree: &SerializableUIElement,
indent: usize,
) -> TreeFormattingResult {
let mut output = String::new();
let mut index_to_bounds = HashMap::new();
let mut next_index = 1u32;
format_node(
tree,
indent,
&mut output,
&mut index_to_bounds,
&mut next_index,
);
TreeFormattingResult {
formatted: output,
index_to_bounds,
element_count: next_index - 1,
}
}
pub fn format_ui_node_as_compact_yaml(tree: &UINode, indent: usize) -> TreeFormattingResult {
let serializable = ui_node_to_serializable(tree);
format_tree_as_compact_yaml(&serializable, indent)
}
fn format_node(
node: &SerializableUIElement,
indent: usize,
output: &mut String,
index_to_bounds: &mut HashMap<u32, (String, String, (f64, f64, f64, f64), Option<String>)>,
next_index: &mut u32,
) {
let indent_str = if indent > 0 {
" ".repeat(indent)
} else {
String::new()
};
output.push_str(&indent_str);
if let Some((x, y, w, h)) = node.bounds {
let idx = *next_index;
*next_index += 1;
output.push_str(&format!("#{idx} "));
output.push_str(&format!("[{}]", node.role));
let name = node.name.clone().unwrap_or_default();
index_to_bounds.insert(
idx,
(node.role.clone(), name, (x, y, w, h), node.selector.clone()),
);
} else {
output.push_str(&format!("- [{}]", node.role));
}
if let Some(ref name) = node.name {
if !name.is_empty() {
output.push_str(&format!(" {name}"));
}
}
let mut context_parts = Vec::new();
if let Some(ref text) = node.text {
if !text.is_empty() {
context_parts.push(format!("text: {text}"));
}
}
if let Some((x, y, w, h)) = node.bounds {
context_parts.push(format!("bounds: [{x:.0},{y:.0},{w:.0},{h:.0}]"));
}
if let Some(enabled) = node.enabled {
if !enabled {
context_parts.push("disabled".to_string());
}
}
if node.is_focused == Some(true) {
context_parts.push("focused".to_string());
}
if node.is_keyboard_focusable == Some(true) {
context_parts.push("focusable".to_string());
}
if node.is_selected == Some(true) {
context_parts.push("selected".to_string());
}
if node.is_toggled == Some(true) {
context_parts.push("toggled".to_string());
}
if let Some(ref value) = node.value {
if !value.is_empty() {
context_parts.push(format!("value: {value}"));
}
}
if node.children.is_none() {
if let Some(count) = node.child_count {
if count > 0 {
context_parts.push(format!("{count} children"));
}
}
}
if !context_parts.is_empty() {
output.push_str(&format!(" ({})", context_parts.join(", ")));
}
output.push('\n');
if let Some(ref children) = node.children {
for child in children {
format_node(child, indent + 1, output, index_to_bounds, next_index);
}
}
}
pub fn format_ocr_tree_as_compact_yaml(tree: &OcrElement, indent: usize) -> OcrFormattingResult {
let mut output = String::new();
let mut index_to_bounds = HashMap::new();
let mut next_index = 1u32;
format_ocr_node(
tree,
indent,
&mut output,
&mut index_to_bounds,
&mut next_index,
);
OcrFormattingResult {
formatted: output,
index_to_bounds,
}
}
fn format_ocr_node(
node: &OcrElement,
indent: usize,
output: &mut String,
index_to_bounds: &mut HashMap<u32, (String, (f64, f64, f64, f64))>,
next_index: &mut u32,
) {
let indent_str = if indent > 0 {
" ".repeat(indent)
} else {
String::new()
};
output.push_str(&indent_str);
let current_index = if node.role == "OcrWord" {
let idx = *next_index;
*next_index += 1;
output.push_str(&format!("#{idx} [{role}]", role = node.role));
Some(idx)
} else {
output.push_str(&format!("- [{}]", node.role));
None
};
if let Some(ref text) = node.text {
if !text.is_empty() {
if node.role == "OcrWord" || node.role == "OcrLine" {
output.push_str(&format!(" \"{text}\""));
}
}
}
let mut context_parts = Vec::new();
if let Some((x, y, w, h)) = node.bounds {
context_parts.push(format!("bounds: [{x:.0},{y:.0},{w:.0},{h:.0}]"));
if let Some(idx) = current_index {
let text = node.text.clone().unwrap_or_default();
index_to_bounds.insert(idx, (text, (x, y, w, h)));
}
}
if let Some(angle) = node.text_angle {
context_parts.push(format!("text_angle: {angle:.1}"));
}
if let Some(confidence) = node.confidence {
context_parts.push(format!("confidence: {confidence:.2}"));
}
if !context_parts.is_empty() {
output.push_str(&format!(" ({})", context_parts.join(", ")));
}
output.push('\n');
if let Some(ref children) = node.children {
for child in children {
format_ocr_node(child, indent + 1, output, index_to_bounds, next_index);
}
}
}
fn min_edge_distance(b1: (f64, f64, f64, f64), b2: (f64, f64, f64, f64)) -> f64 {
let (x1, y1, w1, h1) = b1;
let (x2, y2, w2, h2) = b2;
let h_gap = f64::max(0.0, f64::max(x1 - (x2 + w2), x2 - (x1 + w1)));
let v_gap = f64::max(0.0, f64::max(y1 - (y2 + h2), y2 - (y1 + h1)));
(h_gap * h_gap + v_gap * v_gap).sqrt()
}
fn should_cluster(b1: (f64, f64, f64, f64), b2: (f64, f64, f64, f64)) -> bool {
let smaller_dim = f64::min(f64::min(b1.2, b1.3), f64::min(b2.2, b2.3));
let threshold = smaller_dim * 1.5;
min_edge_distance(b1, b2) < threshold
}
fn cluster_elements(elements: Vec<UnifiedElement>) -> Vec<Vec<UnifiedElement>> {
if elements.is_empty() {
return vec![];
}
let n = elements.len();
let mut parent: Vec<usize> = (0..n).collect();
fn find(parent: &mut [usize], i: usize) -> usize {
if parent[i] != i {
parent[i] = find(parent, parent[i]);
}
parent[i]
}
fn union(parent: &mut [usize], i: usize, j: usize) {
let pi = find(parent, i);
let pj = find(parent, j);
if pi != pj {
parent[pi] = pj;
}
}
for i in 0..n {
for j in (i + 1)..n {
if should_cluster(elements[i].bounds, elements[j].bounds) {
union(&mut parent, i, j);
}
}
}
let mut cluster_map: HashMap<usize, Vec<usize>> = HashMap::new();
for i in 0..n {
let root = find(&mut parent, i);
cluster_map.entry(root).or_default().push(i);
}
let mut clusters: Vec<Vec<UnifiedElement>> = cluster_map
.into_values()
.map(|indices| {
let mut cluster: Vec<UnifiedElement> =
indices.into_iter().map(|i| elements[i].clone()).collect();
cluster.sort_by(|a, b| {
let (_, ay, _, _) = a.bounds;
let (_, by, _, _) = b.bounds;
let (ax, _, _, _) = a.bounds;
let (bx, _, _, _) = b.bounds;
ay.partial_cmp(&by)
.unwrap_or(std::cmp::Ordering::Equal)
.then(ax.partial_cmp(&bx).unwrap_or(std::cmp::Ordering::Equal))
});
cluster
})
.collect();
clusters.sort_by(|a, b| {
let a_first = a.first().map(|e| e.bounds).unwrap_or((0.0, 0.0, 0.0, 0.0));
let b_first = b.first().map(|e| e.bounds).unwrap_or((0.0, 0.0, 0.0, 0.0));
a_first
.1
.partial_cmp(&b_first.1)
.unwrap_or(std::cmp::Ordering::Equal)
.then(
a_first
.0
.partial_cmp(&b_first.0)
.unwrap_or(std::cmp::Ordering::Equal),
)
});
clusters
}
pub fn format_clustered_tree_from_caches(
uia_bounds: &HashMap<u32, (String, String, (f64, f64, f64, f64), Option<String>)>,
dom_bounds: &HashMap<u32, (String, String, (f64, f64, f64, f64))>,
ocr_bounds: &HashMap<u32, (String, (f64, f64, f64, f64))>,
omniparser_items: &HashMap<u32, OmniparserItem>,
vision_items: &HashMap<u32, VisionElement>,
) -> ClusteredFormattingResult {
let mut all_elements: Vec<UnifiedElement> = Vec::new();
for (idx, (role, name, bounds, _selector)) in uia_bounds {
all_elements.push(UnifiedElement {
source: ElementSource::Uia,
index: *idx,
display_type: role.clone(),
text: if name.is_empty() {
None
} else {
Some(name.clone())
},
description: None,
bounds: *bounds,
});
}
for (idx, (tag, identifier, bounds)) in dom_bounds {
all_elements.push(UnifiedElement {
source: ElementSource::Dom,
index: *idx,
display_type: tag.clone(),
text: if identifier.is_empty() {
None
} else {
Some(identifier.clone())
},
description: None,
bounds: *bounds,
});
}
for (idx, (text, bounds)) in ocr_bounds {
all_elements.push(UnifiedElement {
source: ElementSource::Ocr,
index: *idx,
display_type: "OcrWord".to_string(),
text: Some(text.clone()),
description: None,
bounds: *bounds,
});
}
for (idx, item) in omniparser_items {
if let Some(box_2d) = item.box_2d {
let bounds = (
box_2d[0],
box_2d[1],
box_2d[2] - box_2d[0],
box_2d[3] - box_2d[1],
);
all_elements.push(UnifiedElement {
source: ElementSource::Omniparser,
index: *idx,
display_type: item.label.clone(),
text: item.content.clone(),
description: None,
bounds,
});
}
}
for (idx, item) in vision_items {
if let Some(box_2d) = item.box_2d {
let bounds = (
box_2d[0],
box_2d[1],
box_2d[2] - box_2d[0],
box_2d[3] - box_2d[1],
);
all_elements.push(UnifiedElement {
source: ElementSource::Gemini,
index: *idx,
display_type: item.element_type.clone(),
text: item.content.clone(),
description: item.description.clone(),
bounds,
});
}
}
let mut index_to_source_and_bounds: HashMap<
String,
(ElementSource, u32, (f64, f64, f64, f64)),
> = HashMap::new();
for elem in &all_elements {
let key = elem.prefixed_index();
index_to_source_and_bounds.insert(key, (elem.source, elem.index, elem.bounds));
}
let clusters = cluster_elements(all_elements);
let mut output = String::new();
for cluster in clusters {
if cluster.is_empty() {
continue;
}
let (sum_x, sum_y, count) = cluster.iter().fold((0.0, 0.0, 0), |(sx, sy, c), elem| {
let (cx, cy) = elem.center();
(sx + cx, sy + cy, c + 1)
});
let centroid = (sum_x / count as f64, sum_y / count as f64);
output.push_str(&format!(
"# Cluster @({:.0},{:.0})\n",
centroid.0, centroid.1
));
for elem in &cluster {
output.push_str(&format!(
"- [{}] #{} ",
elem.display_type,
elem.prefixed_index()
));
if let Some(ref text) = elem.text {
if !text.is_empty() {
output.push_str(&format!("\"{}\" ", text));
}
}
if let Some(ref desc) = elem.description {
if !desc.is_empty() {
output.push_str(&format!("({}) ", desc));
}
}
let (x, y, w, h) = elem.bounds;
output.push_str(&format!(
"(bounds: [{:.0},{:.0},{:.0},{:.0}])\n",
x, y, w, h
));
}
output.push('\n'); }
ClusteredFormattingResult {
formatted: output,
index_to_source_and_bounds,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_formatting() {
let node = SerializableUIElement {
id: Some("123".to_string()),
role: "Button".to_string(),
name: Some("Submit".to_string()),
bounds: Some((10.0, 20.0, 100.0, 50.0)),
value: None,
description: None,
window_and_application_name: None,
window_title: None,
url: None,
process_id: None,
process_name: None,
children: None,
label: None,
text: None,
is_keyboard_focusable: Some(true),
is_focused: None,
is_toggled: None,
enabled: Some(true),
is_selected: None,
child_count: None,
index_in_parent: None,
selector: None,
};
let result = format_tree_as_compact_yaml(&node, 0);
assert!(result.formatted.contains("#1 [Button] Submit"));
assert!(result.formatted.contains("bounds: [10,20,100,50]"));
assert!(result.formatted.contains("focusable"));
assert_eq!(result.element_count, 1);
assert!(result.index_to_bounds.contains_key(&1));
}
#[test]
fn test_nested_formatting() {
let child = SerializableUIElement {
id: Some("456".to_string()),
role: "Text".to_string(),
name: Some("Label".to_string()),
bounds: None,
value: None,
description: None,
window_and_application_name: None,
window_title: None,
url: None,
process_id: None,
process_name: None,
children: None,
label: None,
text: None,
is_keyboard_focusable: None,
is_focused: None,
is_toggled: None,
enabled: None,
is_selected: None,
child_count: None,
index_in_parent: None,
selector: None,
};
let parent = SerializableUIElement {
id: Some("123".to_string()),
role: "Window".to_string(),
name: Some("Main".to_string()),
bounds: None,
value: None,
description: None,
window_and_application_name: None,
window_title: None,
url: None,
process_id: None,
process_name: None,
children: Some(vec![child]),
label: None,
text: None,
is_keyboard_focusable: None,
is_focused: None,
is_toggled: None,
enabled: None,
is_selected: None,
child_count: None,
index_in_parent: None,
selector: None,
};
let result = format_tree_as_compact_yaml(&parent, 0);
assert!(result.formatted.contains("- [Window] Main"));
assert!(result.formatted.contains(" - [Text] Label"));
assert_eq!(result.element_count, 0); }
#[test]
fn test_mixed_bounds_formatting() {
let child_with_bounds = SerializableUIElement {
id: None,
role: "Button".to_string(),
name: Some("Click Me".to_string()),
bounds: Some((100.0, 200.0, 80.0, 30.0)),
value: None,
description: None,
window_and_application_name: None,
window_title: None,
url: None,
process_id: None,
process_name: None,
children: None,
label: None,
text: None,
is_keyboard_focusable: None,
is_focused: None,
is_toggled: None,
enabled: None,
is_selected: None,
child_count: None,
index_in_parent: None,
selector: Some("role:Button && name:Click Me".to_string()),
};
let child_no_bounds = SerializableUIElement {
id: None,
role: "Text".to_string(),
name: Some("Label".to_string()),
bounds: None,
value: None,
description: None,
window_and_application_name: None,
window_title: None,
url: None,
process_id: None,
process_name: None,
children: None,
label: None,
text: None,
is_keyboard_focusable: None,
is_focused: None,
is_toggled: None,
enabled: None,
is_selected: None,
child_count: None,
index_in_parent: None,
selector: None,
};
let parent = SerializableUIElement {
id: None,
role: "Window".to_string(),
name: Some("Test".to_string()),
bounds: Some((0.0, 0.0, 800.0, 600.0)),
value: None,
description: None,
window_and_application_name: None,
window_title: None,
url: None,
process_id: None,
process_name: None,
children: Some(vec![child_with_bounds, child_no_bounds]),
label: None,
text: None,
is_keyboard_focusable: None,
is_focused: None,
is_toggled: None,
enabled: None,
is_selected: None,
child_count: None,
index_in_parent: None,
selector: None,
};
let result = format_tree_as_compact_yaml(&parent, 0);
assert!(result.formatted.contains("#1 [Window] Test"));
assert!(result.formatted.contains("#2 [Button] Click Me"));
assert!(result.formatted.contains("- [Text] Label")); assert_eq!(result.element_count, 2);
let button_entry = result.index_to_bounds.get(&2).unwrap();
assert_eq!(
button_entry.3,
Some("role:Button && name:Click Me".to_string())
);
}
}