#[derive(Debug, Clone, PartialEq)]
pub struct Rect {
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
}
impl Rect {
#[must_use]
pub fn center(&self) -> (f64, f64) {
(self.x + self.width / 2.0, self.y + self.height / 2.0)
}
#[must_use]
pub fn area(&self) -> f64 {
self.width * self.height
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Position {
TopLeft,
TopRight,
Center,
BottomLeft,
BottomRight,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SizeClass {
Small,
Medium,
Large,
FullWidth,
}
impl SizeClass {
#[must_use]
pub fn from_rect(rect: &Rect) -> Self {
match rect.area() {
a if a < 1_000.0 => Self::Small,
a if a < 10_000.0 => Self::Medium,
a if a < 100_000.0 => Self::Large,
_ => Self::FullWidth,
}
}
#[must_use]
fn confidence(&self) -> f64 {
match self {
Self::Small => 0.5,
Self::Medium => 0.8,
Self::Large => 0.9,
Self::FullWidth => 1.0,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct VisualSignal {
pub bounds: Rect,
pub relative_position: Position,
pub size_class: SizeClass,
}
impl VisualSignal {
#[must_use]
pub fn confidence(&self) -> f64 {
let area_factor = (self.bounds.area() / 10_000.0).min(1.0).sqrt();
(area_factor + self.size_class.confidence()) / 2.0
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct SemanticSignal {
pub role: String,
pub label: String,
pub value: Option<String>,
}
impl SemanticSignal {
#[must_use]
pub fn confidence(&self) -> f64 {
let role_score: f64 = if self.role.is_empty() { 0.0 } else { 0.4 };
let label_score: f64 = match self.label.len() {
0 => 0.0,
1..=3 => 0.2,
4..=20 => 0.5,
_ => 0.4, };
let value_bonus: f64 = if self.value.is_some() { 0.1 } else { 0.0 };
(role_score + label_score + value_bonus).min(1.0_f64)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StructuralSignal {
pub depth: usize,
pub parent_role: String,
pub sibling_index: usize,
pub child_count: usize,
}
impl StructuralSignal {
#[must_use]
pub fn confidence(&self) -> f64 {
let parent_score: f64 = if self.parent_role.is_empty() {
0.3
} else {
0.6
};
let leaf_bonus: f64 = if self.child_count == 0 { 0.2 } else { 0.0 };
let depth_score: f64 = match self.depth {
0 => 0.1, 1..=2 => 0.3,
3..=6 => 0.5,
_ => 0.4, };
(parent_score + leaf_bonus + depth_score).min(1.0_f64)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Inconsistency {
VisuallyDisabledButA11yEnabled,
NotInA11yTree,
LoadingState,
Other(String),
}
#[derive(Debug, Clone)]
pub struct ElementUnderstanding {
pub visual: VisualSignal,
pub semantic: SemanticSignal,
pub structural: StructuralSignal,
pub combined_confidence: f64,
pub inconsistencies: Vec<Inconsistency>,
}
const VISUAL_WEIGHT: f64 = 0.20;
const SEMANTIC_WEIGHT: f64 = 0.50;
const STRUCTURAL_WEIGHT: f64 = 0.30;
pub struct TripleUnderstanding;
impl TripleUnderstanding {
#[must_use]
pub fn build(
visual: VisualSignal,
semantic: SemanticSignal,
structural: StructuralSignal,
) -> ElementUnderstanding {
let combined_confidence = Self::weighted_confidence(&visual, &semantic, &structural);
let inconsistencies = Self::detect_inconsistencies(&visual, &semantic);
ElementUnderstanding {
visual,
semantic,
structural,
combined_confidence,
inconsistencies,
}
}
fn weighted_confidence(
visual: &VisualSignal,
semantic: &SemanticSignal,
structural: &StructuralSignal,
) -> f64 {
VISUAL_WEIGHT * visual.confidence()
+ SEMANTIC_WEIGHT * semantic.confidence()
+ STRUCTURAL_WEIGHT * structural.confidence()
}
fn detect_inconsistencies(
visual: &VisualSignal,
semantic: &SemanticSignal,
) -> Vec<Inconsistency> {
let mut found = Vec::new();
if Self::looks_disabled_visually(visual) && Self::looks_enabled_semantically(semantic) {
found.push(Inconsistency::VisuallyDisabledButA11yEnabled);
}
if semantic.role.is_empty() && visual.bounds.area() > 100.0 {
found.push(Inconsistency::NotInA11yTree);
}
if Self::label_suggests_loading(&semantic.label) {
found.push(Inconsistency::LoadingState);
}
found
}
fn looks_disabled_visually(visual: &VisualSignal) -> bool {
visual.size_class == SizeClass::Small && visual.bounds.area() < 400.0
}
fn looks_enabled_semantically(semantic: &SemanticSignal) -> bool {
!semantic.role.is_empty() && semantic.role.starts_with("AX")
}
fn label_suggests_loading(label: &str) -> bool {
let lower = label.to_lowercase();
["loading", "please wait", "spinner"]
.iter()
.any(|kw| lower.contains(kw))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn medium_button_bounds() -> Rect {
Rect {
x: 10.0,
y: 20.0,
width: 80.0,
height: 30.0,
}
}
fn visual_medium() -> VisualSignal {
VisualSignal {
bounds: medium_button_bounds(),
relative_position: Position::Center,
size_class: SizeClass::Medium,
}
}
fn semantic_button(label: &str) -> SemanticSignal {
SemanticSignal {
role: "AXButton".into(),
label: label.into(),
value: None,
}
}
fn structural_leaf() -> StructuralSignal {
StructuralSignal {
depth: 3,
parent_role: "AXGroup".into(),
sibling_index: 0,
child_count: 0,
}
}
#[test]
fn rect_center_computes_midpoint() {
let r = Rect {
x: 10.0,
y: 20.0,
width: 80.0,
height: 30.0,
};
let (cx, cy) = r.center();
assert_eq!(cx, 50.0);
assert_eq!(cy, 35.0);
}
#[test]
fn rect_area_is_width_times_height() {
let r = Rect {
x: 0.0,
y: 0.0,
width: 50.0,
height: 40.0,
};
assert_eq!(r.area(), 2_000.0);
}
#[test]
fn size_class_from_rect_classifies_small_correctly() {
let r = Rect {
x: 0.0,
y: 0.0,
width: 30.0,
height: 30.0,
};
assert_eq!(SizeClass::from_rect(&r), SizeClass::Small);
}
#[test]
fn size_class_from_rect_classifies_full_width_correctly() {
let r = Rect {
x: 0.0,
y: 0.0,
width: 1920.0,
height: 1080.0,
};
assert_eq!(SizeClass::from_rect(&r), SizeClass::FullWidth);
}
#[test]
fn size_class_from_rect_classifies_medium_correctly() {
let r = Rect {
x: 0.0,
y: 0.0,
width: 80.0,
height: 30.0,
};
assert_eq!(SizeClass::from_rect(&r), SizeClass::Medium);
}
#[test]
fn visual_confidence_is_in_unit_range() {
let v = visual_medium();
let c = v.confidence();
assert!((0.0..=1.0).contains(&c));
}
#[test]
fn visual_full_width_has_higher_confidence_than_small() {
let full = VisualSignal {
bounds: Rect {
x: 0.0,
y: 0.0,
width: 1920.0,
height: 1080.0,
},
relative_position: Position::Center,
size_class: SizeClass::FullWidth,
};
let small = VisualSignal {
bounds: Rect {
x: 0.0,
y: 0.0,
width: 10.0,
height: 10.0,
},
relative_position: Position::TopLeft,
size_class: SizeClass::Small,
};
assert!(full.confidence() > small.confidence());
}
#[test]
fn semantic_empty_role_and_label_has_zero_confidence() {
let s = SemanticSignal {
role: String::new(),
label: String::new(),
value: None,
};
assert_eq!(s.confidence(), 0.0);
}
#[test]
fn semantic_with_role_and_label_has_non_zero_confidence() {
let s = semantic_button("Submit");
assert!(s.confidence() > 0.0);
}
#[test]
fn semantic_value_bonus_increases_confidence() {
let without = semantic_button("Submit");
let with_val = SemanticSignal {
role: "AXTextField".into(),
label: "Email".into(),
value: Some("user@example.com".into()),
};
assert!(with_val.confidence() > without.confidence() - 0.15);
}
#[test]
fn structural_leaf_at_depth_3_has_reasonable_confidence() {
let s = structural_leaf();
let c = s.confidence();
assert!((0.0..=1.0).contains(&c));
assert!(c > 0.5); }
#[test]
fn structural_root_has_lower_confidence_than_leaf() {
let root = StructuralSignal {
depth: 0,
parent_role: String::new(),
sibling_index: 0,
child_count: 5,
};
assert!(structural_leaf().confidence() > root.confidence());
}
#[test]
fn combined_confidence_matches_weighted_average() {
let v = visual_medium();
let s = semantic_button("Submit");
let st = structural_leaf();
let expected = VISUAL_WEIGHT * v.confidence()
+ SEMANTIC_WEIGHT * s.confidence()
+ STRUCTURAL_WEIGHT * st.confidence();
let understanding = TripleUnderstanding::build(v, s, st);
let diff = (understanding.combined_confidence - expected).abs();
assert!(diff < 1e-10, "diff={diff}");
}
#[test]
fn combined_confidence_is_in_unit_range() {
let understanding =
TripleUnderstanding::build(visual_medium(), semantic_button("OK"), structural_leaf());
assert!((0.0..=1.0).contains(&understanding.combined_confidence));
}
#[test]
fn no_inconsistencies_for_normal_button() {
let u = TripleUnderstanding::build(
visual_medium(),
semantic_button("Submit"),
structural_leaf(),
);
assert!(u.inconsistencies.is_empty());
}
#[test]
fn detects_loading_state_inconsistency() {
let u = TripleUnderstanding::build(
visual_medium(),
SemanticSignal {
role: "AXButton".into(),
label: "Loading...".into(),
value: None,
},
structural_leaf(),
);
assert!(u.inconsistencies.contains(&Inconsistency::LoadingState));
}
#[test]
fn detects_not_in_a11y_tree_when_role_empty_but_has_area() {
let u = TripleUnderstanding::build(
VisualSignal {
bounds: Rect {
x: 0.0,
y: 0.0,
width: 200.0,
height: 50.0,
},
relative_position: Position::Center,
size_class: SizeClass::Medium,
},
SemanticSignal {
role: String::new(),
label: String::new(),
value: None,
},
structural_leaf(),
);
assert!(u.inconsistencies.contains(&Inconsistency::NotInA11yTree));
}
#[test]
fn detects_visually_disabled_but_a11y_enabled() {
let u = TripleUnderstanding::build(
VisualSignal {
bounds: Rect {
x: 0.0,
y: 0.0,
width: 15.0,
height: 15.0,
},
relative_position: Position::TopLeft,
size_class: SizeClass::Small,
},
SemanticSignal {
role: "AXButton".into(),
label: "hidden".into(),
value: None,
},
structural_leaf(),
);
assert!(u
.inconsistencies
.contains(&Inconsistency::VisuallyDisabledButA11yEnabled));
}
}