car-browser 0.35.0

Browser automation and perception pipeline for Common Agent Runtime
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

//! LIVE verification — ignored by default (launches real Chrome and uses the
//! system OCR backend). Run explicitly:
//!
//!   cargo test -p car-browser --test live_chrome -- --ignored --nocapture
//!
//! Exercises the vision-augmented perception pipeline end-to-end against a real
//! browser and a crafted page:
//!   - icon-label classification (a glyph-only `☰` button → IconType::Menu),
//!   - OCR label recovery (a button with empty accessible name whose visible
//!     label is drawn on a `<canvas>` → name recovered, source Merged),
//!   - OCR invisible text (canvas text absent from the AX tree → Ocr text block).
//!
//! OCR-dependent assertions are gated on `car_vision::is_available()` so the
//! test still passes (printing a skip notice) on a machine with no OCR backend.

use car_browser::backend::BrowserBackend;
use car_browser::perception::pipeline::PerceptionPipeline;
use car_browser::perception::ui_map::{ElementSource, IconType, TextSource};
use car_browser::perception::VisionPerceptionPipeline;
use car_browser::ChromiumBackend;

const PAGE_HTML: &str = r##"<!doctype html><html><body style="margin:0;background:#fff">
<button id="b1" aria-label=""><canvas id="lbl" width="100" height="36"></canvas></button>
<button id="b2">&#9776;</button>
<canvas id="free" width="340" height="48"></canvas>
<script>
 var a=document.getElementById('lbl').getContext('2d');
 a.fillStyle='#000';a.font='24px Helvetica';a.fillText('GOLABEL',6,26);
 var b=document.getElementById('free').getContext('2d');
 b.fillStyle='#000';b.font='28px Helvetica';b.fillText('CANVASONLY',6,34);
</script>
</body></html>"##;

#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[ignore = "launches real Chrome; run with --ignored"]
async fn live_vision_perception() {
    let ocr_available = car_vision::is_available();
    eprintln!("\n=== car_vision::is_available() = {ocr_available} ===");

    // Write the page to a temp file and load it via file:// — a data: URL with
    // newlines/spaces/entities doesn't reliably parse across Chrome builds.
    let page_path = std::env::temp_dir().join("car-live-perception.html");
    std::fs::write(&page_path, PAGE_HTML).expect("write page");
    let page_url = format!("file://{}", page_path.display());

    let backend = ChromiumBackend::launch_with_viewport(1280, 720)
        .await
        .expect("launch chrome");
    backend.navigate(&page_url).await.expect("navigate");
    // Wait for the document to finish loading, then let the inline canvas
    // scripts paint before snapshotting.
    for _ in 0..40 {
        if backend.is_page_loaded().await.unwrap_or(false) {
            break;
        }
        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
    }
    tokio::time::sleep(std::time::Duration::from_millis(600)).await;

    let ax = backend.get_accessibility_tree().await.expect("ax tree");
    let shot = backend.capture_screenshot().await.expect("screenshot");
    let vp = backend.get_viewport().expect("viewport");
    let url = backend.get_current_url().expect("url");
    eprintln!("ax_nodes={}, screenshot_bytes={}", ax.len(), shot.len());

    let pipe = VisionPerceptionPipeline::new();
    let map = pipe.perceive(&shot, &ax, &url, vp).await.expect("perceive");

    eprintln!("--- format_summary ---\n{}", map.format_summary());
    eprintln!(
        "elements={} text_blocks={}",
        map.elements.len(),
        map.text_blocks.len()
    );
    for e in &map.elements {
        eprintln!(
            "  el {} role={:?} name={:?} icon={:?} src={:?} bounds=({:.0},{:.0},{:.0},{:.0}) interactable={}",
            e.id, e.role, e.name, e.icon_type, e.source,
            e.bounds.x, e.bounds.y, e.bounds.width, e.bounds.height, e.is_interactable()
        );
    }
    for t in &map.text_blocks {
        eprintln!(
            "  text[{:?}] {:?} bounds=({:.0},{:.0},{:.0},{:.0})",
            t.source, t.text, t.bounds.x, t.bounds.y, t.bounds.width, t.bounds.height
        );
    }

    let _ = backend.shutdown().await;

    // --- Assertions ---

    // (1) Icon classification works regardless of OCR: the ☰ button.
    let hamburger = map
        .elements
        .iter()
        .find(|e| e.icon_type == Some(IconType::Menu));
    assert!(
        hamburger.is_some(),
        "expected the ☰ button to classify as IconType::Menu"
    );

    if !ocr_available {
        eprintln!(
            "\n[SKIP] No OCR backend — skipping OCR label-recovery + invisible-text assertions."
        );
        return;
    }

    // (2) Label recovery: the empty-name button adopts the canvas-drawn label
    // and is marked Merged. OCR may read "GOLABEL" with minor variance, so match
    // on a stable core substring.
    let recovered = map.elements.iter().find(|e| {
        matches!(e.source, ElementSource::Merged { .. })
            && e.name
                .as_deref()
                .map(|n| n.to_uppercase().contains("GO"))
                .unwrap_or(false)
    });
    assert!(
        recovered.is_some(),
        "expected OCR to recover the GOLABEL button's name (source Merged)"
    );

    // (3) Invisible text: the free canvas text surfaces as an Ocr text block.
    let invisible = map
        .text_blocks
        .iter()
        .any(|t| t.source == TextSource::Ocr && t.text.to_uppercase().contains("CANVAS"));
    assert!(
        invisible,
        "expected the canvas-only text to surface as an Ocr text block"
    );

    eprintln!(
        "\n[OK] icon classification + OCR label recovery + invisible text all verified live."
    );
}