cel_context/
element.rs

1use serde::{Deserialize, Serialize};
2use std::collections::HashMap;
3
4/// Screen-space rectangle in pixel coordinates.
5#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
6pub struct Bounds {
7    pub x: i32,
8    pub y: i32,
9    pub width: u32,
10    pub height: u32,
11}
12
13/// UI element state flags.
14#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
15pub struct ElementState {
16    pub focused: bool,
17    pub enabled: bool,
18    pub visible: bool,
19    pub selected: bool,
20    pub expanded: Option<bool>,
21    pub checked: Option<bool>,
22}
23
24impl Default for ElementState {
25    fn default() -> Self {
26        Self {
27            focused: false,
28            enabled: true,
29            visible: true,
30            selected: false,
31            expanded: None,
32            checked: None,
33        }
34    }
35}
36
37/// A raw TCP/UDP connection observed by any source.
38#[derive(Debug, Clone, Serialize, Deserialize, Default)]
39pub struct ConnectionEvent {
40    #[serde(default)]
41    pub timestamp_ms: u64,
42    #[serde(default = "default_protocol")]
43    pub protocol: String,
44    #[serde(default)]
45    pub local_addr: String,
46    #[serde(default)]
47    pub local_port: u16,
48    #[serde(default)]
49    pub remote_addr: String,
50    #[serde(default)]
51    pub remote_port: u16,
52    #[serde(default)]
53    pub state: String,
54    pub service: Option<String>,
55    pub process_name: Option<String>,
56    pub pid: Option<u32>,
57}
58
59fn default_protocol() -> String {
60    "tcp".to_string()
61}
62
63/// A real HTTP request/response observed by CDP, proxy, or another source.
64#[derive(Debug, Clone, Serialize, Deserialize, Default)]
65pub struct HttpEvent {
66    pub timestamp_ms: u64,
67    #[serde(default)]
68    pub method: String,
69    pub url: String,
70    #[serde(alias = "status")]
71    pub status_code: Option<u16>,
72    pub content_type: Option<String>,
73    pub duration_ms: Option<f64>,
74    pub size_bytes: Option<u64>,
75    #[serde(default)]
76    pub source: String,
77}
78
79/// Generic clipboard state.
80#[derive(Debug, Clone, Serialize, Deserialize, Default)]
81pub struct ClipboardState {
82    /// Text content, usually truncated by the source for privacy.
83    pub text: Option<String>,
84    pub has_image: bool,
85    pub has_files: bool,
86}
87
88/// Generic visible-window state.
89#[derive(Debug, Clone, Serialize, Deserialize, Default)]
90pub struct WindowState {
91    pub app_name: String,
92    pub title: String,
93    pub x: i32,
94    pub y: i32,
95    pub width: u32,
96    pub height: u32,
97    pub layer: i32,
98    pub is_on_screen: bool,
99    pub pid: u32,
100}
101
102/// Generic audio output state.
103#[derive(Debug, Clone, Serialize, Deserialize, Default)]
104pub struct AudioState {
105    /// System volume, normalized to 0.0-1.0.
106    pub volume: f32,
107    pub is_muted: bool,
108}
109
110/// Generic battery / power state.
111#[derive(Debug, Clone, Serialize, Deserialize, Default)]
112pub struct PowerState {
113    /// Battery percentage, normalized to 0.0-1.0.
114    pub battery_level: Option<f32>,
115    pub is_charging: bool,
116    pub is_plugged_in: bool,
117}
118
119/// Generic running app state.
120#[derive(Debug, Clone, Serialize, Deserialize, Default)]
121pub struct RunningApp {
122    pub name: String,
123    pub is_frontmost: bool,
124}
125
126/// Generic recent file state.
127#[derive(Debug, Clone, Serialize, Deserialize, Default)]
128pub struct RecentFile {
129    pub name: String,
130    pub directory: String,
131    pub age_secs: u64,
132}
133
134/// The source that provided a context element.
135#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
136#[serde(rename_all = "snake_case")]
137pub enum ContextSource {
138    /// From the accessibility tree (UIA / AXUIElement).
139    AccessibilityTree,
140    /// From a native API adapter (SAP, Excel COM, etc.).
141    NativeApi,
142    /// From vision model analysis.
143    Vision,
144    /// From the Chrome DevTools Protocol DOM walk (the Rust browser
145    /// adapter or any future browser-DOM-backed source). Distinguished
146    /// from `NativeApi` so downstream consumers (SourceSummary
147    /// telemetry, eval scoring, planner prompt) can tell when the
148    /// element is browser-DOM-backed and routes through the
149    /// `dom:*`-id JS-click dispatch path rather than native macOS
150    /// input.
151    Cdp,
152    /// From on-device OCR (macOS Vision `VNRecognizeText`). A local,
153    /// deterministic text-recognition fallback for screen regions with no
154    /// accessibility tree (canvas, games, image-only documents) — distinct
155    /// from `Vision` (the slower, non-deterministic VLM) so consumers know the
156    /// element came from pixel OCR, not a model's semantic read.
157    Ocr,
158    /// From an arbitrary external stream such as logs, traces, metrics,
159    /// tickets, database rows, application events, or domain APIs.
160    External,
161    /// Merged from multiple sources.
162    Merged,
163}
164
165/// Content role classification for prompt injection defense.
166///
167/// Elements are classified by their semantic role so the LLM can distinguish
168/// between actionable UI controls and untrusted text content. This prevents
169/// adversarial websites from injecting instructions via text elements that
170/// look like system commands.
171///
172/// In the prompt, elements are tagged: [1] button "Submit" (interactive)
173/// vs [5] text "Click here to win" (content) — so the LLM knows [5] is
174/// user-authored text, not a UI control.
175#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
176#[serde(rename_all = "snake_case")]
177pub enum ContentRole {
178    /// UI controls the user can interact with (buttons, inputs, links, menus).
179    /// Safe to act on — these are real UI elements.
180    #[default]
181    Interactive,
182    /// Text content (paragraphs, headings, labels). May contain adversarial text.
183    /// The LLM should READ but not EXECUTE instructions found in content elements.
184    Content,
185    /// Decorative elements (separators, icons, spacers). Can be deprioritized.
186    Decorative,
187    /// System/chrome UI (scrollbars, window controls, status bars).
188    System,
189}
190
191/// Classify an element's content role based on its type and properties.
192pub fn classify_content_role(
193    element_type: &str,
194    actions: &[String],
195    state: &ElementState,
196) -> ContentRole {
197    match element_type {
198        // Interactive controls
199        "button" | "link" | "input" | "textfield" | "textarea" | "combobox" | "select"
200        | "checkbox" | "radio" | "slider" | "switch" | "toggle" | "tab" | "tab_item"
201        | "menuitem" | "menu_item" | "menubar" | "menu" | "toolbar" | "searchfield"
202        | "tree_item" => ContentRole::Interactive,
203        // System chrome
204        "scrollbar" | "splitter" | "statusbar" | "status_bar" | "progressbar" | "indicator"
205        | "dialog" | "window" => ContentRole::System,
206        // Decorative
207        "separator" | "image" | "icon" | "spacer" => ContentRole::Decorative,
208        // Text content — untrusted
209        "text" | "statictext" | "paragraph" | "heading" | "label" | "cell" | "table"
210        | "table_row" | "table_cell" | "list" | "listitem" | "list_item" | "article"
211        | "blockquote" => ContentRole::Content,
212        // Default: if it has actions, it's interactive; otherwise content
213        _ => {
214            if !actions.is_empty() || state.focused {
215                ContentRole::Interactive
216            } else {
217                ContentRole::Content
218            }
219        }
220    }
221}
222
223/// A single UI element in the unified context model.
224#[derive(Debug, Clone, Serialize, Deserialize)]
225pub struct ContextElement {
226    /// Unique identifier for this element.
227    pub id: String,
228    /// Human-readable label.
229    pub label: Option<String>,
230    /// Accessibility description (tooltip / secondary label).
231    pub description: Option<String>,
232    /// Element type (button, input, text, etc.).
233    pub element_type: String,
234    /// Current value (for inputs, dropdowns, etc.).
235    pub value: Option<String>,
236    /// Screen-space bounding rectangle.
237    pub bounds: Option<Bounds>,
238    /// Current state flags (from accessibility tree).
239    /// Defaults to all-false for sources that don't provide state (e.g., vision).
240    #[serde(default)]
241    pub state: ElementState,
242    /// ID of the parent element (None for root elements).
243    pub parent_id: Option<String>,
244    /// Available actions (from AT-SPI2 Action interface): "click", "press", "activate", etc.
245    #[serde(default, skip_serializing_if = "Vec::is_empty")]
246    pub actions: Vec<String>,
247    /// Confidence score (0.0 - 1.0).
248    pub confidence: f64,
249    /// Which context source provided this element.
250    pub source: ContextSource,
251    /// Content role for prompt injection defense.
252    /// Classifies elements as Interactive (safe to act on), Content (untrusted text),
253    /// Decorative (deprioritize), or System (chrome UI).
254    #[serde(default)]
255    pub content_role: ContentRole,
256    /// Extended properties from the accessibility or native API.
257    /// Keys: "placeholder", "url", "required", "invalid", "role_desc", "selected_text",
258    ///        "dom_id", "document", "filename", "min_value", "max_value", "has_popup",
259    ///        "column_count", "row_count", "loading_progress"
260    #[serde(
261        default,
262        skip_serializing_if = "HashMap::is_empty",
263        deserialize_with = "flexible_properties"
264    )]
265    pub properties: HashMap<String, String>,
266}
267
268/// Deserialize a HashMap where values may be strings, numbers, bools, or nested objects.
269/// Non-string values are converted to their JSON string representation.
270fn flexible_properties<'de, D>(deserializer: D) -> Result<HashMap<String, String>, D::Error>
271where
272    D: serde::Deserializer<'de>,
273{
274    let value = serde_json::Value::deserialize(deserializer)?;
275    let mut map = HashMap::new();
276    if let serde_json::Value::Object(obj) = value {
277        for (key, val) in obj {
278            let string_value = match val {
279                serde_json::Value::String(s) => s,
280                serde_json::Value::Number(n) => n.to_string(),
281                serde_json::Value::Bool(b) => b.to_string(),
282                serde_json::Value::Null => String::new(),
283                // Nested objects/arrays → JSON string
284                other => other.to_string(),
285            };
286            map.insert(key, string_value);
287        }
288    }
289    Ok(map)
290}
291
292/// Coarse spatial region for resilient element targeting.
293/// Uses normalized coordinates (0.0-1.0) so references survive resolution changes.
294#[derive(Debug, Clone, Serialize, Deserialize)]
295pub struct BoundsRegion {
296    /// Spatial quadrant: "top-left", "top-center", "top-right",
297    /// "center-left", "center", "center-right",
298    /// "bottom-left", "bottom-center", "bottom-right"
299    pub quadrant: String,
300    /// Normalized horizontal position (0.0 = left edge, 1.0 = right edge).
301    pub relative_x: f64,
302    /// Normalized vertical position (0.0 = top edge, 1.0 = bottom edge).
303    pub relative_y: f64,
304}
305
306/// A resilient, multi-signal reference to a UI element.
307/// Unlike element IDs (which are ephemeral per snapshot), references survive
308/// across context snapshots by combining multiple identifying signals.
309#[derive(Debug, Clone, Serialize, Deserialize)]
310pub struct ContextReference {
311    /// Element type (button, input, text, etc.) — must match exactly.
312    pub element_type: String,
313    /// Expected label text (fuzzy matched).
314    pub label: Option<String>,
315    /// Ancestor path from root: e.g. \["window:Finder", "toolbar", "group"\].
316    #[serde(default, skip_serializing_if = "Vec::is_empty")]
317    pub ancestor_path: Vec<String>,
318    /// Coarse spatial region where the element was last seen.
319    pub bounds_region: Option<BoundsRegion>,
320    /// Pattern the element's value should match.
321    pub value_pattern: Option<String>,
322}
323
324impl ContextElement {
325    /// Build the ancestor path by walking parent_id chains.
326    /// Returns element_types from root to parent (not including self).
327    pub(crate) fn build_ancestor_path(&self, all_elements: &[ContextElement]) -> Vec<String> {
328        let mut path = Vec::new();
329        let mut current_id = self.parent_id.as_deref();
330        let mut depth = 0;
331        while let Some(pid) = current_id {
332            if depth > 15 {
333                break;
334            }
335            if let Some(parent) = all_elements.iter().find(|e| e.id == pid) {
336                path.push(parent.element_type.clone());
337                current_id = parent.parent_id.as_deref();
338            } else {
339                break;
340            }
341            depth += 1;
342        }
343        path.reverse();
344        path
345    }
346
347    /// Build a resilient reference from this element's current data.
348    /// `screen_width` and `screen_height` are used to compute normalized coordinates.
349    pub fn to_reference(&self, screen_width: u32, screen_height: u32) -> ContextReference {
350        let bounds_region = self.bounds.as_ref().and_then(|b| {
351            if screen_width == 0 || screen_height == 0 {
352                return None;
353            }
354            let cx = b.x as f64 + b.width as f64 / 2.0;
355            let cy = b.y as f64 + b.height as f64 / 2.0;
356            let rx = cx / screen_width as f64;
357            let ry = cy / screen_height as f64;
358
359            let col = if rx < 0.33 {
360                "left"
361            } else if rx < 0.66 {
362                "center"
363            } else {
364                "right"
365            };
366            let row = if ry < 0.33 {
367                "top"
368            } else if ry < 0.66 {
369                "center"
370            } else {
371                "bottom"
372            };
373            let quadrant = if row == "center" && col == "center" {
374                "center".to_string()
375            } else {
376                format!("{}-{}", row, col)
377            };
378
379            Some(BoundsRegion {
380                quadrant,
381                relative_x: rx.clamp(0.0, 1.0),
382                relative_y: ry.clamp(0.0, 1.0),
383            })
384        });
385
386        ContextReference {
387            element_type: self.element_type.clone(),
388            label: self.label.clone(),
389            ancestor_path: Vec::new(),
390            bounds_region,
391            value_pattern: self.value.clone(),
392        }
393    }
394
395    /// Build a resilient reference with ancestor path context.
396    /// `all_elements` is the flattened element list from the ScreenContext.
397    pub fn to_reference_in_context(
398        &self,
399        screen_width: u32,
400        screen_height: u32,
401        all_elements: &[ContextElement],
402    ) -> ContextReference {
403        let mut reference = self.to_reference(screen_width, screen_height);
404        reference.ancestor_path = self.build_ancestor_path(all_elements);
405        reference
406    }
407}
408
409/// A single transcribed speech segment from the audio capture layer.
410/// Source-agnostic: may come from the microphone or system loopback.
411#[derive(Debug, Clone, Serialize, Deserialize)]
412pub struct TranscriptEntry {
413    pub text: String,
414    pub start_ms: u64,
415    pub end_ms: u64,
416    /// "microphone" | "system_output" | "both"
417    pub source: String,
418    #[serde(skip_serializing_if = "Option::is_none")]
419    pub speaker: Option<String>,
420    #[serde(skip_serializing_if = "Option::is_none")]
421    pub confidence: Option<f32>,
422}
423
424/// The complete screen context — the unified world model.
425#[derive(Debug, Clone, Serialize, Deserialize)]
426pub struct ScreenContext {
427    /// Name of the foreground application.
428    pub app: String,
429    /// Title of the active window.
430    pub window: String,
431    /// All detected UI elements, sorted by confidence (highest first).
432    pub elements: Vec<ContextElement>,
433    /// Recent network connections (TCP/UDP level — honest data from lsof or /proc).
434    #[serde(default)]
435    pub network_events: Vec<ConnectionEvent>,
436    /// Real HTTP events from CDP or proxy (never fabricated).
437    #[serde(default, skip_serializing_if = "Vec::is_empty")]
438    pub http_events: Vec<HttpEvent>,
439    /// Timestamp of this context snapshot (ms since epoch).
440    pub timestamp_ms: u64,
441    /// Screen width in pixels (used for spatial normalization in reference resolution).
442    #[serde(default, skip_serializing_if = "Option::is_none")]
443    pub screen_width: Option<u32>,
444    /// Screen height in pixels (used for spatial normalization in reference resolution).
445    #[serde(default, skip_serializing_if = "Option::is_none")]
446    pub screen_height: Option<u32>,
447    /// Clipboard state (text, has_image, has_files).
448    #[serde(default, skip_serializing_if = "Option::is_none")]
449    pub clipboard: Option<ClipboardState>,
450    /// All visible windows on screen (not just focused app).
451    #[serde(default, skip_serializing_if = "Vec::is_empty")]
452    pub window_list: Vec<WindowState>,
453    /// Audio output state (volume, muted).
454    #[serde(default, skip_serializing_if = "Option::is_none")]
455    pub audio: Option<AudioState>,
456    /// Battery/power state.
457    #[serde(default, skip_serializing_if = "Option::is_none")]
458    pub power: Option<PowerState>,
459    /// Running GUI applications.
460    #[serde(default, skip_serializing_if = "Vec::is_empty")]
461    pub running_apps: Vec<RunningApp>,
462    /// Recently created/modified files.
463    #[serde(default, skip_serializing_if = "Vec::is_empty")]
464    pub recent_files: Vec<RecentFile>,
465    /// Transcribed speech segments from any audio stream.
466    #[serde(default, skip_serializing_if = "Vec::is_empty")]
467    pub transcripts: Vec<TranscriptEntry>,
468}
469
470/// High-fidelity context for a single element — the "zoom in" view.
471#[derive(Debug, Clone, Serialize, Deserialize)]
472pub struct FocusedContext {
473    /// The target element with full detail.
474    pub element: ContextElement,
475    /// Children (preserves hierarchy, not flattened).
476    pub subtree: Vec<ContextElement>,
477    /// Parent chain from root to this element: e.g. ["window:Title", "group", "toolbar"].
478    pub ancestor_path: Vec<String>,
479}
480
481#[cfg(test)]
482mod tests {
483    use super::*;
484
485    fn default_state() -> ElementState {
486        ElementState::default()
487    }
488
489    #[test]
490    fn test_classify_interactive_elements() {
491        let types = [
492            "button",
493            "link",
494            "input",
495            "textfield",
496            "textarea",
497            "combobox",
498            "select",
499            "checkbox",
500            "radio",
501            "slider",
502            "switch",
503            "toggle",
504            "tab",
505            "menuitem",
506        ];
507        for t in types {
508            let role = classify_content_role(t, &[], &default_state());
509            assert_eq!(
510                role,
511                ContentRole::Interactive,
512                "Expected Interactive for '{}'",
513                t
514            );
515        }
516    }
517
518    #[test]
519    fn test_classify_content_elements() {
520        let types = [
521            "text",
522            "statictext",
523            "paragraph",
524            "heading",
525            "label",
526            "cell",
527            "table",
528            "table_row",
529            "table_cell",
530            "list",
531            "listitem",
532            "list_item",
533        ];
534        for t in types {
535            let role = classify_content_role(t, &[], &default_state());
536            assert_eq!(role, ContentRole::Content, "Expected Content for '{}'", t);
537        }
538    }
539
540    #[test]
541    fn test_classify_system_elements() {
542        let types = [
543            "scrollbar",
544            "splitter",
545            "statusbar",
546            "status_bar",
547            "progressbar",
548            "dialog",
549            "window",
550        ];
551        for t in types {
552            let role = classify_content_role(t, &[], &default_state());
553            assert_eq!(role, ContentRole::System, "Expected System for '{}'", t);
554        }
555    }
556
557    #[test]
558    fn test_classify_decorative_elements() {
559        let types = ["separator", "image", "icon", "spacer"];
560        for t in types {
561            let role = classify_content_role(t, &[], &default_state());
562            assert_eq!(
563                role,
564                ContentRole::Decorative,
565                "Expected Decorative for '{}'",
566                t
567            );
568        }
569    }
570
571    #[test]
572    fn test_unknown_with_actions_is_interactive() {
573        let role = classify_content_role("custom_widget", &["click".into()], &default_state());
574        assert_eq!(role, ContentRole::Interactive);
575    }
576
577    #[test]
578    fn test_unknown_without_actions_is_content() {
579        let role = classify_content_role("custom_widget", &[], &default_state());
580        assert_eq!(role, ContentRole::Content);
581    }
582
583    #[test]
584    fn test_unknown_focused_is_interactive() {
585        let mut state = default_state();
586        state.focused = true;
587        let role = classify_content_role("unknown", &[], &state);
588        assert_eq!(role, ContentRole::Interactive);
589    }
590
591    #[test]
592    fn test_content_role_default_is_interactive() {
593        assert_eq!(ContentRole::default(), ContentRole::Interactive);
594    }
595
596    #[test]
597    fn test_content_role_serialization() {
598        assert_eq!(
599            serde_json::to_string(&ContentRole::Interactive).unwrap(),
600            "\"interactive\""
601        );
602        assert_eq!(
603            serde_json::to_string(&ContentRole::Content).unwrap(),
604            "\"content\""
605        );
606    }
607}
cel_context/element.rs

cel_context/
element.rs
