cel_context/
element.rs

1use serde::{Deserialize, Serialize};
2use std::collections::HashMap;
3
4/// Screen-space rectangle in pixel coordinates.
5#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
6pub struct Bounds {
7    pub x: i32,
8    pub y: i32,
9    pub width: u32,
10    pub height: u32,
11}
12
13/// UI element state flags.
14#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
15pub struct ElementState {
16    pub focused: bool,
17    pub enabled: bool,
18    pub visible: bool,
19    pub selected: bool,
20    pub expanded: Option<bool>,
21    pub checked: Option<bool>,
22}
23
24impl Default for ElementState {
25    fn default() -> Self {
26        Self {
27            focused: false,
28            enabled: true,
29            visible: true,
30            selected: false,
31            expanded: None,
32            checked: None,
33        }
34    }
35}
36
37/// A raw TCP/UDP connection observed by any source.
38#[derive(Debug, Clone, Serialize, Deserialize, Default)]
39pub struct ConnectionEvent {
40    #[serde(default)]
41    pub timestamp_ms: u64,
42    #[serde(default = "default_protocol")]
43    pub protocol: String,
44    #[serde(default)]
45    pub local_addr: String,
46    #[serde(default)]
47    pub local_port: u16,
48    #[serde(default)]
49    pub remote_addr: String,
50    #[serde(default)]
51    pub remote_port: u16,
52    #[serde(default)]
53    pub state: String,
54    pub service: Option<String>,
55    pub process_name: Option<String>,
56    pub pid: Option<u32>,
57}
58
59fn default_protocol() -> String {
60    "tcp".to_string()
61}
62
63/// A real HTTP request/response observed by CDP, proxy, or another source.
64#[derive(Debug, Clone, Serialize, Deserialize, Default)]
65pub struct HttpEvent {
66    pub timestamp_ms: u64,
67    #[serde(default)]
68    pub method: String,
69    pub url: String,
70    #[serde(alias = "status")]
71    pub status_code: Option<u16>,
72    pub content_type: Option<String>,
73    pub duration_ms: Option<f64>,
74    pub size_bytes: Option<u64>,
75    #[serde(default)]
76    pub source: String,
77}
78
79/// Generic clipboard state.
80#[derive(Debug, Clone, Serialize, Deserialize, Default)]
81pub struct ClipboardState {
82    /// Text content, usually truncated by the source for privacy.
83    pub text: Option<String>,
84    pub has_image: bool,
85    pub has_files: bool,
86}
87
88/// Generic visible-window state.
89#[derive(Debug, Clone, Serialize, Deserialize, Default)]
90pub struct WindowState {
91    pub app_name: String,
92    pub title: String,
93    pub x: i32,
94    pub y: i32,
95    pub width: u32,
96    pub height: u32,
97    pub layer: i32,
98    pub is_on_screen: bool,
99    pub pid: u32,
100}
101
102/// Generic audio output state.
103#[derive(Debug, Clone, Serialize, Deserialize, Default)]
104pub struct AudioState {
105    /// System volume, normalized to 0.0-1.0.
106    pub volume: f32,
107    pub is_muted: bool,
108}
109
110/// Generic battery / power state.
111#[derive(Debug, Clone, Serialize, Deserialize, Default)]
112pub struct PowerState {
113    /// Battery percentage, normalized to 0.0-1.0.
114    pub battery_level: Option<f32>,
115    pub is_charging: bool,
116    pub is_plugged_in: bool,
117}
118
119/// Generic running app state.
120#[derive(Debug, Clone, Serialize, Deserialize, Default)]
121pub struct RunningApp {
122    pub name: String,
123    pub is_frontmost: bool,
124}
125
126/// Generic recent file state.
127#[derive(Debug, Clone, Serialize, Deserialize, Default)]
128pub struct RecentFile {
129    pub name: String,
130    pub directory: String,
131    pub age_secs: u64,
132}
133
134/// The source that provided a context element.
135#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
136#[serde(rename_all = "snake_case")]
137pub enum ContextSource {
138    /// From the accessibility tree (UIA / AXUIElement).
139    AccessibilityTree,
140    /// From a native API adapter (SAP, Excel COM, etc.).
141    NativeApi,
142    /// From vision model analysis.
143    Vision,
144    /// From the Chrome DevTools Protocol DOM walk (the Rust browser
145    /// adapter or any future browser-DOM-backed source). Distinguished
146    /// from `NativeApi` so downstream consumers (SourceSummary
147    /// telemetry, eval scoring, planner prompt) can tell when the
148    /// element is browser-DOM-backed and routes through the
149    /// `dom:*`-id JS-click dispatch path rather than native macOS
150    /// input.
151    Cdp,
152    /// From on-device OCR (macOS Vision `VNRecognizeText`). A local,
153    /// deterministic text-recognition fallback for screen regions with no
154    /// accessibility tree (canvas, games, image-only documents) — distinct
155    /// from `Vision` (the slower, non-deterministic VLM) so consumers know the
156    /// element came from pixel OCR, not a model's semantic read.
157    Ocr,
158    /// Merged from multiple sources.
159    Merged,
160}
161
162/// Content role classification for prompt injection defense.
163///
164/// Elements are classified by their semantic role so the LLM can distinguish
165/// between actionable UI controls and untrusted text content. This prevents
166/// adversarial websites from injecting instructions via text elements that
167/// look like system commands.
168///
169/// In the prompt, elements are tagged: [1] button "Submit" (interactive)
170/// vs [5] text "Click here to win" (content) — so the LLM knows [5] is
171/// user-authored text, not a UI control.
172#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
173#[serde(rename_all = "snake_case")]
174pub enum ContentRole {
175    /// UI controls the user can interact with (buttons, inputs, links, menus).
176    /// Safe to act on — these are real UI elements.
177    #[default]
178    Interactive,
179    /// Text content (paragraphs, headings, labels). May contain adversarial text.
180    /// The LLM should READ but not EXECUTE instructions found in content elements.
181    Content,
182    /// Decorative elements (separators, icons, spacers). Can be deprioritized.
183    Decorative,
184    /// System/chrome UI (scrollbars, window controls, status bars).
185    System,
186}
187
188/// Classify an element's content role based on its type and properties.
189pub fn classify_content_role(
190    element_type: &str,
191    actions: &[String],
192    state: &ElementState,
193) -> ContentRole {
194    match element_type {
195        // Interactive controls
196        "button" | "link" | "input" | "textfield" | "textarea" | "combobox" | "select"
197        | "checkbox" | "radio" | "slider" | "switch" | "toggle" | "tab" | "tab_item"
198        | "menuitem" | "menu_item" | "menubar" | "menu" | "toolbar" | "searchfield"
199        | "tree_item" => ContentRole::Interactive,
200        // System chrome
201        "scrollbar" | "splitter" | "statusbar" | "status_bar" | "progressbar" | "indicator"
202        | "dialog" | "window" => ContentRole::System,
203        // Decorative
204        "separator" | "image" | "icon" | "spacer" => ContentRole::Decorative,
205        // Text content — untrusted
206        "text" | "statictext" | "paragraph" | "heading" | "label" | "cell" | "table"
207        | "table_row" | "table_cell" | "list" | "listitem" | "list_item" | "article"
208        | "blockquote" => ContentRole::Content,
209        // Default: if it has actions, it's interactive; otherwise content
210        _ => {
211            if !actions.is_empty() || state.focused {
212                ContentRole::Interactive
213            } else {
214                ContentRole::Content
215            }
216        }
217    }
218}
219
220/// A single UI element in the unified context model.
221#[derive(Debug, Clone, Serialize, Deserialize)]
222pub struct ContextElement {
223    /// Unique identifier for this element.
224    pub id: String,
225    /// Human-readable label.
226    pub label: Option<String>,
227    /// Accessibility description (tooltip / secondary label).
228    pub description: Option<String>,
229    /// Element type (button, input, text, etc.).
230    pub element_type: String,
231    /// Current value (for inputs, dropdowns, etc.).
232    pub value: Option<String>,
233    /// Screen-space bounding rectangle.
234    pub bounds: Option<Bounds>,
235    /// Current state flags (from accessibility tree).
236    /// Defaults to all-false for sources that don't provide state (e.g., vision).
237    #[serde(default)]
238    pub state: ElementState,
239    /// ID of the parent element (None for root elements).
240    pub parent_id: Option<String>,
241    /// Available actions (from AT-SPI2 Action interface): "click", "press", "activate", etc.
242    #[serde(default, skip_serializing_if = "Vec::is_empty")]
243    pub actions: Vec<String>,
244    /// Confidence score (0.0 - 1.0).
245    pub confidence: f64,
246    /// Which context source provided this element.
247    pub source: ContextSource,
248    /// Content role for prompt injection defense.
249    /// Classifies elements as Interactive (safe to act on), Content (untrusted text),
250    /// Decorative (deprioritize), or System (chrome UI).
251    #[serde(default)]
252    pub content_role: ContentRole,
253    /// Extended properties from the accessibility or native API.
254    /// Keys: "placeholder", "url", "required", "invalid", "role_desc", "selected_text",
255    ///        "dom_id", "document", "filename", "min_value", "max_value", "has_popup",
256    ///        "column_count", "row_count", "loading_progress"
257    #[serde(
258        default,
259        skip_serializing_if = "HashMap::is_empty",
260        deserialize_with = "flexible_properties"
261    )]
262    pub properties: HashMap<String, String>,
263}
264
265/// Deserialize a HashMap where values may be strings, numbers, bools, or nested objects.
266/// Non-string values are converted to their JSON string representation.
267fn flexible_properties<'de, D>(deserializer: D) -> Result<HashMap<String, String>, D::Error>
268where
269    D: serde::Deserializer<'de>,
270{
271    let value = serde_json::Value::deserialize(deserializer)?;
272    let mut map = HashMap::new();
273    if let serde_json::Value::Object(obj) = value {
274        for (key, val) in obj {
275            let string_value = match val {
276                serde_json::Value::String(s) => s,
277                serde_json::Value::Number(n) => n.to_string(),
278                serde_json::Value::Bool(b) => b.to_string(),
279                serde_json::Value::Null => String::new(),
280                // Nested objects/arrays → JSON string
281                other => other.to_string(),
282            };
283            map.insert(key, string_value);
284        }
285    }
286    Ok(map)
287}
288
289/// Coarse spatial region for resilient element targeting.
290/// Uses normalized coordinates (0.0-1.0) so references survive resolution changes.
291#[derive(Debug, Clone, Serialize, Deserialize)]
292pub struct BoundsRegion {
293    /// Spatial quadrant: "top-left", "top-center", "top-right",
294    /// "center-left", "center", "center-right",
295    /// "bottom-left", "bottom-center", "bottom-right"
296    pub quadrant: String,
297    /// Normalized horizontal position (0.0 = left edge, 1.0 = right edge).
298    pub relative_x: f64,
299    /// Normalized vertical position (0.0 = top edge, 1.0 = bottom edge).
300    pub relative_y: f64,
301}
302
303/// A resilient, multi-signal reference to a UI element.
304/// Unlike element IDs (which are ephemeral per snapshot), references survive
305/// across context snapshots by combining multiple identifying signals.
306#[derive(Debug, Clone, Serialize, Deserialize)]
307pub struct ContextReference {
308    /// Element type (button, input, text, etc.) — must match exactly.
309    pub element_type: String,
310    /// Expected label text (fuzzy matched).
311    pub label: Option<String>,
312    /// Ancestor path from root: e.g. \["window:Finder", "toolbar", "group"\].
313    #[serde(default, skip_serializing_if = "Vec::is_empty")]
314    pub ancestor_path: Vec<String>,
315    /// Coarse spatial region where the element was last seen.
316    pub bounds_region: Option<BoundsRegion>,
317    /// Pattern the element's value should match.
318    pub value_pattern: Option<String>,
319}
320
321impl ContextElement {
322    /// Build the ancestor path by walking parent_id chains.
323    /// Returns element_types from root to parent (not including self).
324    pub(crate) fn build_ancestor_path(&self, all_elements: &[ContextElement]) -> Vec<String> {
325        let mut path = Vec::new();
326        let mut current_id = self.parent_id.as_deref();
327        let mut depth = 0;
328        while let Some(pid) = current_id {
329            if depth > 15 {
330                break;
331            }
332            if let Some(parent) = all_elements.iter().find(|e| e.id == pid) {
333                path.push(parent.element_type.clone());
334                current_id = parent.parent_id.as_deref();
335            } else {
336                break;
337            }
338            depth += 1;
339        }
340        path.reverse();
341        path
342    }
343
344    /// Build a resilient reference from this element's current data.
345    /// `screen_width` and `screen_height` are used to compute normalized coordinates.
346    pub fn to_reference(&self, screen_width: u32, screen_height: u32) -> ContextReference {
347        let bounds_region = self.bounds.as_ref().and_then(|b| {
348            if screen_width == 0 || screen_height == 0 {
349                return None;
350            }
351            let cx = b.x as f64 + b.width as f64 / 2.0;
352            let cy = b.y as f64 + b.height as f64 / 2.0;
353            let rx = cx / screen_width as f64;
354            let ry = cy / screen_height as f64;
355
356            let col = if rx < 0.33 {
357                "left"
358            } else if rx < 0.66 {
359                "center"
360            } else {
361                "right"
362            };
363            let row = if ry < 0.33 {
364                "top"
365            } else if ry < 0.66 {
366                "center"
367            } else {
368                "bottom"
369            };
370            let quadrant = if row == "center" && col == "center" {
371                "center".to_string()
372            } else {
373                format!("{}-{}", row, col)
374            };
375
376            Some(BoundsRegion {
377                quadrant,
378                relative_x: rx.clamp(0.0, 1.0),
379                relative_y: ry.clamp(0.0, 1.0),
380            })
381        });
382
383        ContextReference {
384            element_type: self.element_type.clone(),
385            label: self.label.clone(),
386            ancestor_path: Vec::new(),
387            bounds_region,
388            value_pattern: self.value.clone(),
389        }
390    }
391
392    /// Build a resilient reference with ancestor path context.
393    /// `all_elements` is the flattened element list from the ScreenContext.
394    pub fn to_reference_in_context(
395        &self,
396        screen_width: u32,
397        screen_height: u32,
398        all_elements: &[ContextElement],
399    ) -> ContextReference {
400        let mut reference = self.to_reference(screen_width, screen_height);
401        reference.ancestor_path = self.build_ancestor_path(all_elements);
402        reference
403    }
404}
405
406/// A single transcribed speech segment from the audio capture layer.
407/// Source-agnostic: may come from the microphone or system loopback.
408#[derive(Debug, Clone, Serialize, Deserialize)]
409pub struct TranscriptEntry {
410    pub text: String,
411    pub start_ms: u64,
412    pub end_ms: u64,
413    /// "microphone" | "system_output" | "both"
414    pub source: String,
415    #[serde(skip_serializing_if = "Option::is_none")]
416    pub speaker: Option<String>,
417    #[serde(skip_serializing_if = "Option::is_none")]
418    pub confidence: Option<f32>,
419}
420
421/// The complete screen context — the unified world model.
422#[derive(Debug, Clone, Serialize, Deserialize)]
423pub struct ScreenContext {
424    /// Name of the foreground application.
425    pub app: String,
426    /// Title of the active window.
427    pub window: String,
428    /// All detected UI elements, sorted by confidence (highest first).
429    pub elements: Vec<ContextElement>,
430    /// Recent network connections (TCP/UDP level — honest data from lsof or /proc).
431    #[serde(default)]
432    pub network_events: Vec<ConnectionEvent>,
433    /// Real HTTP events from CDP or proxy (never fabricated).
434    #[serde(default, skip_serializing_if = "Vec::is_empty")]
435    pub http_events: Vec<HttpEvent>,
436    /// Timestamp of this context snapshot (ms since epoch).
437    pub timestamp_ms: u64,
438    /// Screen width in pixels (used for spatial normalization in reference resolution).
439    #[serde(default, skip_serializing_if = "Option::is_none")]
440    pub screen_width: Option<u32>,
441    /// Screen height in pixels (used for spatial normalization in reference resolution).
442    #[serde(default, skip_serializing_if = "Option::is_none")]
443    pub screen_height: Option<u32>,
444    /// Clipboard state (text, has_image, has_files).
445    #[serde(default, skip_serializing_if = "Option::is_none")]
446    pub clipboard: Option<ClipboardState>,
447    /// All visible windows on screen (not just focused app).
448    #[serde(default, skip_serializing_if = "Vec::is_empty")]
449    pub window_list: Vec<WindowState>,
450    /// Audio output state (volume, muted).
451    #[serde(default, skip_serializing_if = "Option::is_none")]
452    pub audio: Option<AudioState>,
453    /// Battery/power state.
454    #[serde(default, skip_serializing_if = "Option::is_none")]
455    pub power: Option<PowerState>,
456    /// Running GUI applications.
457    #[serde(default, skip_serializing_if = "Vec::is_empty")]
458    pub running_apps: Vec<RunningApp>,
459    /// Recently created/modified files.
460    #[serde(default, skip_serializing_if = "Vec::is_empty")]
461    pub recent_files: Vec<RecentFile>,
462    /// Transcribed speech segments from the audio capture layer (mic + loopback).
463    /// Populated by cel-cortex when an audio backend is configured.
464    #[serde(default, skip_serializing_if = "Vec::is_empty")]
465    pub transcripts: Vec<TranscriptEntry>,
466}
467
468/// High-fidelity context for a single element — the "zoom in" view.
469#[derive(Debug, Clone, Serialize, Deserialize)]
470pub struct FocusedContext {
471    /// The target element with full detail.
472    pub element: ContextElement,
473    /// Children (preserves hierarchy, not flattened).
474    pub subtree: Vec<ContextElement>,
475    /// Parent chain from root to this element: e.g. ["window:Title", "group", "toolbar"].
476    pub ancestor_path: Vec<String>,
477}
478
479#[cfg(test)]
480mod tests {
481    use super::*;
482
483    fn default_state() -> ElementState {
484        ElementState::default()
485    }
486
487    #[test]
488    fn test_classify_interactive_elements() {
489        let types = [
490            "button",
491            "link",
492            "input",
493            "textfield",
494            "textarea",
495            "combobox",
496            "select",
497            "checkbox",
498            "radio",
499            "slider",
500            "switch",
501            "toggle",
502            "tab",
503            "menuitem",
504        ];
505        for t in types {
506            let role = classify_content_role(t, &[], &default_state());
507            assert_eq!(
508                role,
509                ContentRole::Interactive,
510                "Expected Interactive for '{}'",
511                t
512            );
513        }
514    }
515
516    #[test]
517    fn test_classify_content_elements() {
518        let types = [
519            "text",
520            "statictext",
521            "paragraph",
522            "heading",
523            "label",
524            "cell",
525            "table",
526            "table_row",
527            "table_cell",
528            "list",
529            "listitem",
530            "list_item",
531        ];
532        for t in types {
533            let role = classify_content_role(t, &[], &default_state());
534            assert_eq!(role, ContentRole::Content, "Expected Content for '{}'", t);
535        }
536    }
537
538    #[test]
539    fn test_classify_system_elements() {
540        let types = [
541            "scrollbar",
542            "splitter",
543            "statusbar",
544            "status_bar",
545            "progressbar",
546            "dialog",
547            "window",
548        ];
549        for t in types {
550            let role = classify_content_role(t, &[], &default_state());
551            assert_eq!(role, ContentRole::System, "Expected System for '{}'", t);
552        }
553    }
554
555    #[test]
556    fn test_classify_decorative_elements() {
557        let types = ["separator", "image", "icon", "spacer"];
558        for t in types {
559            let role = classify_content_role(t, &[], &default_state());
560            assert_eq!(
561                role,
562                ContentRole::Decorative,
563                "Expected Decorative for '{}'",
564                t
565            );
566        }
567    }
568
569    #[test]
570    fn test_unknown_with_actions_is_interactive() {
571        let role = classify_content_role("custom_widget", &["click".into()], &default_state());
572        assert_eq!(role, ContentRole::Interactive);
573    }
574
575    #[test]
576    fn test_unknown_without_actions_is_content() {
577        let role = classify_content_role("custom_widget", &[], &default_state());
578        assert_eq!(role, ContentRole::Content);
579    }
580
581    #[test]
582    fn test_unknown_focused_is_interactive() {
583        let mut state = default_state();
584        state.focused = true;
585        let role = classify_content_role("unknown", &[], &state);
586        assert_eq!(role, ContentRole::Interactive);
587    }
588
589    #[test]
590    fn test_content_role_default_is_interactive() {
591        assert_eq!(ContentRole::default(), ContentRole::Interactive);
592    }
593
594    #[test]
595    fn test_content_role_serialization() {
596        assert_eq!(
597            serde_json::to_string(&ContentRole::Interactive).unwrap(),
598            "\"interactive\""
599        );
600        assert_eq!(
601            serde_json::to_string(&ContentRole::Content).unwrap(),
602            "\"content\""
603        );
604    }
605}
cel_context/element.rs

cel_context/
element.rs
