cel_context/
merge.rs

1use crate::element::{ContentRole, ContextElement, ContextSnapshot, ContextSource, FocusedContext};
2
3/// High-level status for streams that contributed to a context snapshot.
4#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
5pub struct StreamStatus {
6    pub accessibility: bool,
7    pub display: bool,
8    pub network: bool,
9    pub signals: bool,
10    pub vision: bool,
11    pub audio_capture: bool,
12}
13
14/// Generic source contribution accepted by [`ContextMerger`].
15#[derive(Debug, Clone, Default)]
16pub struct ContextContribution {
17    pub source_name: String,
18    pub elements: Vec<ContextElement>,
19    pub app: Option<String>,
20    pub window: Option<String>,
21    pub stream_status: StreamStatus,
22}
23
24impl ContextContribution {
25    pub fn new(source_name: impl Into<String>, elements: Vec<ContextElement>) -> Self {
26        Self {
27            source_name: source_name.into(),
28            elements,
29            app: None,
30            window: None,
31            stream_status: StreamStatus::default(),
32        }
33    }
34
35    pub fn with_app(mut self, app: impl Into<String>) -> Self {
36        self.app = Some(app.into());
37        self
38    }
39
40    pub fn with_window(mut self, window: impl Into<String>) -> Self {
41        self.window = Some(window.into());
42        self
43    }
44
45    pub fn with_stream_status(mut self, status: StreamStatus) -> Self {
46        self.stream_status = status;
47        self
48    }
49}
50
51/// Generic merger for already-normalized context contributions.
52///
53/// `cel-context` deliberately does not know how to read accessibility, CDP,
54/// OCR, network, logs, traces, metrics, tickets, or other streams. Those
55/// sources should emit [`ContextElement`]s / [`ContextSnapshot`]s and feed them
56/// here.
57#[derive(Debug, Clone)]
58pub struct ContextMerger {
59    contributions: Vec<ContextContribution>,
60    default_app: String,
61    default_window: String,
62}
63
64impl Default for ContextMerger {
65    fn default() -> Self {
66        Self::new()
67    }
68}
69
70impl ContextMerger {
71    pub fn new() -> Self {
72        Self {
73            contributions: Vec::new(),
74            default_app: String::new(),
75            default_window: String::new(),
76        }
77    }
78
79    /// Compatibility constructor for runtimes that still wire concrete source
80    /// crates externally. `cel-context` ignores the source object and only
81    /// merges already-normalized contributions.
82    pub fn with_all<A, D, N>(_accessibility: A, _display: D, _network: N) -> Self {
83        Self::new()
84    }
85
86    /// Compatibility constructor for a runtime-provided display source.
87    pub fn with_display<A, D>(_accessibility: A, _display: D) -> Self {
88        Self::new()
89    }
90
91    /// Compatibility hook for a runtime-provided signal source.
92    pub fn with_signals<S>(self, _signals: S) -> Self {
93        self
94    }
95
96    /// Compatibility hook for a runtime-provided vision source.
97    pub fn with_vision<V>(self, _vision: V) -> Self {
98        self
99    }
100
101    /// Compatibility hook for a runtime-provided async runtime handle.
102    pub fn with_runtime<R>(self, _runtime: R) -> Self {
103        self
104    }
105
106    /// Compatibility hook for a runtime-provided screenshot fallback.
107    pub fn with_cdp_screenshot_fallback<F, T>(self, _fallback: F) -> Self
108    where
109        F: Fn() -> Option<T> + Send + Sync + 'static,
110    {
111        self
112    }
113
114    /// Compatibility hook retained until the live runtime owns OCR fallback.
115    pub fn run_ocr_fallback(&mut self, _existing: &[ContextElement]) -> Vec<ContextElement> {
116        Vec::new()
117    }
118
119    /// Network events are carried on [`ContextSnapshot`], not owned by the merger.
120    pub fn recent_network_events(&self) -> &[crate::ConnectionEvent] {
121        &[]
122    }
123
124    pub fn with_defaults(mut self, app: impl Into<String>, window: impl Into<String>) -> Self {
125        self.default_app = app.into();
126        self.default_window = window.into();
127        self
128    }
129
130    pub fn push(&mut self, contribution: ContextContribution) {
131        self.contributions.push(contribution);
132    }
133
134    pub fn extend(&mut self, contributions: impl IntoIterator<Item = ContextContribution>) {
135        self.contributions.extend(contributions);
136    }
137
138    pub fn stream_status(&self) -> StreamStatus {
139        self.contributions
140            .iter()
141            .fold(StreamStatus::default(), |mut acc, c| {
142                acc.accessibility |= c.stream_status.accessibility;
143                acc.display |= c.stream_status.display;
144                acc.network |= c.stream_status.network;
145                acc.signals |= c.stream_status.signals;
146                acc.vision |= c.stream_status.vision;
147                acc.audio_capture |= c.stream_status.audio_capture;
148                acc
149            })
150    }
151
152    pub fn build(&self) -> ContextSnapshot {
153        let app = self
154            .contributions
155            .iter()
156            .find_map(|c| c.app.clone())
157            .unwrap_or_else(|| self.default_app.clone());
158        let window = self
159            .contributions
160            .iter()
161            .find_map(|c| c.window.clone())
162            .unwrap_or_else(|| self.default_window.clone());
163
164        let elements = self
165            .contributions
166            .iter()
167            .flat_map(|c| c.elements.clone())
168            .collect();
169
170        build_from_external(elements, Vec::new(), app, window)
171    }
172
173    /// Backwards-compatible alias for callers that think of merging as a read.
174    pub fn get_context(&mut self) -> ContextSnapshot {
175        self.build()
176    }
177
178    pub fn get_context_focused(&mut self, element_id: &str) -> Option<FocusedContext> {
179        let context = self.build();
180        let element = context
181            .elements
182            .iter()
183            .find(|element| element.id == element_id)?
184            .clone();
185        let subtree = context
186            .elements
187            .iter()
188            .filter(|candidate| candidate.parent_id.as_deref() == Some(element_id))
189            .cloned()
190            .collect();
191        let ancestor_path = element.build_ancestor_path(&context.elements);
192
193        Some(FocusedContext {
194            element,
195            subtree,
196            ancestor_path,
197        })
198    }
199}
200
201/// Convert a WAI-ARIA / DOM role into CEL's normalized element type strings.
202pub fn aria_role_to_cel_type(role: &str) -> &'static str {
203    match role {
204        "button" => "button",
205        "link" => "link",
206        "textbox" | "searchbox" => "input",
207        "checkbox" => "checkbox",
208        "radio" => "radio_button",
209        "combobox" | "listbox" => "combobox",
210        "option" => "list_item",
211        "menuitem" => "menu_item",
212        "tab" => "tab_item",
213        "slider" => "slider",
214        "heading" => "heading",
215        "img" | "image" => "image",
216        "table" => "table",
217        "row" => "table_row",
218        "cell" | "gridcell" => "table_cell",
219        "dialog" => "dialog",
220        "navigation" | "banner" | "contentinfo" => "group",
221        _ => "unknown",
222    }
223}
224
225pub fn assign_default_actions(element_type: &str) -> Vec<String> {
226    match element_type {
227        "button" | "link" | "checkbox" | "radio_button" | "menu_item" | "tab_item"
228        | "list_item" | "tree_item" => vec!["click".into()],
229        "input" | "textarea" | "textfield" | "searchfield" => {
230            vec!["focus".into(), "set_value".into()]
231        }
232        "combobox" | "select" => vec!["click".into(), "select".into()],
233        "slider" => vec!["set_value".into()],
234        _ => Vec::new(),
235    }
236}
237
238pub fn build_from_external(
239    mut elements: Vec<ContextElement>,
240    http_events: Vec<crate::HttpEvent>,
241    app: String,
242    window: String,
243) -> ContextSnapshot {
244    for e in &mut elements {
245        e.element_type = aria_role_to_cel_type(&e.element_type).to_string();
246        if e.actions.is_empty() {
247            e.actions = assign_default_actions(&e.element_type);
248        }
249        e.content_role = crate::classify_content_role(&e.element_type, &e.actions, &e.state);
250        e.confidence = score_element_confidence(e);
251    }
252
253    elements.sort_by(|a, b| {
254        b.confidence
255            .partial_cmp(&a.confidence)
256            .unwrap_or(std::cmp::Ordering::Equal)
257    });
258
259    ContextSnapshot {
260        app,
261        window,
262        elements,
263        network_events: Vec::new(),
264        http_events,
265        timestamp_ms: now_ms(),
266        screen_width: None,
267        screen_height: None,
268        clipboard: None,
269        window_list: Vec::new(),
270        audio: None,
271        power: None,
272        running_apps: Vec::new(),
273        recent_files: Vec::new(),
274        transcripts: Vec::new(),
275    }
276}
277
278pub fn score_element_confidence(element: &ContextElement) -> f64 {
279    let mut score: f64 = match element.source {
280        ContextSource::NativeApi | ContextSource::Cdp => 0.95,
281        ContextSource::AccessibilityTree => 0.85,
282        ContextSource::External => 0.8,
283        ContextSource::Ocr => 0.75,
284        ContextSource::Vision => 0.65,
285        ContextSource::Merged => 0.9,
286    };
287
288    if element
289        .label
290        .as_deref()
291        .is_some_and(|s| !s.trim().is_empty())
292    {
293        score += 0.03;
294    }
295    if element.bounds.is_some() {
296        score += 0.02;
297    }
298    if is_actionable_type(&element.element_type) && !element.actions.is_empty() {
299        score += 0.03;
300    }
301    if matches!(element.content_role, ContentRole::Content) {
302        score -= 0.02;
303    }
304
305    score.clamp(0.0, 0.99)
306}
307
308pub fn is_actionable_type(element_type: &str) -> bool {
309    matches!(
310        element_type,
311        "button"
312            | "input"
313            | "textfield"
314            | "textarea"
315            | "searchfield"
316            | "link"
317            | "checkbox"
318            | "radio_button"
319            | "combobox"
320            | "select"
321            | "menu_item"
322            | "tab_item"
323            | "slider"
324            | "list_item"
325            | "tree_item"
326    )
327}
328
329fn now_ms() -> u64 {
330    std::time::SystemTime::now()
331        .duration_since(std::time::UNIX_EPOCH)
332        .unwrap_or_default()
333        .as_millis() as u64
334}
335
336#[cfg(test)]
337mod tests {
338    use super::*;
339    use crate::{Bounds, ElementState};
340    use std::collections::HashMap;
341
342    fn element(id: &str, source: ContextSource) -> ContextElement {
343        ContextElement {
344            id: id.into(),
345            label: Some("Submit".into()),
346            description: None,
347            element_type: "button".into(),
348            value: None,
349            bounds: Some(Bounds {
350                x: 0,
351                y: 0,
352                width: 100,
353                height: 30,
354            }),
355            state: ElementState::default(),
356            parent_id: None,
357            actions: Vec::new(),
358            confidence: 0.0,
359            source,
360            content_role: ContentRole::Interactive,
361            properties: HashMap::new(),
362        }
363    }
364
365    #[test]
366    fn external_elements_are_normalized() {
367        let ctx = build_from_external(
368            vec![element("a", ContextSource::NativeApi)],
369            Vec::new(),
370            "App".into(),
371            "Window".into(),
372        );
373
374        assert_eq!(ctx.app, "App");
375        assert_eq!(ctx.window, "Window");
376        assert_eq!(ctx.elements[0].actions, vec!["click"]);
377        assert!(ctx.elements[0].confidence > 0.9);
378    }
379
380    #[test]
381    fn merger_combines_contributions() {
382        let mut merger = ContextMerger::new().with_defaults("App", "Window");
383        merger.push(ContextContribution::new(
384            "native",
385            vec![element("a", ContextSource::NativeApi)],
386        ));
387        merger.push(ContextContribution::new(
388            "ocr",
389            vec![element("b", ContextSource::Ocr)],
390        ));
391
392        let ctx = merger.build();
393        assert_eq!(ctx.elements.len(), 2);
394    }
395}
cel_context/merge.rs

cel_context/
merge.rs
