cel_contracts/actions.rs
1//! Action contract — what a planner can ask the runtime to do.
2//!
3//! Lives at the cortex/planner boundary so neither side has to depend on the
4//! other to talk about actions.
5
6use serde::{Deserialize, Deserializer, Serialize};
7
8/// Deserialize a field that may be a string, object, array, or null.
9/// LLMs sometimes return structured data where a plain string is expected.
10fn deserialize_string_or_value<'de, D>(deserializer: D) -> Result<String, D::Error>
11where
12 D: Deserializer<'de>,
13{
14 let value = serde_json::Value::deserialize(deserializer)?;
15 match value {
16 serde_json::Value::String(s) => Ok(s),
17 serde_json::Value::Null => Ok(String::new()),
18 other => Ok(other.to_string()),
19 }
20}
21
22/// What the runtime should observe after an action lands to confirm the
23/// side-effect actually materialised.
24///
25/// Cortex polls the page (via CDP `Runtime.evaluate`) until the
26/// expectation holds or the timeout fires. The result is reported back
27/// in the action's `ActionResult` so the planner sees not just
28/// "dispatched ok" but "dispatched ok AND observed the expected change".
29///
30/// Closes the "click reported ok but page didn't react" gap:
31/// `e.preventDefault()` from a validation handler, a remounted DOM node
32/// the click landed on but is no longer wired up, an animation that
33/// swallowed the click — all previously reported `ok` and forced the
34/// planner to verify state from screenshots after the fact. With
35/// `expect_after` the runtime knows immediately the side-effect didn't
36/// materialise and surfaces an `EffectMissing` error to the planner.
37///
38/// All variants carry a `timeout_ms` with a sensible default
39/// (2_000 ms) — long enough for animations / debounced handlers, short
40/// enough not to add real latency on the happy path (most expectations
41/// resolve in one CDP round-trip when the action actually fired).
42#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
43#[serde(tag = "kind", rename_all = "snake_case")]
44pub enum EffectExpectation {
45 /// A new element matching `selector` becomes visible
46 /// (`offsetParent !== null`). Use for "after submit, success
47 /// message appears", "after open, modal appears" patterns.
48 SelectorAppears {
49 selector: String,
50 #[serde(default = "default_effect_timeout_ms")]
51 timeout_ms: u64,
52 },
53 /// An element matching `selector` becomes invisible (detached or
54 /// `offsetParent === null`). Use for "after close, modal disappears",
55 /// "after delete, row goes away" patterns.
56 SelectorDisappears {
57 selector: String,
58 #[serde(default = "default_effect_timeout_ms")]
59 timeout_ms: u64,
60 },
61 /// An element matching `selector` contains the given text. Use for
62 /// state changes like "button label flips from 'Approve' to
63 /// 'Approved ✓'", or "row gains an 'Approved' status cell".
64 SelectorTextContains {
65 selector: String,
66 substring: String,
67 #[serde(default = "default_effect_timeout_ms")]
68 timeout_ms: u64,
69 },
70 /// The page DOM changed in any meaningful way after the action.
71 /// Compares a "before" snapshot (captured at dispatch entry) to a
72 /// "after" snapshot polled until either differs or the timeout
73 /// fires. The snapshot is small + cheap: visible text length,
74 /// interactive element count, and current URL.
75 ///
76 /// Use when the post-state isn't a single named selector but you
77 /// know the action SHOULD cause SOME visible change — e.g.:
78 /// • a delete button that removes a row (count decreases),
79 /// • a tab switch that swaps the entire content panel,
80 /// • a submit that navigates to a thank-you page (URL changes),
81 /// • a "load more" that appends results (text length grows).
82 ///
83 /// Strictly weaker than `SelectorAppears`/`Disappears`/`TextContains`
84 /// — those tell you EXACTLY what should change, this just tells
85 /// you SOMETHING did. Use the selector-based variants when you
86 /// have a verbatim `selector="..."` from perception; fall back
87 /// to `DomChanged` when you don't.
88 ///
89 /// False-positive risk: pages with timestamp tickers / animated
90 /// counters / live data feeds produce diffs every tick. The 2s
91 /// default timeout is short enough that most non-action-triggered
92 /// changes don't have time to land — but if you're on a chatty
93 /// page, prefer a selector-based variant if you can name one.
94 DomChanged {
95 #[serde(default = "default_effect_timeout_ms")]
96 timeout_ms: u64,
97 },
98}
99
100fn default_effect_timeout_ms() -> u64 {
101 2_000
102}
103
104/// How a native-input action delivers its event relative to app focus.
105///
106/// `Foreground` is the historical CEL behavior: the dispatcher brings the
107/// target app to the front (`activate_app` / `ensure_target_app_focus`) and
108/// then posts a session-wide CGEvent at whatever is frontmost. Robust, but
109/// it steals the user's focus and is the root cause of the focus-sensitive
110/// `hybrid`-suite constraint.
111///
112/// `Background` posts the event directly to the target process via
113/// `CGEventPostToPid` without activating it (see `cel_input::background`),
114/// so the user's active window keeps focus. Not every macOS app honors
115/// background-delivered events, so the dispatcher falls back to `Foreground`
116/// when no target PID resolves or the background post is rejected.
117///
118/// Defaults to `Foreground` so existing plans and serialized payloads are
119/// unchanged. WS1 of the Peekaboo-parity plan.
120#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
121#[serde(rename_all = "snake_case")]
122#[repr(u8)]
123pub enum FocusMode {
124 /// Activate the target app, then post input. Steals focus. Default.
125 #[default]
126 Foreground,
127 /// Post input to the target PID without activating it. Non-focus-stealing.
128 Background,
129}
130
131/// The action the planner wants to execute.
132#[derive(Debug, Clone, Serialize, Deserialize)]
133#[serde(tag = "type", rename_all = "snake_case")]
134pub enum PlannedAction {
135 Click {
136 target_id: String,
137 /// Optional post-state verification. When set, the runtime
138 /// polls the page after the click and reports the action as
139 /// failed if the expectation doesn't hold within `timeout_ms`.
140 /// Closes the "click reported ok but DOM didn't react" gap.
141 #[serde(default, skip_serializing_if = "Option::is_none")]
142 expect_after: Option<EffectExpectation>,
143 },
144 Type {
145 /// Optional: if provided, clicks the element first then types.
146 /// If omitted, types into the currently focused element.
147 #[serde(default, skip_serializing_if = "Option::is_none")]
148 target_id: Option<String>,
149 text: String,
150 },
151 Key {
152 key: String,
153 },
154 KeyCombo {
155 keys: Vec<String>,
156 },
157 /// Set a value directly via the accessibility API (bypasses mouse/keyboard).
158 /// More reliable than Type for form fields where settable=true.
159 SetValue {
160 target_id: String,
161 value: String,
162 /// Optional post-state verification — see [`EffectExpectation`].
163 /// Most useful for `<select>` where setting the value should
164 /// flip an `aria-selected` attribute on the chosen option, or
165 /// for `<input type="text">` in framework-controlled forms
166 /// where the visible label should reflect the typed value.
167 #[serde(default, skip_serializing_if = "Option::is_none")]
168 expect_after: Option<EffectExpectation>,
169 },
170 Scroll {
171 dx: i32,
172 dy: i32,
173 },
174 /// Drag from one element/position to another.
175 Drag {
176 from_target_id: String,
177 to_target_id: String,
178 },
179 Wait {
180 ms: u32,
181 },
182 Custom {
183 adapter: String,
184 action: String,
185 #[serde(default)]
186 params: serde_json::Value,
187 },
188 /// Extract: read data from the current page/screen without interaction.
189 /// For read-only goals: "what are the prices?", "read the headlines", "what's on screen?"
190 /// Returns the extracted data in the `data` field — no clicking or navigation needed.
191 Extract {
192 /// What data to extract (natural language description).
193 #[serde(default, deserialize_with = "deserialize_string_or_value")]
194 goal: String,
195 /// The extracted data (filled by the planner from visible context).
196 /// Optional: Gemini 2.5 Flash sometimes omits this field.
197 #[serde(default, deserialize_with = "deserialize_string_or_value")]
198 data: String,
199 },
200 /// Batch: execute multiple simple actions in sequence without re-planning.
201 Batch {
202 actions: Vec<PlannedAction>,
203 },
204 /// Natural language action — CEL resolves the instruction to the best matching element.
205 Act {
206 instruction: String,
207 },
208 /// Terminal: goal achieved. `evidence_ids` should cite element IDs
209 /// from the current context that prove the goal was achieved.
210 Done {
211 #[serde(deserialize_with = "deserialize_string_or_value")]
212 summary: String,
213 #[serde(default)]
214 evidence_ids: Vec<String>,
215 },
216 /// Terminal: cannot proceed.
217 Fail {
218 #[serde(deserialize_with = "deserialize_string_or_value")]
219 reason: String,
220 },
221 /// Native accessibility action — more reliable than coordinate clicks for desktop apps.
222 /// Uses macOS AXUIElementPerformAction under the hood.
223 AxAction {
224 /// AX hash id from the same perception snapshot that produced
225 /// the plan. May be missing (empty / null) when the planner
226 /// only knows the visible label — the cortex falls back to
227 /// `label` + `role_hint` resolution in that case.
228 ///
229 /// Lenient deserialization accepts `null` from the LLM (some
230 /// models emit `"target_id": null` when they want label-only
231 /// dispatch) and treats it as an empty string. Without this,
232 /// the whole turn parse-errors with
233 /// `invalid type: null, expected a string` and the run dies
234 /// before the runner gets a chance to fall back.
235 #[serde(default, deserialize_with = "deserialize_string_or_value")]
236 target_id: String,
237 /// The action to perform: "click" (AXPress), "activate" (AXConfirm),
238 /// "increment", "decrement", "show_menu"
239 action: String,
240 /// Label hint for fallback element resolution. AX IDs are hashes
241 /// that include bounds + depth and therefore change whenever the
242 /// UI mutates between plan time and dispatch time. If the cortex
243 /// can't find `target_id` in the live AX tree (or `target_id`
244 /// is missing entirely), it falls back to searching for the
245 /// first visible element whose role matches `role_hint` (if
246 /// provided) and whose label equals `label`. Planner must
247 /// populate this from the same perception snapshot that
248 /// produced the target_id.
249 #[serde(default, skip_serializing_if = "Option::is_none")]
250 label: Option<String>,
251 #[serde(default, skip_serializing_if = "Option::is_none")]
252 role_hint: Option<String>,
253 /// Optional post-state verification — see [`EffectExpectation`].
254 /// Same shape as the equivalent field on `Click`/`SetValue`.
255 /// Mostly useful when the AX action lands on a browser DOM
256 /// element (the runtime routes those through CDP), so the page
257 /// state is observable via querySelector.
258 #[serde(default, skip_serializing_if = "Option::is_none")]
259 expect_after: Option<EffectExpectation>,
260 },
261 /// Activate (bring to front) a macOS application by name.
262 /// Uses `open -a` under the hood — the most reliable app switching method.
263 ActivateApp {
264 app_name: String,
265 },
266 /// Launch (start) a macOS application by name. Unlike `activate_app`, this
267 /// is about *starting* the app; with `background` it launches without
268 /// stealing focus (`open -g`). Verified by polling until the process
269 /// appears.
270 LaunchApp {
271 app_name: String,
272 /// Launch without bringing the app to the front. Defaults to false.
273 #[serde(default)]
274 background: bool,
275 },
276 /// Quit a macOS application by name, gracefully (AppleScript `quit`, like
277 /// ⌘Q). Never force-kills — an app showing an unsaved-changes dialog stays
278 /// running and the action reports failure. Verified by polling until the
279 /// process is gone.
280 QuitApp {
281 app_name: String,
282 },
283 /// Select text by dragging from one coordinate to another.
284 /// Used for text selection, highlighting, and marking tasks.
285 Select {
286 from_x: i32,
287 from_y: i32,
288 to_x: i32,
289 to_y: i32,
290 },
291 /// Execute JavaScript in the focused browser tab via Chrome DevTools Protocol.
292 /// Fastest way to interact with web pages: click elements, fill forms, extract data,
293 /// dismiss cookie banners — all in a single action.
294 CdpEval {
295 /// JavaScript expression to evaluate in the page context.
296 expression: String,
297 },
298 /// Canonical navigation action. The cortex routes this through the
299 /// browser adapter (Playwright) when one is registered + active, and
300 /// otherwise falls back to in-cortex `cel_cdp::Page.navigate` plus a
301 /// `document.readyState` poll keyed by `wait_until`.
302 ///
303 /// All three control fields are optional with sensible defaults so
304 /// the legacy `{"type":"navigate","url":"..."}` payload still parses
305 /// unchanged. Aliases `href` / `to` on `url` survive too — LLMs
306 /// gravitate toward both.
307 Navigate {
308 #[serde(alias = "href", alias = "to")]
309 url: String,
310 /// One of `"none"`, `"domcontentloaded"`, `"load"`, `"networkidle"`.
311 /// Unknown values are treated as the default. Default: `"domcontentloaded"`.
312 #[serde(default, skip_serializing_if = "Option::is_none")]
313 wait_until: Option<String>,
314 /// Upper bound for the lifecycle wait. Default: 30_000 ms.
315 #[serde(default, skip_serializing_if = "Option::is_none")]
316 timeout_ms: Option<u64>,
317 /// When true (default), the cortex fallback runs a best-effort
318 /// cookie-banner / overlay-dismiss script after the page loads.
319 /// The TS browser adapter does this unconditionally inside its
320 /// own navigate handler (process-driver.ts) so this flag only
321 /// affects the in-cortex fallback path.
322 #[serde(default, skip_serializing_if = "Option::is_none")]
323 dismiss_overlays: Option<bool>,
324 },
325 /// No-op: LLM sometimes puts notebook_writes inside the actions array.
326 /// This variant absorbs that mistake gracefully instead of causing a parse error.
327 /// The actual notebook_writes are processed from the top-level PlannedStep field.
328 #[serde(alias = "notebook_write")]
329 NotebookWrites {
330 #[serde(default)]
331 key: String,
332 #[serde(default)]
333 value: String,
334 #[serde(default)]
335 category: String,
336 },
337 /// Declarative extraction with selector fallbacks.
338 ///
339 /// Replaces the "LLM hand-writes `document.querySelector(...)` in a
340 /// loop" failure mode. Runtime tries each selector in order via
341 /// CDP `Runtime.evaluate`, parses according to `parse_as`, and on
342 /// first success writes the value into `shared_memory[name]`. The
343 /// planner supplies the scenario-specific selector knowledge as
344 /// parameters; the retry/parse machinery is generic.
345 ///
346 /// Contract with the runner: consecutive failures for the same
347 /// `name` (across turns) accumulate toward an auto-null cutoff
348 /// (see the stall/retry budget in `canonical_runner.rs`). After
349 /// the cutoff the runtime records `shared_memory[name] = null` so
350 /// the LLM stops polishing a field that the page does not surface.
351 #[serde(alias = "extract_with_fallbacks", alias = "extract_declarative")]
352 ExtractWithFallback {
353 /// Logical name for this extraction target (e.g. `"btc_price"`).
354 /// Written into `shared_memory` on success, and used by the
355 /// runner to group consecutive failures for retry budgeting.
356 name: String,
357 /// CSS selector or JS expression candidates, tried in order.
358 /// An entry may be either a plain CSS selector (runtime wraps
359 /// it into `document.querySelector(SEL)?.textContent`) or a
360 /// full JS expression starting with `function` or a recognized
361 /// prefix — the runtime auto-detects.
362 selectors: Vec<String>,
363 /// How to parse the raw string yielded by the first matching
364 /// selector. One of: `"text"`, `"float"`, `"int"`, `"html"`.
365 /// Unknown values fall back to `"text"`.
366 #[serde(default = "default_parse_as", alias = "parse", alias = "as")]
367 parse_as: String,
368 },
369 /// Deterministic spreadsheet cell writes via AppleScript (Numbers).
370 ///
371 /// Replaces the flaky keystroke recipe (`activate_app → key(arrows)
372 /// → key(Delete) → type → key(Return)`) with one atomic operation
373 /// against the document model. The keystroke recipe produced
374 /// concatenated garbage values, duplicated headers, and values
375 /// landing in the wrong cells whenever an intermediate step got
376 /// perturbed by focus drift or AX tree lag. `WriteCells` sidesteps
377 /// the entire UI event loop.
378 ///
379 /// Batch-shaped because the AppleScript spawn cost amortizes across
380 /// many cells — single-cell callers pass a length-1 `writes` vector.
381 #[serde(alias = "write_cell")]
382 WriteCells {
383 /// Target app. Currently only `"Numbers"` is implemented; other
384 /// values produce a clean runtime error so the planner can pivot.
385 #[serde(default = "default_spreadsheet_app")]
386 app: String,
387 /// Optional sheet name. `None` = first sheet of first document.
388 #[serde(default)]
389 sheet: Option<String>,
390 /// Optional table name. `None` = first table of selected sheet.
391 #[serde(default)]
392 table: Option<String>,
393 /// Writes to apply, in order.
394 writes: Vec<CellWrite>,
395 /// When true, the runtime reads each cell back after writing
396 /// and includes the readback in the step result's `data` field.
397 /// Recommended: keep `true` — verification is cheap (same
398 /// AppleScript call) and catches Numbers' value coercions.
399 #[serde(default = "default_true")]
400 verify: bool,
401 },
402 /// Deterministic spreadsheet cell reads via AppleScript (Numbers).
403 ///
404 /// Use this when the agent needs spreadsheet truth from the app
405 /// model instead of relying on the accessibility tree to expose
406 /// the values. Particularly useful for Numbers, whose AX surface
407 /// often only exposes the focused cell or formula-bar content.
408 #[serde(alias = "read_cell")]
409 ReadCells {
410 /// Target app. Currently only `"Numbers"` is implemented.
411 #[serde(default = "default_spreadsheet_app")]
412 app: String,
413 /// Optional sheet name. `None` = first sheet of first document.
414 #[serde(default)]
415 sheet: Option<String>,
416 /// Optional table name. `None` = first table of selected sheet.
417 #[serde(default)]
418 table: Option<String>,
419 /// Cells to read, in order.
420 #[serde(alias = "refs", alias = "cells", alias = "addresses")]
421 cell_refs: Vec<String>,
422 },
423 /// Native window management (WS2) — move/resize/minimize/maximize/focus a
424 /// macOS window resolved by `app` + `window_index`. The cortex routes this
425 /// to `cel_accessibility::perform_window_op` and reads geometry back into
426 /// the receipt. `op` is one of: `"move"`, `"resize"`, `"set_bounds"`,
427 /// `"minimize"`, `"unminimize"`, `"maximize"`, `"focus"`. A `preset`
428 /// (WS2.3 tiling, e.g. `"left_half"`) overrides `op` + geometry when set.
429 Window {
430 op: String,
431 /// Target app name. `None` = the frontmost app.
432 #[serde(default, skip_serializing_if = "Option::is_none")]
433 app: Option<String>,
434 /// Window index (0 = the app's frontmost window).
435 #[serde(default)]
436 window_index: usize,
437 #[serde(default, skip_serializing_if = "Option::is_none")]
438 x: Option<f64>,
439 #[serde(default, skip_serializing_if = "Option::is_none")]
440 y: Option<f64>,
441 #[serde(default, skip_serializing_if = "Option::is_none")]
442 width: Option<f64>,
443 #[serde(default, skip_serializing_if = "Option::is_none")]
444 height: Option<f64>,
445 /// Optional tiling preset (WS2.3). When set, overrides `op` + geometry.
446 #[serde(default, skip_serializing_if = "Option::is_none")]
447 preset: Option<String>,
448 /// Optional target display index (WS4). When set, presets/placement
449 /// target that display; otherwise the window's current display.
450 #[serde(default, skip_serializing_if = "Option::is_none")]
451 display: Option<usize>,
452 },
453 /// Native dialog / sheet driver (WS5) — list, click a button by title, set
454 /// a text field, or dismiss the frontmost macOS Open/Save/Print sheet or
455 /// alert. The cortex resolves the dialog's controls via the accessibility
456 /// tree. `op` is one of: `"list"`, `"click"`, `"set_field"`, `"dismiss"`.
457 Dialog {
458 op: String,
459 /// Button title to click (for `op = "click"`); case-insensitive substring.
460 #[serde(default, skip_serializing_if = "Option::is_none")]
461 button: Option<String>,
462 /// Value to set (for `op = "set_field"`).
463 #[serde(default, skip_serializing_if = "Option::is_none")]
464 value: Option<String>,
465 /// Which visible text field to set (0-based; default 0).
466 #[serde(default)]
467 field_index: usize,
468 },
469 /// Native Dock control (WS6) — list items, launch / right-click an item by
470 /// title, or toggle auto-hide. `op` is one of: `"list"`, `"launch"`,
471 /// `"right_click"`, `"hide"`, `"show"`.
472 Dock {
473 op: String,
474 /// Dock item title (for `launch` / `right_click`).
475 #[serde(default, skip_serializing_if = "Option::is_none")]
476 name: Option<String>,
477 },
478 /// Native menu-bar extras (WS7) — list system status items (Wi-Fi,
479 /// Bluetooth, Control Center, …) or click one by title. `op` is `"list"`
480 /// or `"click"`.
481 MenuExtra {
482 op: String,
483 /// Status-item title to click (for `op = "click"`), case-insensitive.
484 #[serde(default, skip_serializing_if = "Option::is_none")]
485 name: Option<String>,
486 },
487}
488
489/// One cell write inside a [`PlannedAction::WriteCells`] batch.
490#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
491pub struct CellWrite {
492 /// A1-notation cell reference, e.g. `"B2"`, `"AA17"`.
493 #[serde(alias = "ref", alias = "cell", alias = "address")]
494 pub cell_ref: String,
495 /// Value to write. Pass raw numeric strings (`"108432.50"`, not
496 /// `"$108,432.50"`); Numbers formats per the cell's display
497 /// format. Text values pass through unchanged.
498 pub value: String,
499}
500
501fn default_spreadsheet_app() -> String {
502 "Numbers".into()
503}
504
505fn default_true() -> bool {
506 true
507}
508
509fn default_parse_as() -> String {
510 "text".into()
511}
512
513impl PlannedAction {
514 /// Target element IDs this action depends on, if any. Used by the runner
515 /// to pre-validate that planned elements still exist in the fresh Cortex
516 /// context before dispatch — missing targets trigger a replan instead of
517 /// silent misfire.
518 ///
519 /// Returns an empty slice for actions with no element target (coordinate
520 /// scrolls, key events, `CdpEval`, terminal actions). `Drag` returns two
521 /// IDs (from + to). `Batch` recurses — every sub-action's targets are
522 /// collected flat, so if any one is missing the whole batch is aborted
523 /// before any side-effect lands. Sub-batches inside batches collapse the
524 /// same way.
525 pub fn target_ids(&self) -> Vec<&str> {
526 match self {
527 Self::Click { target_id, .. }
528 | Self::SetValue { target_id, .. }
529 | Self::AxAction { target_id, .. } => vec![target_id.as_str()],
530 Self::Type {
531 target_id: Some(id),
532 ..
533 } => vec![id.as_str()],
534 Self::Drag {
535 from_target_id,
536 to_target_id,
537 } => vec![from_target_id.as_str(), to_target_id.as_str()],
538 Self::Batch { actions } => actions.iter().flat_map(|a| a.target_ids()).collect(),
539 // No element-level targets:
540 Self::Type {
541 target_id: None, ..
542 }
543 | Self::Key { .. }
544 | Self::KeyCombo { .. }
545 | Self::Scroll { .. }
546 | Self::Wait { .. }
547 | Self::Custom { .. }
548 | Self::Extract { .. }
549 | Self::Act { .. }
550 | Self::Done { .. }
551 | Self::Fail { .. }
552 | Self::ActivateApp { .. }
553 | Self::LaunchApp { .. }
554 | Self::QuitApp { .. }
555 | Self::Select { .. }
556 | Self::CdpEval { .. }
557 | Self::Navigate { .. }
558 | Self::NotebookWrites { .. }
559 | Self::WriteCells { .. }
560 | Self::ReadCells { .. }
561 | Self::ExtractWithFallback { .. }
562 // Window ops target by app + window index, not an element id.
563 | Self::Window { .. }
564 // Dialog ops resolve their controls internally, not by element id.
565 | Self::Dialog { .. }
566 // Dock ops target by item title, not an element id.
567 | Self::Dock { .. }
568 // Menu extras resolve by title, not by element id.
569 | Self::MenuExtra { .. } => vec![],
570 }
571 }
572}
573
574#[cfg(test)]
575mod target_ids_tests {
576 use super::*;
577
578 #[test]
579 fn click_returns_its_target() {
580 let a = PlannedAction::Click {
581 target_id: "a11y:42".into(),
582 expect_after: None,
583 };
584 assert_eq!(a.target_ids(), vec!["a11y:42"]);
585 }
586
587 #[test]
588 fn click_without_expect_after_round_trips_omitting_field() {
589 // Back-compat: planners that don't know about `expect_after`
590 // still round-trip through the same JSON shape they always
591 // emitted (no field added on serialize, no field required on
592 // deserialize).
593 let raw = r#"{"type":"click","target_id":"dom:button:submit"}"#;
594 let a: PlannedAction = serde_json::from_str(raw).unwrap();
595 match a {
596 PlannedAction::Click {
597 target_id,
598 expect_after,
599 } => {
600 assert_eq!(target_id, "dom:button:submit");
601 assert!(expect_after.is_none());
602 }
603 _ => panic!("expected Click"),
604 }
605 let serialised = serde_json::to_string(&PlannedAction::Click {
606 target_id: "dom:button:submit".into(),
607 expect_after: None,
608 })
609 .unwrap();
610 // `skip_serializing_if = "Option::is_none"` keeps the JSON
611 // identical to the pre-`expect_after` shape.
612 assert!(!serialised.contains("expect_after"));
613 }
614
615 #[test]
616 fn click_with_selector_appears_expectation_round_trips() {
617 let raw = r##"{
618 "type": "click",
619 "target_id": "dom:button:submit",
620 "expect_after": {
621 "kind": "selector_appears",
622 "selector": "#success-message",
623 "timeout_ms": 3000
624 }
625 }"##;
626 let a: PlannedAction = serde_json::from_str(raw).unwrap();
627 match a {
628 PlannedAction::Click {
629 target_id,
630 expect_after:
631 Some(EffectExpectation::SelectorAppears {
632 selector,
633 timeout_ms,
634 }),
635 } => {
636 assert_eq!(target_id, "dom:button:submit");
637 assert_eq!(selector, "#success-message");
638 assert_eq!(timeout_ms, 3000);
639 }
640 other => panic!("expected Click with SelectorAppears, got {other:?}"),
641 }
642 }
643
644 #[test]
645 fn effect_expectation_default_timeout_when_omitted() {
646 // `timeout_ms` has a serde default of 2000ms — the planner can
647 // omit it for the common case and only override when the page
648 // is known to be slow (heavy SPA render, lazy-loaded modal).
649 let raw = r##"{"kind": "selector_appears", "selector": "#x"}"##;
650 let e: EffectExpectation = serde_json::from_str(raw).unwrap();
651 match e {
652 EffectExpectation::SelectorAppears {
653 selector,
654 timeout_ms,
655 } => {
656 assert_eq!(selector, "#x");
657 assert_eq!(timeout_ms, 2_000);
658 }
659 other => panic!("expected SelectorAppears, got {other:?}"),
660 }
661 }
662
663 #[test]
664 fn effect_expectation_all_four_variants_parse() {
665 let appears: EffectExpectation =
666 serde_json::from_str(r#"{"kind":"selector_appears","selector":".success"}"#).unwrap();
667 assert!(matches!(appears, EffectExpectation::SelectorAppears { .. }));
668 let disappears: EffectExpectation =
669 serde_json::from_str(r#"{"kind":"selector_disappears","selector":".modal.open"}"#)
670 .unwrap();
671 assert!(matches!(
672 disappears,
673 EffectExpectation::SelectorDisappears { .. }
674 ));
675 let text: EffectExpectation = serde_json::from_str(
676 r##"{"kind":"selector_text_contains","selector":"#status","substring":"Approved"}"##,
677 )
678 .unwrap();
679 assert!(matches!(
680 text,
681 EffectExpectation::SelectorTextContains { .. }
682 ));
683 // `dom_changed` is the diff-based fallback for actions whose
684 // post-state isn't a single named selector. No `selector`
685 // field — just an optional timeout.
686 let changed: EffectExpectation = serde_json::from_str(r#"{"kind":"dom_changed"}"#).unwrap();
687 match changed {
688 EffectExpectation::DomChanged { timeout_ms } => {
689 // Default timeout when omitted.
690 assert_eq!(timeout_ms, 2_000);
691 }
692 other => panic!("expected DomChanged, got {other:?}"),
693 }
694 let changed_custom: EffectExpectation =
695 serde_json::from_str(r#"{"kind":"dom_changed","timeout_ms":5000}"#).unwrap();
696 match changed_custom {
697 EffectExpectation::DomChanged { timeout_ms } => {
698 assert_eq!(timeout_ms, 5_000);
699 }
700 other => panic!("expected DomChanged with custom timeout, got {other:?}"),
701 }
702 }
703
704 #[test]
705 fn type_without_target_returns_empty() {
706 let a = PlannedAction::Type {
707 target_id: None,
708 text: "hi".into(),
709 };
710 assert!(a.target_ids().is_empty());
711 }
712
713 #[test]
714 fn drag_returns_both_endpoints() {
715 let a = PlannedAction::Drag {
716 from_target_id: "a11y:1".into(),
717 to_target_id: "a11y:2".into(),
718 };
719 assert_eq!(a.target_ids(), vec!["a11y:1", "a11y:2"]);
720 }
721
722 #[test]
723 fn batch_flattens_sub_action_targets() {
724 let a = PlannedAction::Batch {
725 actions: vec![
726 PlannedAction::CdpEval {
727 expression: "1".into(),
728 },
729 PlannedAction::Click {
730 target_id: "a11y:ghost".into(),
731 expect_after: None,
732 },
733 PlannedAction::SetValue {
734 target_id: "a11y:input".into(),
735 value: "v".into(),
736 expect_after: None,
737 },
738 ],
739 };
740 assert_eq!(a.target_ids(), vec!["a11y:ghost", "a11y:input"]);
741 }
742
743 #[test]
744 fn nested_batches_flatten() {
745 let a = PlannedAction::Batch {
746 actions: vec![PlannedAction::Batch {
747 actions: vec![PlannedAction::Click {
748 target_id: "a11y:deep".into(),
749 expect_after: None,
750 }],
751 }],
752 };
753 assert_eq!(a.target_ids(), vec!["a11y:deep"]);
754 }
755
756 #[test]
757 fn cdp_eval_and_terminals_have_no_targets() {
758 assert!(PlannedAction::CdpEval {
759 expression: "x".into()
760 }
761 .target_ids()
762 .is_empty());
763 assert!(PlannedAction::Done {
764 summary: "ok".into(),
765 evidence_ids: vec![]
766 }
767 .target_ids()
768 .is_empty());
769 assert!(PlannedAction::Wait { ms: 100 }.target_ids().is_empty());
770 }
771
772 #[test]
773 fn extract_with_fallback_round_trips() {
774 let raw = r#"{"type":"extract_with_fallback","name":"btc_price",
775 "selectors":["fin-streamer[data-field='regularMarketPrice']",
776 "[data-test='qsp-price']"],
777 "parse_as":"float"}"#;
778 let a: PlannedAction = serde_json::from_str(raw).unwrap();
779 match &a {
780 PlannedAction::ExtractWithFallback {
781 name,
782 selectors,
783 parse_as,
784 } => {
785 assert_eq!(name, "btc_price");
786 assert_eq!(selectors.len(), 2);
787 assert_eq!(parse_as, "float");
788 }
789 _ => panic!("expected ExtractWithFallback"),
790 }
791 // Has no element-level targets.
792 assert!(a.target_ids().is_empty());
793 }
794
795 #[test]
796 fn extract_with_fallback_defaults_parse_as_to_text() {
797 let raw = r#"{"type":"extract_with_fallback","name":"title",
798 "selectors":["h1"]}"#;
799 let a: PlannedAction = serde_json::from_str(raw).unwrap();
800 match a {
801 PlannedAction::ExtractWithFallback { parse_as, .. } => {
802 assert_eq!(parse_as, "text");
803 }
804 _ => panic!("expected ExtractWithFallback"),
805 }
806 }
807
808 #[test]
809 fn extract_with_fallback_accepts_parse_alias() {
810 let raw = r#"{"type":"extract_with_fallback","name":"n",
811 "selectors":["a"],"parse":"int"}"#;
812 let a: PlannedAction = serde_json::from_str(raw).unwrap();
813 match a {
814 PlannedAction::ExtractWithFallback { parse_as, .. } => {
815 assert_eq!(parse_as, "int");
816 }
817 _ => panic!("expected ExtractWithFallback"),
818 }
819 }
820
821 #[test]
822 fn read_cells_accepts_cells_alias_and_defaults_app() {
823 let raw = r#"{"type":"read_cells","cells":["A1","B2"]}"#;
824 let a: PlannedAction = serde_json::from_str(raw).unwrap();
825 match a {
826 PlannedAction::ReadCells { app, cell_refs, .. } => {
827 assert_eq!(app, "Numbers");
828 assert_eq!(cell_refs, vec!["A1", "B2"]);
829 }
830 _ => panic!("expected ReadCells"),
831 }
832 }
833
834 #[test]
835 fn ax_action_accepts_null_target_id_as_empty_string() {
836 // Some models emit `"target_id": null` when they only know
837 // the visible label and want the runtime to resolve. The
838 // previous contract treated this as a fatal parse error
839 // (`invalid type: null, expected a string`), killing the
840 // entire turn before the label-fallback dispatcher could
841 // run. Lenient deserialization converts null to an empty
842 // string; the cortex dispatcher already handles empty
843 // target_id by going straight to label resolution.
844 let raw = r#"{
845 "type": "ax_action",
846 "target_id": null,
847 "action": "click",
848 "label": "Export to Notes",
849 "role_hint": "button"
850 }"#;
851 let a: PlannedAction = serde_json::from_str(raw).unwrap();
852 match a {
853 PlannedAction::AxAction {
854 target_id,
855 action,
856 label,
857 role_hint,
858 ..
859 } => {
860 assert_eq!(target_id, "");
861 assert_eq!(action, "click");
862 assert_eq!(label.as_deref(), Some("Export to Notes"));
863 assert_eq!(role_hint.as_deref(), Some("button"));
864 }
865 _ => panic!("expected AxAction"),
866 }
867 }
868
869 #[test]
870 fn ax_action_accepts_missing_target_id_field_entirely() {
871 // Defensive: if the planner omits the field rather than
872 // explicitly sending null, the `#[serde(default)]` falls
873 // back to `String::default()` (empty string), same
874 // dispatch path.
875 let raw = r#"{
876 "type": "ax_action",
877 "action": "click",
878 "label": "Submit"
879 }"#;
880 let a: PlannedAction = serde_json::from_str(raw).unwrap();
881 match a {
882 PlannedAction::AxAction {
883 target_id, label, ..
884 } => {
885 assert_eq!(target_id, "");
886 assert_eq!(label.as_deref(), Some("Submit"));
887 }
888 _ => panic!("expected AxAction"),
889 }
890 }
891
892 #[test]
893 fn ax_action_still_accepts_explicit_target_id_string() {
894 // Regression guard: the lenient path must not break the
895 // normal case where the planner emits a real id.
896 let raw = r#"{
897 "type": "ax_action",
898 "target_id": "ax:AXButton/0x1234",
899 "action": "click"
900 }"#;
901 let a: PlannedAction = serde_json::from_str(raw).unwrap();
902 match a {
903 PlannedAction::AxAction { target_id, .. } => {
904 assert_eq!(target_id, "ax:AXButton/0x1234");
905 }
906 _ => panic!("expected AxAction"),
907 }
908 }
909
910 #[test]
911 fn navigate_legacy_payload_still_parses() {
912 // The pre-canonical wire shape — no wait/timeout/dismiss
913 // fields — must keep working unchanged. Every existing
914 // planner (LangGraph, internal tests, MCP clients on older
915 // SDKs) emits this; flipping to required fields would silently
916 // brick navigation.
917 let raw = r#"{"type":"navigate","url":"https://example.com"}"#;
918 let a: PlannedAction = serde_json::from_str(raw).unwrap();
919 match a {
920 PlannedAction::Navigate {
921 url,
922 wait_until,
923 timeout_ms,
924 dismiss_overlays,
925 } => {
926 assert_eq!(url, "https://example.com");
927 assert!(wait_until.is_none());
928 assert!(timeout_ms.is_none());
929 assert!(dismiss_overlays.is_none());
930 }
931 _ => panic!("expected Navigate"),
932 }
933 }
934
935 #[test]
936 fn navigate_extended_payload_parses_all_fields() {
937 let raw = r#"{
938 "type":"navigate",
939 "url":"https://example.com",
940 "wait_until":"load",
941 "timeout_ms":10000,
942 "dismiss_overlays":false
943 }"#;
944 let a: PlannedAction = serde_json::from_str(raw).unwrap();
945 match a {
946 PlannedAction::Navigate {
947 url,
948 wait_until,
949 timeout_ms,
950 dismiss_overlays,
951 } => {
952 assert_eq!(url, "https://example.com");
953 assert_eq!(wait_until.as_deref(), Some("load"));
954 assert_eq!(timeout_ms, Some(10_000));
955 assert_eq!(dismiss_overlays, Some(false));
956 }
957 _ => panic!("expected Navigate"),
958 }
959 }
960
961 #[test]
962 fn navigate_url_aliases_href_and_to_still_work() {
963 // The original `Navigate` variant accepted `href` and `to`
964 // aliases on the URL field for LLM ergonomics. The extended
965 // variant inherits those aliases — pin both forms to catch
966 // an accidental drop during a future refactor.
967 for raw in [
968 r#"{"type":"navigate","href":"https://example.com"}"#,
969 r#"{"type":"navigate","to":"https://example.com"}"#,
970 ] {
971 let a: PlannedAction = serde_json::from_str(raw).expect(raw);
972 match a {
973 PlannedAction::Navigate { url, .. } => {
974 assert_eq!(url, "https://example.com")
975 }
976 _ => panic!("expected Navigate for {raw}"),
977 }
978 }
979 }
980}