cel_contracts/actions.rs
1//! Action contract — what a planner can ask a runtime to do.
2//!
3//! Lives at the planner/runtime boundary so neither side has to depend on the
4//! other to talk about actions.
5
6use serde::{Deserialize, Deserializer, Serialize};
7
8/// Deserialize a field that may be a string, object, array, or null.
9/// LLMs sometimes return structured data where a plain string is expected.
10fn deserialize_string_or_value<'de, D>(deserializer: D) -> Result<String, D::Error>
11where
12 D: Deserializer<'de>,
13{
14 let value = serde_json::Value::deserialize(deserializer)?;
15 match value {
16 serde_json::Value::String(s) => Ok(s),
17 serde_json::Value::Null => Ok(String::new()),
18 other => Ok(other.to_string()),
19 }
20}
21
22/// What the runtime should observe after an action lands to confirm the
23/// side-effect actually materialised.
24///
25/// A runtime can poll the page (for example via CDP `Runtime.evaluate`) until
26/// the expectation holds or the timeout fires. The result is reported back in
27/// the action result so the planner sees not just
28/// "dispatched ok" but "dispatched ok AND observed the expected change".
29///
30/// Closes the "click reported ok but page didn't react" gap:
31/// `e.preventDefault()` from a validation handler, a remounted DOM node
32/// the click landed on but is no longer wired up, an animation that
33/// swallowed the click — all previously reported `ok` and forced the
34/// planner to verify state from screenshots after the fact. With
35/// `expect_after` the runtime knows immediately the side-effect didn't
36/// materialise and surfaces an `EffectMissing` error to the planner.
37///
38/// All variants carry a `timeout_ms` with a sensible default
39/// (2_000 ms) — long enough for animations / debounced handlers, short
40/// enough not to add real latency on the happy path (most expectations
41/// resolve in one CDP round-trip when the action actually fired).
42#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
43#[serde(tag = "kind", rename_all = "snake_case")]
44pub enum EffectExpectation {
45 /// A new element matching `selector` becomes visible
46 /// (`offsetParent !== null`). Use for "after submit, success
47 /// message appears", "after open, modal appears" patterns.
48 SelectorAppears {
49 selector: String,
50 #[serde(default = "default_effect_timeout_ms")]
51 timeout_ms: u64,
52 },
53 /// An element matching `selector` becomes invisible (detached or
54 /// `offsetParent === null`). Use for "after close, modal disappears",
55 /// "after delete, row goes away" patterns.
56 SelectorDisappears {
57 selector: String,
58 #[serde(default = "default_effect_timeout_ms")]
59 timeout_ms: u64,
60 },
61 /// An element matching `selector` contains the given text. Use for
62 /// state changes like "button label flips from 'Approve' to
63 /// 'Approved ✓'", or "row gains an 'Approved' status cell".
64 SelectorTextContains {
65 selector: String,
66 substring: String,
67 #[serde(default = "default_effect_timeout_ms")]
68 timeout_ms: u64,
69 },
70 /// The page DOM changed in any meaningful way after the action.
71 /// Compares a "before" snapshot (captured at dispatch entry) to a
72 /// "after" snapshot polled until either differs or the timeout
73 /// fires. The snapshot is small + cheap: visible text length,
74 /// interactive element count, and current URL.
75 ///
76 /// Use when the post-state isn't a single named selector but you
77 /// know the action SHOULD cause SOME visible change — e.g.:
78 /// • a delete button that removes a row (count decreases),
79 /// • a tab switch that swaps the entire content panel,
80 /// • a submit that navigates to a thank-you page (URL changes),
81 /// • a "load more" that appends results (text length grows).
82 ///
83 /// Strictly weaker than `SelectorAppears`/`Disappears`/`TextContains`
84 /// — those tell you EXACTLY what should change, this just tells
85 /// you SOMETHING did. Use the selector-based variants when you
86 /// have a verbatim `selector="..."` from perception; fall back
87 /// to `DomChanged` when you don't.
88 ///
89 /// False-positive risk: pages with timestamp tickers / animated
90 /// counters / live data feeds produce diffs every tick. The 2s
91 /// default timeout is short enough that most non-action-triggered
92 /// changes don't have time to land — but if you're on a chatty
93 /// page, prefer a selector-based variant if you can name one.
94 DomChanged {
95 #[serde(default = "default_effect_timeout_ms")]
96 timeout_ms: u64,
97 },
98}
99
100fn default_effect_timeout_ms() -> u64 {
101 2_000
102}
103
104/// How a native-input action delivers its event relative to app focus.
105///
106/// `Foreground` means the dispatcher brings the target app to the front and
107/// then posts a session-wide input event at whatever is frontmost. Robust, but
108/// it steals the user's focus.
109///
110/// `Background` posts the event directly to the target process via
111/// `CGEventPostToPid` without activating it, so the user's active window keeps
112/// focus. Not every macOS app honors
113/// background-delivered events, so the dispatcher falls back to `Foreground`
114/// when no target PID resolves or the background post is rejected.
115///
116/// Defaults to `Foreground` so existing plans and serialized payloads are
117/// unchanged.
118#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
119#[serde(rename_all = "snake_case")]
120#[repr(u8)]
121pub enum FocusMode {
122 /// Activate the target app, then post input. Steals focus. Default.
123 #[default]
124 Foreground,
125 /// Post input to the target PID without activating it. Non-focus-stealing.
126 Background,
127}
128
129/// The action the planner wants to execute.
130#[derive(Debug, Clone, Serialize, Deserialize)]
131#[serde(tag = "type", rename_all = "snake_case")]
132pub enum PlannedAction {
133 Click {
134 target_id: String,
135 /// Optional post-state verification. When set, the runtime
136 /// polls the page after the click and reports the action as
137 /// failed if the expectation doesn't hold within `timeout_ms`.
138 /// Closes the "click reported ok but DOM didn't react" gap.
139 #[serde(default, skip_serializing_if = "Option::is_none")]
140 expect_after: Option<EffectExpectation>,
141 },
142 Type {
143 /// Optional: if provided, clicks the element first then types.
144 /// If omitted, types into the currently focused element.
145 #[serde(default, skip_serializing_if = "Option::is_none")]
146 target_id: Option<String>,
147 text: String,
148 },
149 Key {
150 key: String,
151 },
152 KeyCombo {
153 keys: Vec<String>,
154 },
155 /// Set a value directly via the accessibility API (bypasses mouse/keyboard).
156 /// More reliable than Type for form fields where settable=true.
157 SetValue {
158 target_id: String,
159 value: String,
160 /// Optional post-state verification — see [`EffectExpectation`].
161 /// Most useful for `<select>` where setting the value should
162 /// flip an `aria-selected` attribute on the chosen option, or
163 /// for `<input type="text">` in framework-controlled forms
164 /// where the visible label should reflect the typed value.
165 #[serde(default, skip_serializing_if = "Option::is_none")]
166 expect_after: Option<EffectExpectation>,
167 },
168 Scroll {
169 dx: i32,
170 dy: i32,
171 },
172 /// Drag from one element/position to another.
173 Drag {
174 from_target_id: String,
175 to_target_id: String,
176 },
177 Wait {
178 ms: u32,
179 },
180 Custom {
181 adapter: String,
182 action: String,
183 #[serde(default)]
184 params: serde_json::Value,
185 },
186 /// Extract: read data from the current page/screen without interaction.
187 /// For read-only goals: "what are the prices?", "read the headlines", "what's on screen?"
188 /// Returns the extracted data in the `data` field — no clicking or navigation needed.
189 Extract {
190 /// What data to extract (natural language description).
191 #[serde(default, deserialize_with = "deserialize_string_or_value")]
192 goal: String,
193 /// The extracted data (filled by the planner from visible context).
194 /// Optional: Gemini 2.5 Flash sometimes omits this field.
195 #[serde(default, deserialize_with = "deserialize_string_or_value")]
196 data: String,
197 },
198 /// Batch: execute multiple simple actions in sequence without re-planning.
199 Batch {
200 actions: Vec<PlannedAction>,
201 },
202 /// Natural language action — a runtime resolves the instruction to the best matching element.
203 Act {
204 instruction: String,
205 },
206 /// Terminal: goal achieved. `evidence_ids` should cite element IDs
207 /// from the current context that prove the goal was achieved.
208 Done {
209 #[serde(deserialize_with = "deserialize_string_or_value")]
210 summary: String,
211 #[serde(default)]
212 evidence_ids: Vec<String>,
213 },
214 /// Terminal: cannot proceed.
215 Fail {
216 #[serde(deserialize_with = "deserialize_string_or_value")]
217 reason: String,
218 },
219 /// Native accessibility action — more reliable than coordinate clicks for desktop apps.
220 /// Uses macOS AXUIElementPerformAction under the hood.
221 AxAction {
222 /// AX hash id from the same perception snapshot that produced
223 /// the plan. May be missing (empty / null) when the planner
224 /// only knows the visible label — the runtime can fall back to
225 /// `label` + `role_hint` resolution in that case.
226 ///
227 /// Lenient deserialization accepts `null` from the LLM (some
228 /// models emit `"target_id": null` when they want label-only
229 /// dispatch) and treats it as an empty string. Without this,
230 /// the whole turn parse-errors with
231 /// `invalid type: null, expected a string` and the run dies
232 /// before the runner gets a chance to fall back.
233 #[serde(default, deserialize_with = "deserialize_string_or_value")]
234 target_id: String,
235 /// The action to perform: "click" (AXPress), "activate" (AXConfirm),
236 /// "increment", "decrement", "show_menu"
237 action: String,
238 /// Label hint for fallback element resolution. AX IDs are hashes
239 /// that include bounds + depth and therefore change whenever the
240 /// UI mutates between plan time and dispatch time. If the runtime
241 /// can't find `target_id` in the live AX tree (or `target_id`
242 /// is missing entirely), it falls back to searching for the
243 /// first visible element whose role matches `role_hint` (if
244 /// provided) and whose label equals `label`. Planner must
245 /// populate this from the same perception snapshot that
246 /// produced the target_id.
247 #[serde(default, skip_serializing_if = "Option::is_none")]
248 label: Option<String>,
249 #[serde(default, skip_serializing_if = "Option::is_none")]
250 role_hint: Option<String>,
251 /// Optional post-state verification — see [`EffectExpectation`].
252 /// Same shape as the equivalent field on `Click`/`SetValue`.
253 /// Mostly useful when the AX action lands on a browser DOM
254 /// element (the runtime routes those through CDP), so the page
255 /// state is observable via querySelector.
256 #[serde(default, skip_serializing_if = "Option::is_none")]
257 expect_after: Option<EffectExpectation>,
258 },
259 /// Activate (bring to front) a macOS application by name.
260 /// Uses `open -a` under the hood — the most reliable app switching method.
261 ActivateApp {
262 app_name: String,
263 },
264 /// Launch (start) a macOS application by name. Unlike `activate_app`, this
265 /// is about *starting* the app; with `background` it launches without
266 /// stealing focus (`open -g`). Verified by polling until the process
267 /// appears.
268 LaunchApp {
269 app_name: String,
270 /// Launch without bringing the app to the front. Defaults to false.
271 #[serde(default)]
272 background: bool,
273 },
274 /// Quit a macOS application by name, gracefully (AppleScript `quit`, like
275 /// ⌘Q). Never force-kills — an app showing an unsaved-changes dialog stays
276 /// running and the action reports failure. Verified by polling until the
277 /// process is gone.
278 QuitApp {
279 app_name: String,
280 },
281 /// Select text by dragging from one coordinate to another.
282 /// Used for text selection, highlighting, and marking tasks.
283 Select {
284 from_x: i32,
285 from_y: i32,
286 to_x: i32,
287 to_y: i32,
288 },
289 /// Execute JavaScript in the focused browser tab via Chrome DevTools Protocol.
290 /// Fastest way to interact with web pages: click elements, fill forms, extract data,
291 /// dismiss cookie banners — all in a single action.
292 CdpEval {
293 /// JavaScript expression to evaluate in the page context.
294 expression: String,
295 },
296 /// Canonical navigation action. A runtime may route this through a browser
297 /// adapter, CDP, WebDriver, or another navigation backend and then wait
298 /// according to `wait_until`.
299 ///
300 /// All three control fields are optional with sensible defaults so
301 /// the legacy `{"type":"navigate","url":"..."}` payload still parses
302 /// unchanged. Aliases `href` / `to` on `url` survive too — LLMs
303 /// gravitate toward both.
304 Navigate {
305 #[serde(alias = "href", alias = "to")]
306 url: String,
307 /// One of `"none"`, `"domcontentloaded"`, `"load"`, `"networkidle"`.
308 /// Unknown values are treated as the default. Default: `"domcontentloaded"`.
309 #[serde(default, skip_serializing_if = "Option::is_none")]
310 wait_until: Option<String>,
311 /// Upper bound for the lifecycle wait. Default: 30_000 ms.
312 #[serde(default, skip_serializing_if = "Option::is_none")]
313 timeout_ms: Option<u64>,
314 /// When true (default), the runtime may run a best-effort cookie-banner
315 /// / overlay-dismiss script after the page loads.
316 #[serde(default, skip_serializing_if = "Option::is_none")]
317 dismiss_overlays: Option<bool>,
318 },
319 /// No-op: LLM sometimes puts notebook_writes inside the actions array.
320 /// This variant absorbs that mistake gracefully instead of causing a parse error.
321 /// The actual notebook_writes are processed from the top-level PlannedStep field.
322 #[serde(alias = "notebook_write")]
323 NotebookWrites {
324 #[serde(default)]
325 key: String,
326 #[serde(default)]
327 value: String,
328 #[serde(default)]
329 category: String,
330 },
331 /// Declarative extraction with selector fallbacks.
332 ///
333 /// Replaces the "LLM hand-writes `document.querySelector(...)` in a
334 /// loop" failure mode. Runtime tries each selector in order via
335 /// CDP `Runtime.evaluate`, parses according to `parse_as`, and on
336 /// first success writes the value into `shared_memory[name]`. The
337 /// planner supplies the scenario-specific selector knowledge as
338 /// parameters; the retry/parse machinery is generic.
339 ///
340 /// Contract with the runner: consecutive failures for the same
341 /// `name` (across turns) accumulate toward an auto-null cutoff
342 /// (see the stall/retry budget in `canonical_runner.rs`). After
343 /// the cutoff the runtime records `shared_memory[name] = null` so
344 /// the LLM stops polishing a field that the page does not surface.
345 #[serde(alias = "extract_with_fallbacks", alias = "extract_declarative")]
346 ExtractWithFallback {
347 /// Logical name for this extraction target (e.g. `"btc_price"`).
348 /// Written into `shared_memory` on success, and used by the
349 /// runner to group consecutive failures for retry budgeting.
350 name: String,
351 /// CSS selector or JS expression candidates, tried in order.
352 /// An entry may be either a plain CSS selector (runtime wraps
353 /// it into `document.querySelector(SEL)?.textContent`) or a
354 /// full JS expression starting with `function` or a recognized
355 /// prefix — the runtime auto-detects.
356 selectors: Vec<String>,
357 /// How to parse the raw string yielded by the first matching
358 /// selector. One of: `"text"`, `"float"`, `"int"`, `"html"`.
359 /// Unknown values fall back to `"text"`.
360 #[serde(default = "default_parse_as", alias = "parse", alias = "as")]
361 parse_as: String,
362 },
363 /// Deterministic spreadsheet cell writes via AppleScript (Numbers).
364 ///
365 /// Replaces the flaky keystroke recipe (`activate_app → key(arrows)
366 /// → key(Delete) → type → key(Return)`) with one atomic operation
367 /// against the document model. The keystroke recipe produced
368 /// concatenated garbage values, duplicated headers, and values
369 /// landing in the wrong cells whenever an intermediate step got
370 /// perturbed by focus drift or AX tree lag. `WriteCells` sidesteps
371 /// the entire UI event loop.
372 ///
373 /// Batch-shaped because the AppleScript spawn cost amortizes across
374 /// many cells — single-cell callers pass a length-1 `writes` vector.
375 #[serde(alias = "write_cell")]
376 WriteCells {
377 /// Target app. Currently only `"Numbers"` is implemented; other
378 /// values produce a clean runtime error so the planner can pivot.
379 #[serde(default = "default_spreadsheet_app")]
380 app: String,
381 /// Optional sheet name. `None` = first sheet of first document.
382 #[serde(default)]
383 sheet: Option<String>,
384 /// Optional table name. `None` = first table of selected sheet.
385 #[serde(default)]
386 table: Option<String>,
387 /// Writes to apply, in order.
388 writes: Vec<CellWrite>,
389 /// When true, the runtime reads each cell back after writing
390 /// and includes the readback in the step result's `data` field.
391 /// Recommended: keep `true` — verification is cheap (same
392 /// AppleScript call) and catches Numbers' value coercions.
393 #[serde(default = "default_true")]
394 verify: bool,
395 },
396 /// Deterministic spreadsheet cell reads via AppleScript (Numbers).
397 ///
398 /// Use this when the agent needs spreadsheet truth from the app
399 /// model instead of relying on the accessibility tree to expose
400 /// the values. Particularly useful for Numbers, whose AX surface
401 /// often only exposes the focused cell or formula-bar content.
402 #[serde(alias = "read_cell")]
403 ReadCells {
404 /// Target app. Currently only `"Numbers"` is implemented.
405 #[serde(default = "default_spreadsheet_app")]
406 app: String,
407 /// Optional sheet name. `None` = first sheet of first document.
408 #[serde(default)]
409 sheet: Option<String>,
410 /// Optional table name. `None` = first table of selected sheet.
411 #[serde(default)]
412 table: Option<String>,
413 /// Cells to read, in order.
414 #[serde(alias = "refs", alias = "cells", alias = "addresses")]
415 cell_refs: Vec<String>,
416 },
417 /// Native window management — move/resize/minimize/maximize/focus a macOS
418 /// window resolved by `app` + `window_index`. A runtime executes the
419 /// operation and reads geometry back into the receipt. `op` is one of:
420 /// `"move"`, `"resize"`, `"set_bounds"`,
421 /// `"minimize"`, `"unminimize"`, `"maximize"`, `"focus"`. A `preset`
422 /// (WS2.3 tiling, e.g. `"left_half"`) overrides `op` + geometry when set.
423 Window {
424 op: String,
425 /// Target app name. `None` = the frontmost app.
426 #[serde(default, skip_serializing_if = "Option::is_none")]
427 app: Option<String>,
428 /// Window index (0 = the app's frontmost window).
429 #[serde(default)]
430 window_index: usize,
431 #[serde(default, skip_serializing_if = "Option::is_none")]
432 x: Option<f64>,
433 #[serde(default, skip_serializing_if = "Option::is_none")]
434 y: Option<f64>,
435 #[serde(default, skip_serializing_if = "Option::is_none")]
436 width: Option<f64>,
437 #[serde(default, skip_serializing_if = "Option::is_none")]
438 height: Option<f64>,
439 /// Optional tiling preset (WS2.3). When set, overrides `op` + geometry.
440 #[serde(default, skip_serializing_if = "Option::is_none")]
441 preset: Option<String>,
442 /// Optional target display index (WS4). When set, presets/placement
443 /// target that display; otherwise the window's current display.
444 #[serde(default, skip_serializing_if = "Option::is_none")]
445 display: Option<usize>,
446 },
447 /// Native dialog / sheet driver (WS5) — list, click a button by title, set
448 /// a text field, or dismiss the frontmost macOS Open/Save/Print sheet or
449 /// alert. The runtime resolves the dialog's controls via the accessibility
450 /// tree. `op` is one of: `"list"`, `"click"`, `"set_field"`, `"dismiss"`.
451 Dialog {
452 op: String,
453 /// Button title to click (for `op = "click"`); case-insensitive substring.
454 #[serde(default, skip_serializing_if = "Option::is_none")]
455 button: Option<String>,
456 /// Value to set (for `op = "set_field"`).
457 #[serde(default, skip_serializing_if = "Option::is_none")]
458 value: Option<String>,
459 /// Which visible text field to set (0-based; default 0).
460 #[serde(default)]
461 field_index: usize,
462 },
463 /// Native Dock control (WS6) — list items, launch / right-click an item by
464 /// title, or toggle auto-hide. `op` is one of: `"list"`, `"launch"`,
465 /// `"right_click"`, `"hide"`, `"show"`.
466 Dock {
467 op: String,
468 /// Dock item title (for `launch` / `right_click`).
469 #[serde(default, skip_serializing_if = "Option::is_none")]
470 name: Option<String>,
471 },
472 /// Native menu-bar extras (WS7) — list system status items (Wi-Fi,
473 /// Bluetooth, Control Center, …) or click one by title. `op` is `"list"`
474 /// or `"click"`.
475 MenuExtra {
476 op: String,
477 /// Status-item title to click (for `op = "click"`), case-insensitive.
478 #[serde(default, skip_serializing_if = "Option::is_none")]
479 name: Option<String>,
480 },
481}
482
483/// One cell write inside a [`PlannedAction::WriteCells`] batch.
484#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
485pub struct CellWrite {
486 /// A1-notation cell reference, e.g. `"B2"`, `"AA17"`.
487 #[serde(alias = "ref", alias = "cell", alias = "address")]
488 pub cell_ref: String,
489 /// Value to write. Pass raw numeric strings (`"108432.50"`, not
490 /// `"$108,432.50"`); Numbers formats per the cell's display
491 /// format. Text values pass through unchanged.
492 pub value: String,
493}
494
495fn default_spreadsheet_app() -> String {
496 "Numbers".into()
497}
498
499fn default_true() -> bool {
500 true
501}
502
503fn default_parse_as() -> String {
504 "text".into()
505}
506
507impl PlannedAction {
508 /// Target element IDs this action depends on, if any. Used by the runner
509 /// to pre-validate that planned elements still exist in the fresh runtime
510 /// context before dispatch — missing targets trigger a replan instead of
511 /// silent misfire.
512 ///
513 /// Returns an empty slice for actions with no element target (coordinate
514 /// scrolls, key events, `CdpEval`, terminal actions). `Drag` returns two
515 /// IDs (from + to). `Batch` recurses — every sub-action's targets are
516 /// collected flat, so if any one is missing the whole batch is aborted
517 /// before any side-effect lands. Sub-batches inside batches collapse the
518 /// same way.
519 pub fn target_ids(&self) -> Vec<&str> {
520 match self {
521 Self::Click { target_id, .. }
522 | Self::SetValue { target_id, .. }
523 | Self::AxAction { target_id, .. } => vec![target_id.as_str()],
524 Self::Type {
525 target_id: Some(id),
526 ..
527 } => vec![id.as_str()],
528 Self::Drag {
529 from_target_id,
530 to_target_id,
531 } => vec![from_target_id.as_str(), to_target_id.as_str()],
532 Self::Batch { actions } => actions.iter().flat_map(|a| a.target_ids()).collect(),
533 // No element-level targets:
534 Self::Type {
535 target_id: None, ..
536 }
537 | Self::Key { .. }
538 | Self::KeyCombo { .. }
539 | Self::Scroll { .. }
540 | Self::Wait { .. }
541 | Self::Custom { .. }
542 | Self::Extract { .. }
543 | Self::Act { .. }
544 | Self::Done { .. }
545 | Self::Fail { .. }
546 | Self::ActivateApp { .. }
547 | Self::LaunchApp { .. }
548 | Self::QuitApp { .. }
549 | Self::Select { .. }
550 | Self::CdpEval { .. }
551 | Self::Navigate { .. }
552 | Self::NotebookWrites { .. }
553 | Self::WriteCells { .. }
554 | Self::ReadCells { .. }
555 | Self::ExtractWithFallback { .. }
556 // Window ops target by app + window index, not an element id.
557 | Self::Window { .. }
558 // Dialog ops resolve their controls internally, not by element id.
559 | Self::Dialog { .. }
560 // Dock ops target by item title, not an element id.
561 | Self::Dock { .. }
562 // Menu extras resolve by title, not by element id.
563 | Self::MenuExtra { .. } => vec![],
564 }
565 }
566}
567
568#[cfg(test)]
569mod target_ids_tests {
570 use super::*;
571
572 #[test]
573 fn click_returns_its_target() {
574 let a = PlannedAction::Click {
575 target_id: "a11y:42".into(),
576 expect_after: None,
577 };
578 assert_eq!(a.target_ids(), vec!["a11y:42"]);
579 }
580
581 #[test]
582 fn click_without_expect_after_round_trips_omitting_field() {
583 // Back-compat: planners that don't know about `expect_after`
584 // still round-trip through the same JSON shape they always
585 // emitted (no field added on serialize, no field required on
586 // deserialize).
587 let raw = r#"{"type":"click","target_id":"dom:button:submit"}"#;
588 let a: PlannedAction = serde_json::from_str(raw).unwrap();
589 match a {
590 PlannedAction::Click {
591 target_id,
592 expect_after,
593 } => {
594 assert_eq!(target_id, "dom:button:submit");
595 assert!(expect_after.is_none());
596 }
597 _ => panic!("expected Click"),
598 }
599 let serialised = serde_json::to_string(&PlannedAction::Click {
600 target_id: "dom:button:submit".into(),
601 expect_after: None,
602 })
603 .unwrap();
604 // `skip_serializing_if = "Option::is_none"` keeps the JSON
605 // identical to the pre-`expect_after` shape.
606 assert!(!serialised.contains("expect_after"));
607 }
608
609 #[test]
610 fn click_with_selector_appears_expectation_round_trips() {
611 let raw = r##"{
612 "type": "click",
613 "target_id": "dom:button:submit",
614 "expect_after": {
615 "kind": "selector_appears",
616 "selector": "#success-message",
617 "timeout_ms": 3000
618 }
619 }"##;
620 let a: PlannedAction = serde_json::from_str(raw).unwrap();
621 match a {
622 PlannedAction::Click {
623 target_id,
624 expect_after:
625 Some(EffectExpectation::SelectorAppears {
626 selector,
627 timeout_ms,
628 }),
629 } => {
630 assert_eq!(target_id, "dom:button:submit");
631 assert_eq!(selector, "#success-message");
632 assert_eq!(timeout_ms, 3000);
633 }
634 other => panic!("expected Click with SelectorAppears, got {other:?}"),
635 }
636 }
637
638 #[test]
639 fn effect_expectation_default_timeout_when_omitted() {
640 // `timeout_ms` has a serde default of 2000ms — the planner can
641 // omit it for the common case and only override when the page
642 // is known to be slow (heavy SPA render, lazy-loaded modal).
643 let raw = r##"{"kind": "selector_appears", "selector": "#x"}"##;
644 let e: EffectExpectation = serde_json::from_str(raw).unwrap();
645 match e {
646 EffectExpectation::SelectorAppears {
647 selector,
648 timeout_ms,
649 } => {
650 assert_eq!(selector, "#x");
651 assert_eq!(timeout_ms, 2_000);
652 }
653 other => panic!("expected SelectorAppears, got {other:?}"),
654 }
655 }
656
657 #[test]
658 fn effect_expectation_all_four_variants_parse() {
659 let appears: EffectExpectation =
660 serde_json::from_str(r#"{"kind":"selector_appears","selector":".success"}"#).unwrap();
661 assert!(matches!(appears, EffectExpectation::SelectorAppears { .. }));
662 let disappears: EffectExpectation =
663 serde_json::from_str(r#"{"kind":"selector_disappears","selector":".modal.open"}"#)
664 .unwrap();
665 assert!(matches!(
666 disappears,
667 EffectExpectation::SelectorDisappears { .. }
668 ));
669 let text: EffectExpectation = serde_json::from_str(
670 r##"{"kind":"selector_text_contains","selector":"#status","substring":"Approved"}"##,
671 )
672 .unwrap();
673 assert!(matches!(
674 text,
675 EffectExpectation::SelectorTextContains { .. }
676 ));
677 // `dom_changed` is the diff-based fallback for actions whose
678 // post-state isn't a single named selector. No `selector`
679 // field — just an optional timeout.
680 let changed: EffectExpectation = serde_json::from_str(r#"{"kind":"dom_changed"}"#).unwrap();
681 match changed {
682 EffectExpectation::DomChanged { timeout_ms } => {
683 // Default timeout when omitted.
684 assert_eq!(timeout_ms, 2_000);
685 }
686 other => panic!("expected DomChanged, got {other:?}"),
687 }
688 let changed_custom: EffectExpectation =
689 serde_json::from_str(r#"{"kind":"dom_changed","timeout_ms":5000}"#).unwrap();
690 match changed_custom {
691 EffectExpectation::DomChanged { timeout_ms } => {
692 assert_eq!(timeout_ms, 5_000);
693 }
694 other => panic!("expected DomChanged with custom timeout, got {other:?}"),
695 }
696 }
697
698 #[test]
699 fn type_without_target_returns_empty() {
700 let a = PlannedAction::Type {
701 target_id: None,
702 text: "hi".into(),
703 };
704 assert!(a.target_ids().is_empty());
705 }
706
707 #[test]
708 fn drag_returns_both_endpoints() {
709 let a = PlannedAction::Drag {
710 from_target_id: "a11y:1".into(),
711 to_target_id: "a11y:2".into(),
712 };
713 assert_eq!(a.target_ids(), vec!["a11y:1", "a11y:2"]);
714 }
715
716 #[test]
717 fn batch_flattens_sub_action_targets() {
718 let a = PlannedAction::Batch {
719 actions: vec![
720 PlannedAction::CdpEval {
721 expression: "1".into(),
722 },
723 PlannedAction::Click {
724 target_id: "a11y:ghost".into(),
725 expect_after: None,
726 },
727 PlannedAction::SetValue {
728 target_id: "a11y:input".into(),
729 value: "v".into(),
730 expect_after: None,
731 },
732 ],
733 };
734 assert_eq!(a.target_ids(), vec!["a11y:ghost", "a11y:input"]);
735 }
736
737 #[test]
738 fn nested_batches_flatten() {
739 let a = PlannedAction::Batch {
740 actions: vec![PlannedAction::Batch {
741 actions: vec![PlannedAction::Click {
742 target_id: "a11y:deep".into(),
743 expect_after: None,
744 }],
745 }],
746 };
747 assert_eq!(a.target_ids(), vec!["a11y:deep"]);
748 }
749
750 #[test]
751 fn cdp_eval_and_terminals_have_no_targets() {
752 assert!(PlannedAction::CdpEval {
753 expression: "x".into()
754 }
755 .target_ids()
756 .is_empty());
757 assert!(PlannedAction::Done {
758 summary: "ok".into(),
759 evidence_ids: vec![]
760 }
761 .target_ids()
762 .is_empty());
763 assert!(PlannedAction::Wait { ms: 100 }.target_ids().is_empty());
764 }
765
766 #[test]
767 fn extract_with_fallback_round_trips() {
768 let raw = r#"{"type":"extract_with_fallback","name":"btc_price",
769 "selectors":["fin-streamer[data-field='regularMarketPrice']",
770 "[data-test='qsp-price']"],
771 "parse_as":"float"}"#;
772 let a: PlannedAction = serde_json::from_str(raw).unwrap();
773 match &a {
774 PlannedAction::ExtractWithFallback {
775 name,
776 selectors,
777 parse_as,
778 } => {
779 assert_eq!(name, "btc_price");
780 assert_eq!(selectors.len(), 2);
781 assert_eq!(parse_as, "float");
782 }
783 _ => panic!("expected ExtractWithFallback"),
784 }
785 // Has no element-level targets.
786 assert!(a.target_ids().is_empty());
787 }
788
789 #[test]
790 fn extract_with_fallback_defaults_parse_as_to_text() {
791 let raw = r#"{"type":"extract_with_fallback","name":"title",
792 "selectors":["h1"]}"#;
793 let a: PlannedAction = serde_json::from_str(raw).unwrap();
794 match a {
795 PlannedAction::ExtractWithFallback { parse_as, .. } => {
796 assert_eq!(parse_as, "text");
797 }
798 _ => panic!("expected ExtractWithFallback"),
799 }
800 }
801
802 #[test]
803 fn extract_with_fallback_accepts_parse_alias() {
804 let raw = r#"{"type":"extract_with_fallback","name":"n",
805 "selectors":["a"],"parse":"int"}"#;
806 let a: PlannedAction = serde_json::from_str(raw).unwrap();
807 match a {
808 PlannedAction::ExtractWithFallback { parse_as, .. } => {
809 assert_eq!(parse_as, "int");
810 }
811 _ => panic!("expected ExtractWithFallback"),
812 }
813 }
814
815 #[test]
816 fn read_cells_accepts_cells_alias_and_defaults_app() {
817 let raw = r#"{"type":"read_cells","cells":["A1","B2"]}"#;
818 let a: PlannedAction = serde_json::from_str(raw).unwrap();
819 match a {
820 PlannedAction::ReadCells { app, cell_refs, .. } => {
821 assert_eq!(app, "Numbers");
822 assert_eq!(cell_refs, vec!["A1", "B2"]);
823 }
824 _ => panic!("expected ReadCells"),
825 }
826 }
827
828 #[test]
829 fn ax_action_accepts_null_target_id_as_empty_string() {
830 // Some models emit `"target_id": null` when they only know
831 // the visible label and want the runtime to resolve. The
832 // previous contract treated this as a fatal parse error
833 // (`invalid type: null, expected a string`), killing the
834 // entire turn before the label-fallback dispatcher could
835 // run. Lenient deserialization converts null to an empty
836 // string; runtimes can handle empty target_id by going straight to
837 // label resolution.
838 let raw = r#"{
839 "type": "ax_action",
840 "target_id": null,
841 "action": "click",
842 "label": "Export to Notes",
843 "role_hint": "button"
844 }"#;
845 let a: PlannedAction = serde_json::from_str(raw).unwrap();
846 match a {
847 PlannedAction::AxAction {
848 target_id,
849 action,
850 label,
851 role_hint,
852 ..
853 } => {
854 assert_eq!(target_id, "");
855 assert_eq!(action, "click");
856 assert_eq!(label.as_deref(), Some("Export to Notes"));
857 assert_eq!(role_hint.as_deref(), Some("button"));
858 }
859 _ => panic!("expected AxAction"),
860 }
861 }
862
863 #[test]
864 fn ax_action_accepts_missing_target_id_field_entirely() {
865 // Defensive: if the planner omits the field rather than
866 // explicitly sending null, the `#[serde(default)]` falls
867 // back to `String::default()` (empty string), same
868 // dispatch path.
869 let raw = r#"{
870 "type": "ax_action",
871 "action": "click",
872 "label": "Submit"
873 }"#;
874 let a: PlannedAction = serde_json::from_str(raw).unwrap();
875 match a {
876 PlannedAction::AxAction {
877 target_id, label, ..
878 } => {
879 assert_eq!(target_id, "");
880 assert_eq!(label.as_deref(), Some("Submit"));
881 }
882 _ => panic!("expected AxAction"),
883 }
884 }
885
886 #[test]
887 fn ax_action_still_accepts_explicit_target_id_string() {
888 // Regression guard: the lenient path must not break the
889 // normal case where the planner emits a real id.
890 let raw = r#"{
891 "type": "ax_action",
892 "target_id": "ax:AXButton/0x1234",
893 "action": "click"
894 }"#;
895 let a: PlannedAction = serde_json::from_str(raw).unwrap();
896 match a {
897 PlannedAction::AxAction { target_id, .. } => {
898 assert_eq!(target_id, "ax:AXButton/0x1234");
899 }
900 _ => panic!("expected AxAction"),
901 }
902 }
903
904 #[test]
905 fn navigate_legacy_payload_still_parses() {
906 // The pre-canonical wire shape — no wait/timeout/dismiss
907 // fields — must keep working unchanged. Every existing
908 // planner (LangGraph, internal tests, MCP clients on older
909 // SDKs) emits this; flipping to required fields would silently
910 // brick navigation.
911 let raw = r#"{"type":"navigate","url":"https://example.com"}"#;
912 let a: PlannedAction = serde_json::from_str(raw).unwrap();
913 match a {
914 PlannedAction::Navigate {
915 url,
916 wait_until,
917 timeout_ms,
918 dismiss_overlays,
919 } => {
920 assert_eq!(url, "https://example.com");
921 assert!(wait_until.is_none());
922 assert!(timeout_ms.is_none());
923 assert!(dismiss_overlays.is_none());
924 }
925 _ => panic!("expected Navigate"),
926 }
927 }
928
929 #[test]
930 fn navigate_extended_payload_parses_all_fields() {
931 let raw = r#"{
932 "type":"navigate",
933 "url":"https://example.com",
934 "wait_until":"load",
935 "timeout_ms":10000,
936 "dismiss_overlays":false
937 }"#;
938 let a: PlannedAction = serde_json::from_str(raw).unwrap();
939 match a {
940 PlannedAction::Navigate {
941 url,
942 wait_until,
943 timeout_ms,
944 dismiss_overlays,
945 } => {
946 assert_eq!(url, "https://example.com");
947 assert_eq!(wait_until.as_deref(), Some("load"));
948 assert_eq!(timeout_ms, Some(10_000));
949 assert_eq!(dismiss_overlays, Some(false));
950 }
951 _ => panic!("expected Navigate"),
952 }
953 }
954
955 #[test]
956 fn navigate_url_aliases_href_and_to_still_work() {
957 // The original `Navigate` variant accepted `href` and `to`
958 // aliases on the URL field for LLM ergonomics. The extended
959 // variant inherits those aliases — pin both forms to catch
960 // an accidental drop during a future refactor.
961 for raw in [
962 r#"{"type":"navigate","href":"https://example.com"}"#,
963 r#"{"type":"navigate","to":"https://example.com"}"#,
964 ] {
965 let a: PlannedAction = serde_json::from_str(raw).expect(raw);
966 match a {
967 PlannedAction::Navigate { url, .. } => {
968 assert_eq!(url, "https://example.com")
969 }
970 _ => panic!("expected Navigate for {raw}"),
971 }
972 }
973 }
974}