use serde::{Deserialize, Deserializer, Serialize};
fn deserialize_string_or_value<'de, D>(deserializer: D) -> Result<String, D::Error>
where
D: Deserializer<'de>,
{
let value = serde_json::Value::deserialize(deserializer)?;
match value {
serde_json::Value::String(s) => Ok(s),
serde_json::Value::Null => Ok(String::new()),
other => Ok(other.to_string()),
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum EffectExpectation {
SelectorAppears {
selector: String,
#[serde(default = "default_effect_timeout_ms")]
timeout_ms: u64,
},
SelectorDisappears {
selector: String,
#[serde(default = "default_effect_timeout_ms")]
timeout_ms: u64,
},
SelectorTextContains {
selector: String,
substring: String,
#[serde(default = "default_effect_timeout_ms")]
timeout_ms: u64,
},
DomChanged {
#[serde(default = "default_effect_timeout_ms")]
timeout_ms: u64,
},
}
fn default_effect_timeout_ms() -> u64 {
2_000
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
#[repr(u8)]
pub enum FocusMode {
#[default]
Foreground,
Background,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum PlannedAction {
Click {
target_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
expect_after: Option<EffectExpectation>,
},
Type {
#[serde(default, skip_serializing_if = "Option::is_none")]
target_id: Option<String>,
text: String,
},
Key {
key: String,
},
KeyCombo {
keys: Vec<String>,
},
SetValue {
target_id: String,
value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
expect_after: Option<EffectExpectation>,
},
Scroll {
dx: i32,
dy: i32,
},
Drag {
from_target_id: String,
to_target_id: String,
},
Wait {
ms: u32,
},
Custom {
adapter: String,
action: String,
#[serde(default)]
params: serde_json::Value,
},
Extract {
#[serde(default, deserialize_with = "deserialize_string_or_value")]
goal: String,
#[serde(default, deserialize_with = "deserialize_string_or_value")]
data: String,
},
Batch {
actions: Vec<PlannedAction>,
},
Act {
instruction: String,
},
Done {
#[serde(deserialize_with = "deserialize_string_or_value")]
summary: String,
#[serde(default)]
evidence_ids: Vec<String>,
},
Fail {
#[serde(deserialize_with = "deserialize_string_or_value")]
reason: String,
},
AxAction {
#[serde(default, deserialize_with = "deserialize_string_or_value")]
target_id: String,
action: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
label: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
role_hint: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
expect_after: Option<EffectExpectation>,
},
ActivateApp {
app_name: String,
},
LaunchApp {
app_name: String,
#[serde(default)]
background: bool,
},
QuitApp {
app_name: String,
},
Select {
from_x: i32,
from_y: i32,
to_x: i32,
to_y: i32,
},
CdpEval {
expression: String,
},
Navigate {
#[serde(alias = "href", alias = "to")]
url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
wait_until: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
timeout_ms: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
dismiss_overlays: Option<bool>,
},
#[serde(alias = "notebook_write")]
NotebookWrites {
#[serde(default)]
key: String,
#[serde(default)]
value: String,
#[serde(default)]
category: String,
},
#[serde(alias = "extract_with_fallbacks", alias = "extract_declarative")]
ExtractWithFallback {
name: String,
selectors: Vec<String>,
#[serde(default = "default_parse_as", alias = "parse", alias = "as")]
parse_as: String,
},
#[serde(alias = "write_cell")]
WriteCells {
#[serde(default = "default_spreadsheet_app")]
app: String,
#[serde(default)]
sheet: Option<String>,
#[serde(default)]
table: Option<String>,
writes: Vec<CellWrite>,
#[serde(default = "default_true")]
verify: bool,
},
#[serde(alias = "read_cell")]
ReadCells {
#[serde(default = "default_spreadsheet_app")]
app: String,
#[serde(default)]
sheet: Option<String>,
#[serde(default)]
table: Option<String>,
#[serde(alias = "refs", alias = "cells", alias = "addresses")]
cell_refs: Vec<String>,
},
Window {
op: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
app: Option<String>,
#[serde(default)]
window_index: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
y: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
width: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
height: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
preset: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
display: Option<usize>,
},
Dialog {
op: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
button: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
value: Option<String>,
#[serde(default)]
field_index: usize,
},
Dock {
op: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
name: Option<String>,
},
MenuExtra {
op: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
name: Option<String>,
},
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct CellWrite {
#[serde(alias = "ref", alias = "cell", alias = "address")]
pub cell_ref: String,
pub value: String,
}
fn default_spreadsheet_app() -> String {
"Numbers".into()
}
fn default_true() -> bool {
true
}
fn default_parse_as() -> String {
"text".into()
}
impl PlannedAction {
pub fn target_ids(&self) -> Vec<&str> {
match self {
Self::Click { target_id, .. }
| Self::SetValue { target_id, .. }
| Self::AxAction { target_id, .. } => vec![target_id.as_str()],
Self::Type {
target_id: Some(id),
..
} => vec![id.as_str()],
Self::Drag {
from_target_id,
to_target_id,
} => vec![from_target_id.as_str(), to_target_id.as_str()],
Self::Batch { actions } => actions.iter().flat_map(|a| a.target_ids()).collect(),
Self::Type {
target_id: None, ..
}
| Self::Key { .. }
| Self::KeyCombo { .. }
| Self::Scroll { .. }
| Self::Wait { .. }
| Self::Custom { .. }
| Self::Extract { .. }
| Self::Act { .. }
| Self::Done { .. }
| Self::Fail { .. }
| Self::ActivateApp { .. }
| Self::LaunchApp { .. }
| Self::QuitApp { .. }
| Self::Select { .. }
| Self::CdpEval { .. }
| Self::Navigate { .. }
| Self::NotebookWrites { .. }
| Self::WriteCells { .. }
| Self::ReadCells { .. }
| Self::ExtractWithFallback { .. }
| Self::Window { .. }
| Self::Dialog { .. }
| Self::Dock { .. }
| Self::MenuExtra { .. } => vec![],
}
}
}
#[cfg(test)]
mod target_ids_tests {
use super::*;
#[test]
fn click_returns_its_target() {
let a = PlannedAction::Click {
target_id: "a11y:42".into(),
expect_after: None,
};
assert_eq!(a.target_ids(), vec!["a11y:42"]);
}
#[test]
fn click_without_expect_after_round_trips_omitting_field() {
let raw = r#"{"type":"click","target_id":"dom:button:submit"}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::Click {
target_id,
expect_after,
} => {
assert_eq!(target_id, "dom:button:submit");
assert!(expect_after.is_none());
}
_ => panic!("expected Click"),
}
let serialised = serde_json::to_string(&PlannedAction::Click {
target_id: "dom:button:submit".into(),
expect_after: None,
})
.unwrap();
assert!(!serialised.contains("expect_after"));
}
#[test]
fn click_with_selector_appears_expectation_round_trips() {
let raw = r##"{
"type": "click",
"target_id": "dom:button:submit",
"expect_after": {
"kind": "selector_appears",
"selector": "#success-message",
"timeout_ms": 3000
}
}"##;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::Click {
target_id,
expect_after:
Some(EffectExpectation::SelectorAppears {
selector,
timeout_ms,
}),
} => {
assert_eq!(target_id, "dom:button:submit");
assert_eq!(selector, "#success-message");
assert_eq!(timeout_ms, 3000);
}
other => panic!("expected Click with SelectorAppears, got {other:?}"),
}
}
#[test]
fn effect_expectation_default_timeout_when_omitted() {
let raw = r##"{"kind": "selector_appears", "selector": "#x"}"##;
let e: EffectExpectation = serde_json::from_str(raw).unwrap();
match e {
EffectExpectation::SelectorAppears {
selector,
timeout_ms,
} => {
assert_eq!(selector, "#x");
assert_eq!(timeout_ms, 2_000);
}
other => panic!("expected SelectorAppears, got {other:?}"),
}
}
#[test]
fn effect_expectation_all_four_variants_parse() {
let appears: EffectExpectation =
serde_json::from_str(r#"{"kind":"selector_appears","selector":".success"}"#).unwrap();
assert!(matches!(appears, EffectExpectation::SelectorAppears { .. }));
let disappears: EffectExpectation =
serde_json::from_str(r#"{"kind":"selector_disappears","selector":".modal.open"}"#)
.unwrap();
assert!(matches!(
disappears,
EffectExpectation::SelectorDisappears { .. }
));
let text: EffectExpectation = serde_json::from_str(
r##"{"kind":"selector_text_contains","selector":"#status","substring":"Approved"}"##,
)
.unwrap();
assert!(matches!(
text,
EffectExpectation::SelectorTextContains { .. }
));
let changed: EffectExpectation = serde_json::from_str(r#"{"kind":"dom_changed"}"#).unwrap();
match changed {
EffectExpectation::DomChanged { timeout_ms } => {
assert_eq!(timeout_ms, 2_000);
}
other => panic!("expected DomChanged, got {other:?}"),
}
let changed_custom: EffectExpectation =
serde_json::from_str(r#"{"kind":"dom_changed","timeout_ms":5000}"#).unwrap();
match changed_custom {
EffectExpectation::DomChanged { timeout_ms } => {
assert_eq!(timeout_ms, 5_000);
}
other => panic!("expected DomChanged with custom timeout, got {other:?}"),
}
}
#[test]
fn type_without_target_returns_empty() {
let a = PlannedAction::Type {
target_id: None,
text: "hi".into(),
};
assert!(a.target_ids().is_empty());
}
#[test]
fn drag_returns_both_endpoints() {
let a = PlannedAction::Drag {
from_target_id: "a11y:1".into(),
to_target_id: "a11y:2".into(),
};
assert_eq!(a.target_ids(), vec!["a11y:1", "a11y:2"]);
}
#[test]
fn batch_flattens_sub_action_targets() {
let a = PlannedAction::Batch {
actions: vec![
PlannedAction::CdpEval {
expression: "1".into(),
},
PlannedAction::Click {
target_id: "a11y:ghost".into(),
expect_after: None,
},
PlannedAction::SetValue {
target_id: "a11y:input".into(),
value: "v".into(),
expect_after: None,
},
],
};
assert_eq!(a.target_ids(), vec!["a11y:ghost", "a11y:input"]);
}
#[test]
fn nested_batches_flatten() {
let a = PlannedAction::Batch {
actions: vec![PlannedAction::Batch {
actions: vec![PlannedAction::Click {
target_id: "a11y:deep".into(),
expect_after: None,
}],
}],
};
assert_eq!(a.target_ids(), vec!["a11y:deep"]);
}
#[test]
fn cdp_eval_and_terminals_have_no_targets() {
assert!(PlannedAction::CdpEval {
expression: "x".into()
}
.target_ids()
.is_empty());
assert!(PlannedAction::Done {
summary: "ok".into(),
evidence_ids: vec![]
}
.target_ids()
.is_empty());
assert!(PlannedAction::Wait { ms: 100 }.target_ids().is_empty());
}
#[test]
fn extract_with_fallback_round_trips() {
let raw = r#"{"type":"extract_with_fallback","name":"btc_price",
"selectors":["fin-streamer[data-field='regularMarketPrice']",
"[data-test='qsp-price']"],
"parse_as":"float"}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match &a {
PlannedAction::ExtractWithFallback {
name,
selectors,
parse_as,
} => {
assert_eq!(name, "btc_price");
assert_eq!(selectors.len(), 2);
assert_eq!(parse_as, "float");
}
_ => panic!("expected ExtractWithFallback"),
}
assert!(a.target_ids().is_empty());
}
#[test]
fn extract_with_fallback_defaults_parse_as_to_text() {
let raw = r#"{"type":"extract_with_fallback","name":"title",
"selectors":["h1"]}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::ExtractWithFallback { parse_as, .. } => {
assert_eq!(parse_as, "text");
}
_ => panic!("expected ExtractWithFallback"),
}
}
#[test]
fn extract_with_fallback_accepts_parse_alias() {
let raw = r#"{"type":"extract_with_fallback","name":"n",
"selectors":["a"],"parse":"int"}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::ExtractWithFallback { parse_as, .. } => {
assert_eq!(parse_as, "int");
}
_ => panic!("expected ExtractWithFallback"),
}
}
#[test]
fn read_cells_accepts_cells_alias_and_defaults_app() {
let raw = r#"{"type":"read_cells","cells":["A1","B2"]}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::ReadCells { app, cell_refs, .. } => {
assert_eq!(app, "Numbers");
assert_eq!(cell_refs, vec!["A1", "B2"]);
}
_ => panic!("expected ReadCells"),
}
}
#[test]
fn ax_action_accepts_null_target_id_as_empty_string() {
let raw = r#"{
"type": "ax_action",
"target_id": null,
"action": "click",
"label": "Export to Notes",
"role_hint": "button"
}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::AxAction {
target_id,
action,
label,
role_hint,
..
} => {
assert_eq!(target_id, "");
assert_eq!(action, "click");
assert_eq!(label.as_deref(), Some("Export to Notes"));
assert_eq!(role_hint.as_deref(), Some("button"));
}
_ => panic!("expected AxAction"),
}
}
#[test]
fn ax_action_accepts_missing_target_id_field_entirely() {
let raw = r#"{
"type": "ax_action",
"action": "click",
"label": "Submit"
}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::AxAction {
target_id, label, ..
} => {
assert_eq!(target_id, "");
assert_eq!(label.as_deref(), Some("Submit"));
}
_ => panic!("expected AxAction"),
}
}
#[test]
fn ax_action_still_accepts_explicit_target_id_string() {
let raw = r#"{
"type": "ax_action",
"target_id": "ax:AXButton/0x1234",
"action": "click"
}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::AxAction { target_id, .. } => {
assert_eq!(target_id, "ax:AXButton/0x1234");
}
_ => panic!("expected AxAction"),
}
}
#[test]
fn navigate_legacy_payload_still_parses() {
let raw = r#"{"type":"navigate","url":"https://example.com"}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::Navigate {
url,
wait_until,
timeout_ms,
dismiss_overlays,
} => {
assert_eq!(url, "https://example.com");
assert!(wait_until.is_none());
assert!(timeout_ms.is_none());
assert!(dismiss_overlays.is_none());
}
_ => panic!("expected Navigate"),
}
}
#[test]
fn navigate_extended_payload_parses_all_fields() {
let raw = r#"{
"type":"navigate",
"url":"https://example.com",
"wait_until":"load",
"timeout_ms":10000,
"dismiss_overlays":false
}"#;
let a: PlannedAction = serde_json::from_str(raw).unwrap();
match a {
PlannedAction::Navigate {
url,
wait_until,
timeout_ms,
dismiss_overlays,
} => {
assert_eq!(url, "https://example.com");
assert_eq!(wait_until.as_deref(), Some("load"));
assert_eq!(timeout_ms, Some(10_000));
assert_eq!(dismiss_overlays, Some(false));
}
_ => panic!("expected Navigate"),
}
}
#[test]
fn navigate_url_aliases_href_and_to_still_work() {
for raw in [
r#"{"type":"navigate","href":"https://example.com"}"#,
r#"{"type":"navigate","to":"https://example.com"}"#,
] {
let a: PlannedAction = serde_json::from_str(raw).expect(raw);
match a {
PlannedAction::Navigate { url, .. } => {
assert_eq!(url, "https://example.com")
}
_ => panic!("expected Navigate for {raw}"),
}
}
}
}