use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use crate::persistent_refs::RefStore;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum RecordedAction {
Click { x: f64, y: f64 },
Type { text: String },
KeyPress { key: String, modifiers: Vec<String> },
Scroll { dx: f64, dy: f64 },
Wait { duration_ms: u64 },
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct RecordedEvent {
pub timestamp: u64,
pub action: RecordedAction,
pub element_fingerprint: u64,
pub element_label: String,
pub element_role: String,
}
pub struct WorkflowRecorder {
events: Vec<RecordedEvent>,
recording: bool,
start_ms: u64,
}
impl WorkflowRecorder {
#[must_use]
pub fn new() -> Self {
Self {
events: Vec::new(),
recording: false,
start_ms: 0,
}
}
pub fn start_recording(&mut self) {
self.events.clear();
self.recording = true;
self.start_ms = now_ms();
}
pub fn stop_recording(&mut self) -> Vec<RecordedEvent> {
self.recording = false;
std::mem::take(&mut self.events)
}
#[must_use]
pub fn is_recording(&self) -> bool {
self.recording
}
#[must_use]
pub fn event_count(&self) -> usize {
self.events.len()
}
pub fn record_event(&mut self, mut event: RecordedEvent) {
if !self.recording {
return;
}
event.timestamp = now_ms().saturating_sub(self.start_ms);
self.events.push(event);
}
pub fn record_click(&mut self, x: f64, y: f64, fingerprint: u64, label: &str, role: &str) {
self.record_event(RecordedEvent {
timestamp: 0,
action: RecordedAction::Click { x, y },
element_fingerprint: fingerprint,
element_label: label.to_owned(),
element_role: role.to_owned(),
});
}
pub fn record_type(&mut self, text: &str, fingerprint: u64, label: &str, role: &str) {
self.record_event(RecordedEvent {
timestamp: 0,
action: RecordedAction::Type {
text: text.to_owned(),
},
element_fingerprint: fingerprint,
element_label: label.to_owned(),
element_role: role.to_owned(),
});
}
pub fn serialize(events: &[RecordedEvent]) -> Result<String, serde_json::Error> {
serde_json::to_string_pretty(events)
}
pub fn deserialize(json: &str) -> Result<Vec<RecordedEvent>, serde_json::Error> {
serde_json::from_str(json)
}
}
impl Default for WorkflowRecorder {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ReplayResult {
pub success: bool,
pub events_executed: usize,
pub total_events: usize,
pub failures: Vec<ReplayFailure>,
pub adapted_events: Vec<usize>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ReplayFailure {
pub event_index: usize,
pub reason: String,
}
impl ReplayResult {
fn new(total: usize) -> Self {
Self {
success: true,
events_executed: 0,
total_events: total,
failures: Vec::new(),
adapted_events: Vec::new(),
}
}
fn add_failure(&mut self, index: usize, reason: impl Into<String>) {
self.success = false;
self.failures.push(ReplayFailure {
event_index: index,
reason: reason.into(),
});
}
}
pub struct WorkflowPlayer {
pub abort_on_failure: bool,
pub timing_scale: f64,
}
impl WorkflowPlayer {
#[must_use]
pub fn new() -> Self {
Self {
abort_on_failure: false,
timing_scale: 1.0,
}
}
pub fn replay(&self, events: &[RecordedEvent], ref_store: &RefStore) -> ReplayResult {
let mut result = ReplayResult::new(events.len());
for (i, event) in events.iter().enumerate() {
let dispatch_outcome = self.dispatch_event(i, event, ref_store, &mut result);
if dispatch_outcome.is_ok() {
result.events_executed += 1;
} else if self.abort_on_failure {
break;
}
}
result
}
fn dispatch_event(
&self,
index: usize,
event: &RecordedEvent,
ref_store: &RefStore,
result: &mut ReplayResult,
) -> Result<(), ()> {
match &event.action {
RecordedAction::Wait { duration_ms } => {
self.execute_wait(*duration_ms);
Ok(())
}
RecordedAction::Click { x, y } => {
self.execute_click(index, event, ref_store, result, *x, *y)
}
RecordedAction::Type { text } => {
self.execute_type(index, event, ref_store, result, text)
}
RecordedAction::KeyPress { key, modifiers } => {
self.execute_key_press(index, event, ref_store, result, key, modifiers)
}
RecordedAction::Scroll { dx, dy } => {
self.execute_scroll(index, event, ref_store, result, *dx, *dy)
}
}
}
fn resolve_element(
&self,
event: &RecordedEvent,
ref_store: &RefStore,
event_index: usize,
result: &mut ReplayResult,
) -> Option<crate::persistent_refs::ElementRef> {
if event.element_fingerprint != 0 {
if let Some(elem_ref) = ref_store
.resolve_by_fingerprint(event.element_fingerprint)
.filter(|r| r.alive)
{
return Some(elem_ref.clone());
}
}
let candidates = ref_store.find_by_label(&event.element_label);
let alive: Vec<_> = candidates.into_iter().filter(|r| r.alive).collect();
if let Some(best) = alive.into_iter().find(|r| r.role == event.element_role) {
result.adapted_events.push(event_index);
return Some(best.clone());
}
None
}
fn execute_wait(&self, duration_ms: u64) {
let scaled = (duration_ms as f64 * self.timing_scale) as u64;
if scaled > 0 {
std::thread::sleep(std::time::Duration::from_millis(scaled));
}
}
fn execute_click(
&self,
index: usize,
event: &RecordedEvent,
ref_store: &RefStore,
result: &mut ReplayResult,
fallback_x: f64,
fallback_y: f64,
) -> Result<(), ()> {
match self.resolve_element(event, ref_store, index, result) {
Some(elem_ref) => {
let (x, y, w, h) = elem_ref.bounds;
let _ = (x + w / 2.0, y + h / 2.0); Ok(())
}
None if event.element_fingerprint == 0 => {
let _ = (fallback_x, fallback_y);
Ok(())
}
None => {
result.add_failure(
index,
format!(
"Element not found: '{}' ({})",
event.element_label, event.element_role
),
);
Err(())
}
}
}
fn execute_type(
&self,
index: usize,
event: &RecordedEvent,
ref_store: &RefStore,
result: &mut ReplayResult,
text: &str,
) -> Result<(), ()> {
match self.resolve_element(event, ref_store, index, result) {
Some(_elem_ref) => {
let _ = text; Ok(())
}
None => {
result.add_failure(
index,
format!(
"Type target not found: '{}' ({})",
event.element_label, event.element_role
),
);
Err(())
}
}
}
fn execute_key_press(
&self,
index: usize,
event: &RecordedEvent,
ref_store: &RefStore,
result: &mut ReplayResult,
key: &str,
modifiers: &[String],
) -> Result<(), ()> {
let _ = (key, modifiers); match self.resolve_element(event, ref_store, index, result) {
Some(_) => Ok(()),
None if event.element_fingerprint == 0 => Ok(()),
None => {
result.add_failure(
index,
format!(
"KeyPress target not found: '{}' ({})",
event.element_label, event.element_role
),
);
Err(())
}
}
}
fn execute_scroll(
&self,
index: usize,
event: &RecordedEvent,
ref_store: &RefStore,
result: &mut ReplayResult,
dx: f64,
dy: f64,
) -> Result<(), ()> {
let _ = (dx, dy); match self.resolve_element(event, ref_store, index, result) {
Some(_) => Ok(()),
None if event.element_fingerprint == 0 => Ok(()),
None => {
result.add_failure(
index,
format!(
"Scroll target not found: '{}' ({})",
event.element_label, event.element_role
),
);
Err(())
}
}
}
}
impl Default for WorkflowPlayer {
fn default() -> Self {
Self::new()
}
}
fn now_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::persistent_refs::{ElementSnapshot, RefStore};
fn button_event(label: &str, fp: u64) -> RecordedEvent {
RecordedEvent {
timestamp: 0,
action: RecordedAction::Click { x: 10.0, y: 20.0 },
element_fingerprint: fp,
element_label: label.to_owned(),
element_role: "AXButton".to_owned(),
}
}
fn wait_event(ms: u64) -> RecordedEvent {
RecordedEvent {
timestamp: 0,
action: RecordedAction::Wait { duration_ms: ms },
element_fingerprint: 0,
element_label: String::new(),
element_role: String::new(),
}
}
fn store_with_button(label: &str) -> (RefStore, u64) {
let mut store = RefStore::new();
let snap = ElementSnapshot {
role: "AXButton".into(),
label: label.into(),
path: vec!["AXWindow".into(), format!("AXButton:{label}")],
bounds: (100.0, 200.0, 80.0, 30.0),
};
let elem_ref = store.track(snap);
(store, elem_ref.fingerprint)
}
#[test]
fn recorder_starts_empty_and_not_recording() {
let recorder = WorkflowRecorder::new();
assert!(!recorder.is_recording());
assert_eq!(recorder.event_count(), 0);
}
#[test]
fn recorder_start_clears_previous_events() {
let mut recorder = WorkflowRecorder::new();
recorder.start_recording();
recorder.record_event(button_event("Save", 1));
recorder.stop_recording();
recorder.start_recording();
assert_eq!(recorder.event_count(), 0);
}
#[test]
fn recorder_ignores_events_when_not_recording() {
let mut recorder = WorkflowRecorder::new();
recorder.record_event(button_event("Ignored", 0));
assert_eq!(recorder.event_count(), 0);
}
#[test]
fn recorder_accumulates_events_in_order() {
let mut recorder = WorkflowRecorder::new();
recorder.start_recording();
recorder.record_event(button_event("First", 1));
recorder.record_event(button_event("Second", 2));
assert_eq!(recorder.event_count(), 2);
}
#[test]
fn stop_recording_returns_all_events_and_stops() {
let mut recorder = WorkflowRecorder::new();
recorder.start_recording();
recorder.record_event(button_event("A", 1));
recorder.record_event(button_event("B", 2));
let events = recorder.stop_recording();
assert_eq!(events.len(), 2);
assert!(!recorder.is_recording());
assert_eq!(recorder.event_count(), 0);
}
#[test]
fn convenience_helpers_record_correct_action_types() {
let mut recorder = WorkflowRecorder::new();
recorder.start_recording();
recorder.record_click(50.0, 60.0, 7, "OK", "AXButton");
recorder.record_type("hello", 8, "Search", "AXTextField");
let events = recorder.stop_recording();
assert!(matches!(events[0].action, RecordedAction::Click { .. }));
assert!(matches!(events[1].action, RecordedAction::Type { .. }));
assert_eq!(events[1].element_label, "Search");
}
#[test]
fn serialize_and_deserialize_round_trip() {
let events = vec![
RecordedEvent {
timestamp: 0,
action: RecordedAction::Click { x: 1.0, y: 2.0 },
element_fingerprint: 42,
element_label: "OK".into(),
element_role: "AXButton".into(),
},
RecordedEvent {
timestamp: 100,
action: RecordedAction::Type {
text: "hello".into(),
},
element_fingerprint: 0,
element_label: "Input".into(),
element_role: "AXTextField".into(),
},
RecordedEvent {
timestamp: 200,
action: RecordedAction::KeyPress {
key: "Return".into(),
modifiers: vec!["Cmd".into()],
},
element_fingerprint: 0,
element_label: String::new(),
element_role: String::new(),
},
RecordedEvent {
timestamp: 300,
action: RecordedAction::Scroll { dx: 0.0, dy: -50.0 },
element_fingerprint: 0,
element_label: "List".into(),
element_role: "AXList".into(),
},
RecordedEvent {
timestamp: 400,
action: RecordedAction::Wait { duration_ms: 500 },
element_fingerprint: 0,
element_label: String::new(),
element_role: String::new(),
},
];
let json = WorkflowRecorder::serialize(&events).unwrap();
let restored = WorkflowRecorder::deserialize(&json).unwrap();
assert_eq!(events, restored);
}
#[test]
fn player_replays_wait_without_element_lookup() {
let events = vec![wait_event(0)];
let store = RefStore::new();
let player = WorkflowPlayer::new();
let result = player.replay(&events, &store);
assert!(result.success);
assert_eq!(result.events_executed, 1);
assert!(result.failures.is_empty());
}
#[test]
fn player_succeeds_when_element_found_by_fingerprint() {
let (store, fp) = store_with_button("Save");
let events = vec![button_event("Save", fp)];
let player = WorkflowPlayer::new();
let result = player.replay(&events, &store);
assert!(result.success);
assert_eq!(result.events_executed, 1);
assert!(result.adapted_events.is_empty());
}
#[test]
fn player_adapts_when_fingerprint_changes_but_label_matches() {
let (store, _real_fp) = store_with_button("Save");
let stale_fp = 9_999_999; let events = vec![button_event("Save", stale_fp)];
let player = WorkflowPlayer::new();
let result = player.replay(&events, &store);
assert!(result.success);
assert_eq!(result.events_executed, 1);
assert!(result.adapted_events.contains(&0));
}
#[test]
fn player_records_failure_when_element_not_found() {
let store = RefStore::new();
let events = vec![button_event("Ghost", 123)];
let player = WorkflowPlayer::new();
let result = player.replay(&events, &store);
assert!(!result.success);
assert_eq!(result.events_executed, 0);
assert_eq!(result.failures.len(), 1);
assert_eq!(result.failures[0].event_index, 0);
}
#[test]
fn player_continues_after_failure_when_not_aborting() {
let store = RefStore::new();
let events = vec![button_event("Ghost", 1), wait_event(0)];
let mut player = WorkflowPlayer::new();
player.abort_on_failure = false;
let result = player.replay(&events, &store);
assert!(!result.success);
assert_eq!(result.events_executed, 1); assert_eq!(result.failures.len(), 1);
}
#[test]
fn player_aborts_on_first_failure_when_configured() {
let store = RefStore::new();
let events = vec![button_event("Ghost", 1), wait_event(0)];
let mut player = WorkflowPlayer::new();
player.abort_on_failure = true;
let result = player.replay(&events, &store);
assert!(!result.success);
assert_eq!(result.events_executed, 0);
assert_eq!(result.total_events, 2);
}
#[test]
fn player_uses_raw_coordinates_for_zero_fingerprint_click() {
let store = RefStore::new();
let events = vec![RecordedEvent {
timestamp: 0,
action: RecordedAction::Click { x: 300.0, y: 400.0 },
element_fingerprint: 0,
element_label: String::new(),
element_role: String::new(),
}];
let player = WorkflowPlayer::new();
let result = player.replay(&events, &store);
assert!(result.success);
assert_eq!(result.events_executed, 1);
}
#[test]
fn replay_result_tracks_total_vs_executed_counts() {
let store = RefStore::new();
let events = vec![wait_event(0), button_event("Missing", 1), wait_event(0)];
let player = WorkflowPlayer::new();
let result = player.replay(&events, &store);
assert_eq!(result.total_events, 3);
assert_eq!(result.events_executed, 2);
}
}