use std::collections::HashMap;
use std::path::Path;
use std::time::{Duration, Instant};
use anyhow::{Result, bail};
use pixelactions_core::convert::{
ResolvedPoint, Space, native_space, near_screen_corner, to_space,
};
use pixelactions_core::flow::{Flow, Step, Verify};
use pixelactions_core::plan::Plan;
use pixelactions_core::report::{RunReport, StepOutcome, StepReport};
use pixelcoords_core::session::MonitorRecord;
use crate::inject::{Button, Injector};
use crate::verify;
pub const WATCHDOG: Duration = Duration::from_secs(120);
pub type Corrections = HashMap<String, ResolvedPoint>;
pub fn preflight(
flow: &Flow,
session: &Path,
monitors: &[MonitorRecord],
space: Space,
verifier: &mut dyn FnMut(&Path, Option<&str>) -> Result<verify::FindReport>,
found: &dyn Fn(&str),
) -> Result<Corrections> {
if !flow.settings.relocate {
return Ok(Corrections::new());
}
let targets = flow.acting_targets();
if targets.is_empty() {
return Ok(Corrections::new());
}
match locate_each(&targets, session, monitors, space, verifier, found)? {
Ok(corrections) => Ok(corrections),
Err(unconfirmed) => bail!(
"the screen no longer matches the session: {}. Re-mark with pixelcoords, or set \
relocate = false in the flow to act on the saved coordinates anyway",
unconfirmed.join("; ")
),
}
}
fn locate_each(
targets: &[&str],
session: &Path,
monitors: &[MonitorRecord],
space: Space,
verifier: &mut dyn FnMut(&Path, Option<&str>) -> Result<verify::FindReport>,
found: &dyn Fn(&str),
) -> Result<std::result::Result<Corrections, Vec<String>>> {
let mut located = Corrections::new();
let mut unconfirmed = Vec::new();
for label in targets {
let report = verifier(session, Some(label))?;
if !report.is_confirmed(label) {
let reason = report
.result_for(label)
.map_or_else(|| "not in the report".to_string(), describe);
unconfirmed.push(format!("{label} ({reason})"));
continue;
}
located.extend(corrections(&report, &[label], monitors, space));
found(label);
}
if unconfirmed.is_empty() {
return Ok(Ok(located));
}
Ok(Err(unconfirmed))
}
fn precheck(
flow: &Flow,
planned: &pixelactions_core::plan::PlannedStep,
session: &Path,
monitors: &[MonitorRecord],
known: &mut Corrections,
verifier: &mut dyn FnMut(&Path, Option<&str>) -> Result<verify::FindReport>,
) -> Result<()> {
if flow.settings.verify != Verify::Each || !planned.step.injects() {
return Ok(());
}
let targets = planned.step.targets();
if targets.is_empty() {
return Ok(());
}
match locate_each(
&targets,
session,
monitors,
flow.settings.space,
verifier,
&|_| {},
)? {
Err(unconfirmed) => bail!(
"cannot act on {}. Nothing was injected for this step",
unconfirmed.join("; ")
),
Ok(fresh) => {
for label in &targets {
match fresh.get(*label) {
Some(point) => known.insert((*label).to_string(), *point),
None => known.remove(*label),
};
}
Ok(())
}
}
}
fn resolve_space(space: Space) -> Space {
match space {
Space::Auto => native_space(),
other => other,
}
}
fn failsafe_gate(
flow: &Flow,
injector: &mut dyn Injector,
monitors: &[MonitorRecord],
) -> Result<()> {
if !flow.settings.failsafe {
return Ok(());
}
let (x, y) = injector.cursor().map_err(|error| {
anyhow::anyhow!("failsafe is on but the cursor position could not be read: {error}")
})?;
if !near_screen_corner(
monitors,
resolve_space(flow.settings.space),
x,
y,
flow.settings.failsafe_margin,
) {
return Ok(());
}
bail!(
"kill switch: the cursor is in a screen corner ({x:.0}, {y:.0}), so the run stopped \
before this step. Move it away and run again, or set failsafe = false"
)
}
fn poll_until(
flow: &Flow,
session: &Path,
target: &str,
want_present: bool,
waiter: &Waiter<'_>,
verifier: &mut dyn FnMut(&Path, Option<&str>) -> Result<verify::FindReport>,
) -> Result<StepOutcome> {
let report = waiter(
session,
target,
want_present,
Duration::from_millis(flow.settings.timeout_ms),
Duration::from_millis(flow.settings.poll_ms.max(1)),
)?;
if report.ok {
return Ok(StepOutcome::Verified);
}
let wanted = if want_present { "appear" } else { "disappear" };
let last_look = drift_note(session, target, verifier);
bail!(
"timed out after {}ms waiting for {target:?} to {wanted} \
({} polls, best match score {:.3}) — {last_look}",
report.elapsed_ms,
report.polls,
report.best_score(),
);
}
fn drift_note(
session: &Path,
target: &str,
verifier: &mut dyn FnMut(&Path, Option<&str>) -> Result<verify::FindReport>,
) -> String {
let Ok(report) = verifier(session, Some(target)) else {
return "a follow-up look failed too".to_string();
};
let Some(result) = report.result_for(target) else {
return "the report never mentioned it".to_string();
};
match result.delta {
Some(delta) if result.found && !result.ambiguous && (delta.dx != 0 || delta.dy != 0) => {
format!(
"last look: it is on screen, {} physical px from where it was marked, \
so `wait` was scoring the old position",
format_args!("({}, {})", delta.dx, delta.dy),
)
}
_ => format!("last look: {}", describe(result)),
}
}
pub fn corrections(
report: &verify::FindReport,
targets: &[&str],
monitors: &[MonitorRecord],
space: Space,
) -> Corrections {
let mut corrections = Corrections::new();
for label in targets {
let moved = report
.result_for(label)
.and_then(|result| result.delta)
.is_some_and(|delta| delta.dx != 0 || delta.dy != 0);
if !moved {
continue;
}
let Some((monitor_index, local)) = report.corrected_point(label) else {
continue;
};
let Some(monitor) = monitors.iter().find(|m| m.index == monitor_index) else {
continue;
};
let global_x = monitor.origin_px.x + local.x;
let global_y = monitor.origin_px.y + local.y;
if let Some(point) = to_space(monitors, global_x, global_y, space) {
corrections.insert((*label).to_string(), point);
}
}
corrections
}
fn describe(result: &verify::FindResult) -> String {
if result.ambiguous {
return format!("matched in more than one place, score {:.3}", result.score);
}
if !result.found {
let Some(reason) = result.reason.clone() else {
return format!("no match above the floor, best score {:.3}", result.score);
};
return format!("{reason} (score {:.3})", result.score);
}
match result.delta {
Some(delta) if delta.dx != 0 || delta.dy != 0 => {
format!("moved by ({}, {})", delta.dx, delta.dy)
}
_ => "unchanged".to_string(),
}
}
pub struct Context<'a> {
pub flow: &'a Flow,
pub plan: &'a Plan,
pub session: &'a Path,
pub monitors: &'a [MonitorRecord],
pub corrections: &'a Corrections,
pub checked: bool,
pub progress: &'a dyn Fn(&StepReport),
pub waiter: &'a Waiter<'a>,
}
pub type Waiter<'a> =
dyn Fn(&Path, &str, bool, Duration, Duration) -> Result<verify::WaitReport> + 'a;
pub fn real_waiter() -> &'static Waiter<'static> {
&|session, label, want_present, timeout, interval| {
verify::wait(session, label, want_present, timeout, interval)
}
}
pub fn silent() -> &'static dyn Fn(&StepReport) {
&|_: &StepReport| {}
}
pub fn execute(
injector: &mut dyn Injector,
context: &Context<'_>,
verifier: &mut dyn FnMut(&Path, Option<&str>) -> Result<verify::FindReport>,
) -> RunReport {
let Context {
flow,
plan,
session,
monitors,
corrections,
checked,
progress,
waiter,
} = *context;
let started = Instant::now();
let settle = Duration::from_millis(flow.settings.settle_ms);
let mut steps: Vec<StepReport> = Vec::with_capacity(plan.steps.len());
let mut failed = false;
let mut corrections = corrections.clone();
let mut fresh = checked;
for planned in &plan.steps {
if failed {
let skipped = record(planned, StepOutcome::Skipped, None, 0);
progress(&skipped);
steps.push(skipped);
continue;
}
if started.elapsed() > WATCHDOG {
let timed_out = record(
planned,
StepOutcome::Failed,
Some("watchdog: the run exceeded its time budget".into()),
0,
);
progress(&timed_out);
steps.push(timed_out);
failed = true;
continue;
}
let step_started = Instant::now();
let gate = failsafe_gate(flow, injector, monitors).and_then(|()| {
if fresh {
return Ok(());
}
precheck(flow, planned, session, monitors, &mut corrections, verifier)
});
if let Err(refusal) = gate {
let refused = record_with(
planned,
StepOutcome::Refused,
Some(format!("{refusal:#}")),
step_started.elapsed().as_millis() as u64,
corrected_points(planned, &corrections),
);
progress(&refused);
steps.push(refused);
failed = true;
continue;
}
fresh = false;
let points = corrected_points(planned, &corrections);
let outcome = perform(injector, &planned.step, planned, &points, settle)
.and_then(|()| confirm(flow, &planned.step, session, waiter, verifier));
let elapsed = step_started.elapsed().as_millis() as u64;
let done = match outcome {
Ok(outcome) => record_with(planned, outcome, None, elapsed, points),
Err(error) => {
failed = true;
record_with(
planned,
StepOutcome::Failed,
Some(format!("{error:#}")),
elapsed,
points,
)
}
};
progress(&done);
steps.push(done);
}
RunReport {
schema: RunReport::SCHEMA,
session: session.display().to_string(),
executed: true,
steps,
}
}
fn record(
planned: &pixelactions_core::plan::PlannedStep,
outcome: StepOutcome,
detail: Option<String>,
elapsed_ms: u64,
) -> StepReport {
let points = planned.points.clone();
record_with(planned, outcome, detail, elapsed_ms, points)
}
fn record_with(
planned: &pixelactions_core::plan::PlannedStep,
outcome: StepOutcome,
detail: Option<String>,
elapsed_ms: u64,
points: Vec<ResolvedPoint>,
) -> StepReport {
StepReport {
index: planned.index,
summary: planned.summary.clone(),
outcome,
points,
detail,
elapsed_ms,
}
}
fn corrected_points(
planned: &pixelactions_core::plan::PlannedStep,
corrections: &Corrections,
) -> Vec<ResolvedPoint> {
planned
.step
.targets()
.iter()
.zip(planned.points.iter())
.map(|(label, planned_point)| corrections.get(*label).copied().unwrap_or(*planned_point))
.collect()
}
fn perform(
injector: &mut dyn Injector,
step: &Step,
planned: &pixelactions_core::plan::PlannedStep,
points: &[ResolvedPoint],
settle: Duration,
) -> Result<()> {
let points_for_step = points.to_vec();
match step {
Step::Click { .. } => {
let point = first_point(planned, &points_for_step)?;
injector.move_to(point.0, point.1)?;
std::thread::sleep(settle);
injector.click(Button::Left)?;
}
Step::DoubleClick { .. } => {
let point = first_point(planned, &points_for_step)?;
injector.move_to(point.0, point.1)?;
std::thread::sleep(settle);
injector.double_click(Button::Left)?;
}
Step::Drag { .. } => {
let [from, to] = two_points(&points_for_step)?;
injector.move_to(from.0, from.1)?;
std::thread::sleep(settle);
injector.press(Button::Left)?;
for step_index in 1..=DRAG_STEPS {
let progress = f64::from(step_index) / f64::from(DRAG_STEPS);
injector.move_to(
from.0 + (to.0 - from.0) * progress,
from.1 + (to.1 - from.1) * progress,
)?;
}
injector.release(Button::Left)?;
}
Step::Scroll { amount, axis, .. } => {
let point = first_point(planned, &points_for_step)?;
injector.move_to(point.0, point.1)?;
std::thread::sleep(settle);
injector.scroll(*amount, *axis)?;
}
Step::Type { text } => injector.text(text)?,
Step::Key { chord } => injector.chord(chord)?,
Step::Pause { ms } => std::thread::sleep(Duration::from_millis(*ms)),
Step::Verify { .. } | Step::WaitFor { .. } | Step::WaitGone { .. } => {}
}
std::thread::sleep(settle);
Ok(())
}
const DRAG_STEPS: u32 = 12;
fn confirm(
flow: &Flow,
step: &Step,
session: &Path,
waiter: &Waiter<'_>,
verifier: &mut dyn FnMut(&Path, Option<&str>) -> Result<verify::FindReport>,
) -> Result<StepOutcome> {
match step {
Step::WaitFor { target } => poll_until(flow, session, target, true, waiter, verifier),
Step::WaitGone { target } => poll_until(flow, session, target, false, waiter, verifier),
Step::Verify { target } => check_once(session, target, verifier),
_ => Ok(StepOutcome::Executed),
}
}
fn check_once(
session: &Path,
target: &str,
verifier: &mut dyn FnMut(&Path, Option<&str>) -> Result<verify::FindReport>,
) -> Result<StepOutcome> {
let report = verifier(session, Some(target))?;
if report.is_confirmed(target) {
return Ok(StepOutcome::Verified);
}
let detail = report.result_for(target).map_or_else(
|| format!("region {target:?} was not in the report"),
|result| format!("region {target:?}: {}", describe(result)),
);
bail!(detail)
}
fn first_point(
planned: &pixelactions_core::plan::PlannedStep,
points: &[ResolvedPoint],
) -> Result<(f64, f64)> {
let Some(point) = points.first() else {
bail!("step {} resolved to no point", planned.index + 1);
};
Ok((point.x, point.y))
}
fn two_points(points: &[ResolvedPoint]) -> Result<[(f64, f64); 2]> {
let [from, to] = points else {
bail!("drag needs two resolved points, got {}", points.len());
};
Ok([(from.x, from.y), (to.x, to.y)])
}
#[cfg(test)]
mod tests {
use pixelactions_core::convert::{ResolvedPoint, Space};
use pixelactions_core::plan::PlannedStep;
use super::*;
use crate::inject::Recording;
fn point(x: f64, y: f64) -> ResolvedPoint {
ResolvedPoint {
x,
y,
space: Space::Logical,
monitor: 0,
scale: 2.0,
}
}
fn planned(index: usize, step: Step, points: Vec<ResolvedPoint>) -> PlannedStep {
PlannedStep {
index,
summary: step.summary(),
step,
points,
}
}
fn flow_with(verify: Verify) -> Flow {
let mut flow =
Flow::parse("session = \"s\"\n\n[[step]]\naction = \"type\"\ntext = \"x\"\n")
.expect("valid");
flow.settings.verify = verify;
flow.settings.settle_ms = 0;
flow
}
const TEST_LABELS: &[&str] = &[
"submit", "next", "email", "results", "a", "b", "x", "dialog",
];
fn report_of(label: Option<&str>, found: bool) -> verify::FindReport {
let labels: Vec<&str> = label.map_or_else(|| TEST_LABELS.to_vec(), |one| vec![one]);
let results: Vec<String> = labels
.iter()
.map(|l| format!(r#"{{"label":"{l}","found":{found}}}"#))
.collect();
serde_json::from_str(&format!(
r#"{{"all_relocated":{found},"results":[{}]}}"#,
results.join(",")
))
.expect("fixture")
}
fn always_found() -> impl FnMut(&Path, Option<&str>) -> Result<verify::FindReport> {
|_, label| Ok(report_of(label, true))
}
fn never_found() -> impl FnMut(&Path, Option<&str>) -> Result<verify::FindReport> {
|_, label| Ok(report_of(label, false))
}
fn waits_ok() -> &'static Waiter<'static> {
&|_, _, _, _, _| {
Ok(verify::WaitReport {
ok: true,
polls: 1,
elapsed_ms: 0,
results: vec![verify::WaitResult { score: 1.0 }],
})
}
}
fn waits_out(score: f64) -> &'static Waiter<'static> {
Box::leak(Box::new(
move |_: &Path, _: &str, _: bool, timeout: Duration, interval: Duration| {
let polls = u32::try_from(timeout.as_millis() / interval.as_millis().max(1))
.unwrap_or(u32::MAX);
Ok(verify::WaitReport {
ok: false,
polls,
elapsed_ms: u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX),
results: vec![verify::WaitResult { score }],
})
},
))
}
#[test]
fn a_click_moves_then_clicks_in_that_order() {
let mut injector = Recording::default();
let plan = Plan {
steps: vec![planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(100.0, 200.0)],
)],
};
let report = execute(
&mut injector,
&Context {
flow: &flow_with(Verify::None),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(injector.events, vec!["move 100,200", "click"]);
assert_eq!(report.exit_code(), 0);
}
#[test]
fn a_cursor_in_a_corner_stops_the_run_before_the_step() {
let mut injector = Recording {
cursor: (1511.0, 981.0),
..Default::default()
};
let plan = Plan {
steps: vec![planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(100.0, 200.0)],
)],
};
let mut flow = flow_with(Verify::None);
flow.settings.space = Space::Logical;
let report = execute(
&mut injector,
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(
injector.events,
Vec::<String>::new(),
"nothing may be injected once the kill switch trips"
);
assert_eq!(
report.exit_code(),
3,
"a person stopping the run is a refusal, not a failure"
);
assert_eq!(report.steps[0].outcome, StepOutcome::Refused);
let detail = report.steps[0].detail.as_deref().unwrap_or_default();
assert!(detail.contains("kill switch"), "says why: {detail}");
}
#[test]
fn the_kill_switch_can_be_turned_off_deliberately() {
let mut injector = Recording {
cursor: (1511.0, 981.0),
..Default::default()
};
let plan = Plan {
steps: vec![planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(100.0, 200.0)],
)],
};
let mut flow = flow_with(Verify::None);
flow.settings.space = Space::Logical;
flow.settings.failsafe = false;
let report = execute(
&mut injector,
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(injector.events, vec!["move 100,200", "click"]);
assert_eq!(report.exit_code(), 0);
}
#[test]
fn an_unreadable_cursor_refuses_the_step_rather_than_waving_it_through() {
struct Blind;
impl Injector for Blind {
fn move_to(&mut self, _x: f64, _y: f64) -> Result<()> {
Ok(())
}
fn click(&mut self, _button: Button) -> Result<()> {
Ok(())
}
fn double_click(&mut self, _button: Button) -> Result<()> {
Ok(())
}
fn press(&mut self, _button: Button) -> Result<()> {
Ok(())
}
fn release(&mut self, _button: Button) -> Result<()> {
Ok(())
}
fn text(&mut self, _text: &str) -> Result<()> {
Ok(())
}
fn chord(&mut self, _chord: &str) -> Result<()> {
Ok(())
}
fn scroll(&mut self, _amount: i32, _axis: pixelactions_core::flow::Axis) -> Result<()> {
Ok(())
}
fn cursor(&mut self) -> Result<(f64, f64)> {
bail!("no cursor here")
}
fn probe(&mut self) -> Result<()> {
Ok(())
}
}
let plan = Plan {
steps: vec![planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(100.0, 200.0)],
)],
};
let report = execute(
&mut Blind,
&Context {
flow: &flow_with(Verify::None),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(report.exit_code(), 3);
let detail = report.steps[0].detail.as_deref().unwrap_or_default();
assert!(detail.contains("failsafe"), "names the setting: {detail}");
}
#[test]
fn looking_at_a_moved_region_is_never_refused_on_bounds() {
let mut corrections = Corrections::new();
corrections.insert("submit".to_string(), point(9_000.0, 9_000.0));
let plan = Plan {
steps: vec![planned(
0,
Step::WaitFor {
target: "submit".into(),
},
vec![point(9_000.0, 9_000.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow_with(Verify::Each),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &corrections,
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Verified);
}
#[test]
fn a_scroll_hovers_the_region_first_then_turns_the_wheel() {
let mut injector = Recording::default();
let plan = Plan {
steps: vec![planned(
0,
Step::Scroll {
target: "results".into(),
amount: -3,
axis: pixelactions_core::flow::Axis::Vertical,
},
vec![point(100.0, 200.0)],
)],
};
execute(
&mut injector,
&Context {
flow: &flow_with(Verify::None),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(injector.events, vec!["move 100,200", "scroll v-3"]);
}
#[test]
fn a_scroll_is_never_checked_against_its_own_region_afterwards() {
let mut looks = 0;
let mut once = move |_: &Path, label: Option<&str>| {
looks += 1;
Ok(report_of(label, looks == 1))
};
let plan = Plan {
steps: vec![planned(
0,
Step::Scroll {
target: "results".into(),
amount: 3,
axis: pixelactions_core::flow::Axis::Vertical,
},
vec![point(100.0, 200.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow_with(Verify::Each),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut once,
);
assert_eq!(report.steps[0].outcome, StepOutcome::Executed);
assert_eq!(report.exit_code(), 0);
}
#[test]
fn a_scroll_still_obeys_the_kill_switch() {
let mut injector = Recording {
cursor: (1511.0, 981.0),
..Default::default()
};
let plan = Plan {
steps: vec![planned(
0,
Step::Scroll {
target: "results".into(),
amount: 3,
axis: pixelactions_core::flow::Axis::Vertical,
},
vec![point(100.0, 200.0)],
)],
};
let mut flow = flow_with(Verify::None);
flow.settings.space = Space::Logical;
let report = execute(
&mut injector,
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Refused);
assert!(injector.events.is_empty(), "nothing was injected");
}
#[test]
fn a_drag_presses_moves_through_and_releases() {
let mut injector = Recording::default();
let plan = Plan {
steps: vec![planned(
0,
Step::Drag {
from: "a".into(),
to: "b".into(),
},
vec![point(0.0, 0.0), point(120.0, 0.0)],
)],
};
execute(
&mut injector,
&Context {
flow: &flow_with(Verify::None),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(injector.events.first().expect("moved"), "move 0,0");
assert_eq!(injector.events.get(1).expect("pressed"), "press");
assert_eq!(injector.events.last().expect("released"), "release");
let moves = injector
.events
.iter()
.filter(|e| e.starts_with("move"))
.count();
assert!(moves > 2, "drag interpolates: {:?}", injector.events);
}
#[test]
fn a_region_that_cannot_be_found_refuses_before_injecting_anything() {
let mut injector = Recording::default();
let plan = Plan {
steps: vec![
planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(10.0, 10.0)],
),
planned(
1,
Step::Click {
target: "next".into(),
},
vec![point(20.0, 20.0)],
),
],
};
let report = execute(
&mut injector,
&Context {
flow: &flow_with(Verify::Each),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut never_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Refused);
assert_eq!(report.steps[1].outcome, StepOutcome::Skipped);
assert_eq!(report.exit_code(), 3);
assert!(
injector.events.is_empty(),
"nothing may be injected when the region could not be confirmed"
);
}
#[test]
fn an_acting_step_reports_executed_never_verified() {
let plan = Plan {
steps: vec![planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(1.0, 1.0)],
)],
};
for verify in [Verify::None, Verify::Each] {
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow_with(verify),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(
report.steps[0].outcome,
StepOutcome::Executed,
"verify = {verify:?}"
);
}
}
#[test]
fn the_check_runs_before_the_step_not_after() {
let mut looks = 0;
let mut once = move |_: &Path, label: Option<&str>| {
looks += 1;
Ok(report_of(label, looks == 1))
};
let plan = Plan {
steps: vec![planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(5.0, 6.0)],
)],
};
let mut injector = Recording::default();
let report = execute(
&mut injector,
&Context {
flow: &flow_with(Verify::Each),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut once,
);
assert_eq!(report.steps[0].outcome, StepOutcome::Executed);
assert_eq!(injector.events, vec!["move 5,6", "click"]);
}
#[test]
fn a_step_acts_on_where_a_region_moved_mid_run() {
let mut looks = 0;
let mut drifting = move |_: &Path, _label: Option<&str>| {
looks += 1;
let shift = if looks >= 2 {
r#","new_px":{"x":800,"y":280,"w":100,"h":80},"delta":{"dx":0,"dy":-120}"#
} else {
""
};
Ok(serde_json::from_str(&format!(
r#"{{"all_relocated":true,"results":[{{"label":"submit","found":true,"monitor":0{shift}}}]}}"#
))
.expect("fixture"))
};
let step = || {
planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(425.0, 220.0)],
)
};
let plan = Plan {
steps: vec![step(), step()],
};
let mut flow = flow_with(Verify::Each);
flow.settings.space = Space::Logical;
let mut injector = Recording::default();
execute(
&mut injector,
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut drifting,
);
assert_eq!(
injector.events,
vec!["move 425,220", "click", "move 425,160", "click"]
);
}
#[test]
fn a_verify_step_checks_even_when_the_run_setting_is_none() {
let plan = Plan {
steps: vec![planned(
0,
Step::Verify {
target: "done".into(),
},
vec![point(1.0, 1.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow_with(Verify::None),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut never_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Failed);
}
fn monitors() -> Vec<MonitorRecord> {
vec![MonitorRecord {
index: 0,
name: "retina".into(),
primary: true,
origin_px: pixelcoords_core::geometry::Point::new(0, 0),
size_px: pixelcoords_core::geometry::Size::new(3024, 1964),
scale: 2.0,
}]
}
fn moved_up() -> impl FnMut(&Path, Option<&str>) -> Result<verify::FindReport> {
|_, _| {
Ok(serde_json::from_str(
r#"{"all_relocated":true,"results":[
{"label":"submit","found":true,"ambiguous":false,"score":0.99,
"monitor":0,"new_px":{"x":812,"y":320,"w":96,"h":40},
"delta":{"dx":0,"dy":-120}}]}"#,
)
.expect("fixture"))
}
}
#[test]
fn a_moved_region_is_clicked_where_it_is_now_not_where_it_was() {
let flow =
Flow::parse("session = \"s\"\n\n[[step]]\naction = \"click\"\ntarget = \"submit\"\n")
.expect("valid");
let corrections = preflight(
&flow,
Path::new("/tmp/session"),
&monitors(),
Space::Logical,
&mut moved_up(),
&|_| {},
)
.expect("regions found");
let corrected = corrections.get("submit").expect("corrected");
assert!((corrected.x - 430.0).abs() < f64::EPSILON);
assert!((corrected.y - 170.0).abs() < f64::EPSILON);
let mut injector = Recording::default();
let plan = Plan {
steps: vec![planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(406.0, 230.0)],
)],
};
let report = execute(
&mut injector,
&Context {
flow: &flow_with(Verify::None),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &corrections,
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut moved_up(),
);
assert_eq!(injector.events, vec!["move 430,170", "click"]);
assert!((report.steps[0].points[0].x - 430.0).abs() < f64::EPSILON);
}
#[test]
fn preflight_refuses_when_a_target_cannot_be_found() {
let flow =
Flow::parse("session = \"s\"\n\n[[step]]\naction = \"click\"\ntarget = \"submit\"\n")
.expect("valid");
let error = preflight(
&flow,
Path::new("/tmp/session"),
&monitors(),
Space::Logical,
&mut never_found(),
&|_| {},
)
.expect_err("must refuse");
let message = format!("{error:#}");
assert!(message.contains("no longer matches"), "message: {message}");
assert!(message.contains("submit"), "names the region: {message}");
}
#[test]
fn preflight_is_skipped_entirely_when_relocation_is_off() {
let mut flow =
Flow::parse("session = \"s\"\n\n[[step]]\naction = \"click\"\ntarget = \"submit\"\n")
.expect("valid");
flow.settings.relocate = false;
let corrections = preflight(
&flow,
Path::new("/tmp/session"),
&monitors(),
Space::Logical,
&mut never_found(),
&|_| {},
)
.expect("no preflight");
assert!(corrections.is_empty());
}
fn appears_after(
polls: usize,
) -> impl FnMut(&Path, Option<&str>) -> Result<verify::FindReport> {
let mut seen = 0usize;
move |_, label| {
seen += 1;
let found = seen > polls;
Ok(serde_json::from_str(&format!(
r#"{{"all_relocated":{found},"results":[{{"label":"{}","found":{found}}}]}}"#,
label.unwrap_or("x")
))
.expect("fixture"))
}
}
fn waiting_flow(step: &str, timeout_ms: u64) -> Flow {
let mut flow = Flow::parse(&format!("session = \"s\"\n\n{step}")).expect("valid");
flow.settings.settle_ms = 0;
flow.settings.poll_ms = 1;
flow.settings.timeout_ms = timeout_ms;
flow
}
#[test]
fn wait_for_polls_until_the_region_appears() {
let flow = waiting_flow(
"[[step]]\naction = \"wait_for\"\ntarget = \"dialog\"\n",
5_000,
);
let plan = Plan {
steps: vec![planned(
0,
Step::WaitFor {
target: "dialog".into(),
},
vec![point(1.0, 1.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut appears_after(3),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Verified);
}
#[test]
fn a_timeout_reports_how_hard_it_looked() {
let mut flow = flow_with(Verify::Each);
flow.settings.timeout_ms = 30;
flow.settings.poll_ms = 10;
let plan = Plan {
steps: vec![planned(
0,
Step::WaitFor {
target: "dialog".into(),
},
vec![point(10.0, 10.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_out(0.42),
},
&mut never_found(),
);
let detail = report.steps[0].detail.clone().expect("explained");
assert!(detail.contains("polls"), "says how many times: {detail}");
assert!(
detail.contains("best match score"),
"says how close: {detail}"
);
assert!(detail.contains("last look"), "says what it saw: {detail}");
}
#[test]
fn an_ambiguous_match_is_never_reported_as_not_matching() {
let mut ambiguous = |_: &Path, label: Option<&str>| {
Ok(serde_json::from_str(&format!(
r#"{{"all_relocated":false,"results":[{{"label":"{}","found":true,"ambiguous":true,"score":1.0}}]}}"#,
label.unwrap_or("x")
))
.expect("fixture"))
};
let plan = Plan {
steps: vec![planned(
0,
Step::Verify {
target: "profile".into(),
},
vec![point(10.0, 10.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow_with(Verify::Each),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut ambiguous,
);
let detail = report.steps[0].detail.clone().expect("explained");
assert!(
detail.contains("more than one place"),
"names the real problem: {detail}"
);
}
#[test]
fn wait_for_times_out_honestly_rather_than_hanging() {
let flow = waiting_flow("[[step]]\naction = \"wait_for\"\ntarget = \"dialog\"\n", 5);
let plan = Plan {
steps: vec![planned(
0,
Step::WaitFor {
target: "dialog".into(),
},
vec![point(1.0, 1.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_out(0.10),
},
&mut never_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Failed);
let detail = report.steps[0].detail.clone().expect("explained");
assert!(detail.contains("timed out"), "detail: {detail}");
assert!(
detail.contains("appear"),
"says what it waited for: {detail}"
);
}
#[test]
fn a_timeout_on_a_region_that_moved_says_so() {
let flow = waiting_flow("[[step]]\naction = \"wait_for\"\ntarget = \"submit\"\n", 5);
let plan = Plan {
steps: vec![planned(
0,
Step::WaitFor {
target: "submit".into(),
},
vec![point(1.0, 1.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_out(0.05),
},
&mut moved_up(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Failed);
let detail = report.steps[0].detail.clone().expect("explained");
assert!(detail.contains("timed out"), "detail: {detail}");
assert!(
detail.contains("it is on screen"),
"distinguishes moved from absent: {detail}"
);
assert!(
detail.contains("(0, -120)"),
"names the drift so it can be fixed: {detail}"
);
}
#[test]
fn wait_gone_succeeds_when_the_region_is_absent() {
let flow = waiting_flow(
"[[step]]\naction = \"wait_gone\"\ntarget = \"spinner\"\n",
5_000,
);
let plan = Plan {
steps: vec![planned(
0,
Step::WaitGone {
target: "spinner".into(),
},
vec![point(1.0, 1.0)],
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut never_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Verified);
}
#[test]
fn a_region_that_moved_a_long_way_is_still_acted_on() {
let mut corrections = Corrections::new();
corrections.insert("submit".into(), point(10.0, 10.0));
let plan = Plan {
steps: vec![planned(
0,
Step::Click {
target: "submit".into(),
},
vec![point(900.0, 700.0)],
)],
};
let mut injector = Recording::default();
let report = execute(
&mut injector,
&Context {
flow: &flow_with(Verify::None),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &corrections,
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Executed);
assert_eq!(
injector.events,
vec!["move 10,10", "click"],
"acts where the region is now, not where it was marked"
);
}
#[test]
fn an_ambiguous_region_never_produces_a_correction_to_act_on() {
let report: verify::FindReport = serde_json::from_str(
r#"{"all_relocated":false,"results":[{"label":"submit","found":true,"ambiguous":true,"score":1.0,"monitor":0,"new_px":{"x":10,"y":10,"w":4,"h":4}}]}"#,
)
.expect("fixture");
assert!(!report.is_confirmed("submit"));
assert!(
report.corrected_point("submit").is_none(),
"an ambiguous match must never become a point to act on"
);
let corrections = corrections(&report, &["submit"], &monitors(), Space::Logical);
assert!(corrections.is_empty());
}
#[test]
fn a_pause_step_injects_nothing_and_succeeds() {
let mut flow = flow_with(Verify::Each);
flow.settings.settle_ms = 0;
let plan = Plan {
steps: vec![planned(0, Step::Pause { ms: 1 }, Vec::new())],
};
let mut injector = Recording::default();
let report = execute(
&mut injector,
&Context {
flow: &flow,
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut never_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Executed);
assert!(injector.events.is_empty());
}
#[test]
fn keyboard_steps_report_executed_never_verified() {
let plan = Plan {
steps: vec![planned(
0,
Step::Type {
text: "hello".into(),
},
Vec::new(),
)],
};
let report = execute(
&mut Recording::default(),
&Context {
flow: &flow_with(Verify::Each),
plan: &plan,
session: Path::new("/tmp/session"),
monitors: &monitors(),
corrections: &Corrections::new(),
checked: false,
progress: silent(),
waiter: waits_ok(),
},
&mut always_found(),
);
assert_eq!(report.steps[0].outcome, StepOutcome::Executed);
}
}