1use std::collections::BTreeMap;
2use std::fmt::{self, Display};
3use std::fs::{self, OpenOptions};
4use std::io::Write;
5use std::path::{Path, PathBuf};
6use std::thread;
7use std::time::Duration;
8use std::time::{SystemTime, UNIX_EPOCH};
9
10pub type Result<T> = std::result::Result<T, FerrisError>;
11
12#[derive(Debug, Clone)]
13pub struct FerrisError {
14 pub kind: ErrorKind,
15 pub message: String,
16}
17
18impl FerrisError {
19 pub fn new(kind: ErrorKind, message: impl Into<String>) -> Self {
20 Self {
21 kind,
22 message: message.into(),
23 }
24 }
25}
26
27impl Display for FerrisError {
28 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
29 write!(f, "{}: {}", self.kind.as_str(), self.message)
30 }
31}
32
33impl std::error::Error for FerrisError {}
34
35impl From<std::io::Error> for FerrisError {
36 fn from(error: std::io::Error) -> Self {
37 Self::new(ErrorKind::Storage, error.to_string())
38 }
39}
40
41#[derive(Debug, Clone, Copy, PartialEq, Eq)]
42pub enum ErrorKind {
43 Capture,
44 Permission,
45 Coordinate,
46 Agent,
47 Protocol,
48 Execution,
49 Storage,
50 Platform,
51 UserInterrupt,
52}
53
54impl ErrorKind {
55 pub fn as_str(self) -> &'static str {
56 match self {
57 Self::Capture => "capture_error",
58 Self::Permission => "permission_error",
59 Self::Coordinate => "coordinate_error",
60 Self::Agent => "agent_error",
61 Self::Protocol => "protocol_error",
62 Self::Execution => "execution_error",
63 Self::Storage => "storage_error",
64 Self::Platform => "platform_error",
65 Self::UserInterrupt => "user_interrupt",
66 }
67 }
68}
69
70#[derive(Debug, Clone, PartialEq)]
71pub struct ScreenInfo {
72 pub screen_id: String,
73 pub name: String,
74 pub is_primary: bool,
75 pub origin_x: i32,
76 pub origin_y: i32,
77 pub native_width: u32,
78 pub native_height: u32,
79 pub scale_factor: f32,
80}
81
82#[derive(Debug, Clone, PartialEq)]
83pub struct CapturedScreen {
84 pub screen: ScreenInfo,
85 pub image_width: u32,
86 pub image_height: u32,
87 pub screenshot_path: PathBuf,
88 pub metadata_path: PathBuf,
89}
90
91#[derive(Debug, Clone, Copy, PartialEq, Eq)]
92pub enum CoordinateMode {
93 Normalized1000,
94 ImagePixels,
95 NativePixels,
96}
97
98impl CoordinateMode {
99 pub fn as_str(self) -> &'static str {
100 match self {
101 Self::Normalized1000 => "normalized-1000",
102 Self::ImagePixels => "image-pixels",
103 Self::NativePixels => "native-pixels",
104 }
105 }
106}
107
108#[derive(Debug, Clone, PartialEq, Eq)]
109pub enum CaptureTarget {
110 All,
111 Screen(String),
112}
113
114#[derive(Debug, Clone, PartialEq, Eq)]
115pub enum ImageFormat {
116 Jpg,
117 Png,
118}
119
120impl ImageFormat {
121 pub fn extension(&self) -> &'static str {
122 match self {
123 Self::Jpg => "jpg",
124 Self::Png => "png",
125 }
126 }
127}
128
129impl std::str::FromStr for ImageFormat {
130 type Err = FerrisError;
131
132 fn from_str(value: &str) -> Result<Self> {
133 match value {
134 "jpg" | "jpeg" => Ok(Self::Jpg),
135 "png" => Ok(Self::Png),
136 other => Err(FerrisError::new(
137 ErrorKind::Protocol,
138 format!("unsupported image format: {other}"),
139 )),
140 }
141 }
142}
143
144#[derive(Debug, Clone, Copy, PartialEq, Eq)]
145pub enum ImageSizeLimit {
146 Native,
147 FixedMaxEdge(u32),
148 Adaptive {
149 min_long_edge: u32,
150 min_short_edge: u32,
151 },
152}
153
154impl ImageSizeLimit {
155 pub fn description(self) -> String {
156 match self {
157 Self::Native => "native".to_string(),
158 Self::FixedMaxEdge(edge) => edge.to_string(),
159 Self::Adaptive {
160 min_long_edge,
161 min_short_edge,
162 } => {
163 format!("adaptive min_long_edge={min_long_edge} min_short_edge={min_short_edge}")
164 }
165 }
166 }
167}
168
169#[derive(Debug, Clone)]
170pub struct ObserveRequest {
171 pub output_dir: PathBuf,
172 pub session: Option<String>,
173 pub screen_id: Option<String>,
174 pub format: ImageFormat,
175 pub grid_overlay: bool,
176 pub image_size_limit: ImageSizeLimit,
177}
178
179#[derive(Debug, Clone)]
180pub struct ObserveResult {
181 pub session_dir: PathBuf,
182 pub step: u32,
183 pub coordinate_mode: CoordinateMode,
184 pub image_size_limit: ImageSizeLimit,
185 pub screens: Vec<CapturedScreen>,
186}
187
188#[derive(Debug, Clone)]
189pub struct ActRequest {
190 pub output_dir: PathBuf,
191 pub session: Option<String>,
192 pub default_screen_id: Option<String>,
193 pub input_markdown: String,
194 pub dry_run: bool,
195 pub format: ImageFormat,
196 pub grid_overlay: bool,
197 pub image_size_limit: ImageSizeLimit,
198}
199
200#[derive(Debug, Clone)]
201pub struct ActResult {
202 pub session_dir: PathBuf,
203 pub step: u32,
204 pub action_summary: String,
205 pub wait_after_ms: u64,
206 pub result: String,
207 pub dry_run: bool,
208 pub image_size_limit: ImageSizeLimit,
209 pub screens: Vec<CapturedScreen>,
210}
211
212#[derive(Debug, Clone)]
213pub struct ActionErrorResult {
214 pub session_dir: Option<PathBuf>,
215 pub step: Option<u32>,
216 pub error_type: String,
217 pub reason: String,
218 pub available_screens: Vec<CapturedScreen>,
219}
220
221#[derive(Debug, Clone)]
222pub struct DoctorReport {
223 pub os: String,
224 pub capture: String,
225 pub input: String,
226 pub output_dir: String,
227 pub screens: Vec<ScreenInfo>,
228 pub ffmpeg: String,
229}
230
231#[derive(Debug, Clone, PartialEq)]
232pub struct AgentAction {
233 pub status: ActionStatus,
234 pub kind: Option<ActionKind>,
235 pub wait_after_ms: Option<u64>,
236 pub confidence: Option<f32>,
237 pub reason: Option<String>,
238}
239
240#[derive(Debug, Clone, PartialEq, Eq)]
241pub enum ActionStatus {
242 Action,
243 Done,
244 Fail,
245}
246
247#[derive(Debug, Clone, PartialEq)]
248pub enum ActionKind {
249 Click {
250 screen_id: Option<String>,
251 x: i32,
252 y: i32,
253 button: MouseButton,
254 },
255 DoubleClick {
256 screen_id: Option<String>,
257 x: i32,
258 y: i32,
259 button: MouseButton,
260 },
261 RightClick {
262 screen_id: Option<String>,
263 x: i32,
264 y: i32,
265 },
266 MoveMouse {
267 screen_id: Option<String>,
268 x: i32,
269 y: i32,
270 },
271 Drag {
272 screen_id: Option<String>,
273 from_x: i32,
274 from_y: i32,
275 to_x: i32,
276 to_y: i32,
277 duration_ms: u64,
278 button: MouseButton,
279 },
280 Scroll {
281 screen_id: Option<String>,
282 x: Option<i32>,
283 y: Option<i32>,
284 delta_x: i32,
285 delta_y: i32,
286 },
287 Type {
288 text: String,
289 },
290 PressKey {
291 key: String,
292 },
293 Hotkey {
294 keys: Vec<String>,
295 },
296 Wait {
297 duration_ms: u64,
298 },
299}
300
301impl ActionKind {
302 pub fn screen_id(&self) -> Option<&str> {
303 match self {
304 Self::Click { screen_id, .. }
305 | Self::DoubleClick { screen_id, .. }
306 | Self::RightClick { screen_id, .. }
307 | Self::MoveMouse { screen_id, .. }
308 | Self::Drag { screen_id, .. }
309 | Self::Scroll { screen_id, .. } => screen_id.as_deref(),
310 Self::Type { .. } | Self::PressKey { .. } | Self::Hotkey { .. } | Self::Wait { .. } => {
311 None
312 }
313 }
314 }
315
316 fn accepts_screen_id(&self) -> bool {
317 matches!(
318 self,
319 Self::Click { .. }
320 | Self::DoubleClick { .. }
321 | Self::RightClick { .. }
322 | Self::MoveMouse { .. }
323 | Self::Drag { .. }
324 | Self::Scroll { .. }
325 )
326 }
327
328 fn requires_screen_id(&self) -> bool {
329 match self {
330 Self::Click { .. }
331 | Self::DoubleClick { .. }
332 | Self::RightClick { .. }
333 | Self::MoveMouse { .. }
334 | Self::Drag { .. } => true,
335 Self::Scroll { x, y, .. } => x.is_some() || y.is_some(),
336 Self::Type { .. } | Self::PressKey { .. } | Self::Hotkey { .. } | Self::Wait { .. } => {
337 false
338 }
339 }
340 }
341
342 pub fn with_screen_id(self, resolved: Option<String>) -> Self {
343 match self {
344 Self::Click { x, y, button, .. } => Self::Click {
345 screen_id: resolved,
346 x,
347 y,
348 button,
349 },
350 Self::DoubleClick { x, y, button, .. } => Self::DoubleClick {
351 screen_id: resolved,
352 x,
353 y,
354 button,
355 },
356 Self::RightClick { x, y, .. } => Self::RightClick {
357 screen_id: resolved,
358 x,
359 y,
360 },
361 Self::MoveMouse { x, y, .. } => Self::MoveMouse {
362 screen_id: resolved,
363 x,
364 y,
365 },
366 Self::Drag {
367 from_x,
368 from_y,
369 to_x,
370 to_y,
371 duration_ms,
372 button,
373 ..
374 } => Self::Drag {
375 screen_id: resolved,
376 from_x,
377 from_y,
378 to_x,
379 to_y,
380 duration_ms,
381 button,
382 },
383 Self::Scroll {
384 x,
385 y,
386 delta_x,
387 delta_y,
388 ..
389 } => Self::Scroll {
390 screen_id: resolved,
391 x,
392 y,
393 delta_x,
394 delta_y,
395 },
396 other => other,
397 }
398 }
399}
400
401#[derive(Debug, Clone, Copy, PartialEq, Eq)]
402pub enum MouseButton {
403 Left,
404 Right,
405 Middle,
406}
407
408impl MouseButton {
409 pub fn as_str(self) -> &'static str {
410 match self {
411 Self::Left => "left",
412 Self::Right => "right",
413 Self::Middle => "middle",
414 }
415 }
416}
417
418#[derive(Debug, Clone, PartialEq)]
419pub enum NativeAction {
420 Click {
421 x: i32,
422 y: i32,
423 button: MouseButton,
424 },
425 DoubleClick {
426 x: i32,
427 y: i32,
428 button: MouseButton,
429 },
430 RightClick {
431 x: i32,
432 y: i32,
433 },
434 MoveMouse {
435 x: i32,
436 y: i32,
437 },
438 Drag {
439 from_x: i32,
440 from_y: i32,
441 to_x: i32,
442 to_y: i32,
443 duration_ms: u64,
444 button: MouseButton,
445 },
446 Scroll {
447 x: Option<i32>,
448 y: Option<i32>,
449 delta_x: i32,
450 delta_y: i32,
451 },
452 Type {
453 text: String,
454 },
455 PressKey {
456 key: String,
457 },
458 Hotkey {
459 keys: Vec<String>,
460 },
461 Wait {
462 duration_ms: u64,
463 },
464}
465
466#[derive(Debug, Clone)]
467pub struct InputExecution {
468 pub summary: String,
469}
470
471#[derive(Debug, Clone)]
472pub struct InputCapabilities {
473 pub can_mouse: bool,
474 pub can_keyboard: bool,
475}
476
477pub trait CaptureBackend {
478 fn name(&self) -> &'static str;
479 fn list_screens(&self) -> Result<Vec<ScreenInfo>>;
480 fn capture(
481 &self,
482 target: CaptureTarget,
483 frame_dir: &Path,
484 format: &ImageFormat,
485 grid_overlay: bool,
486 image_size_limit: ImageSizeLimit,
487 ) -> Result<Vec<CapturedScreen>>;
488}
489
490pub trait InputBackend {
491 fn name(&self) -> &'static str;
492 fn capabilities(&self) -> InputCapabilities;
493 fn execute(&self, action: &NativeAction) -> Result<InputExecution>;
494}
495
496#[derive(Debug, Clone)]
497pub struct SessionStore {
498 root: PathBuf,
499}
500
501impl SessionStore {
502 pub fn new(root: impl Into<PathBuf>) -> Self {
503 Self { root: root.into() }
504 }
505
506 pub fn root(&self) -> &Path {
507 &self.root
508 }
509
510 pub fn ensure_root(&self) -> Result<()> {
511 fs::create_dir_all(self.root.join("sessions"))?;
512 let config = self.root.join("config.toml");
513 if !config.exists() {
514 fs::write(
515 &config,
516 "default_output_dir = \".ferrisgrid\"\nstorage_mode = \"all\"\n",
517 )?;
518 }
519 Ok(())
520 }
521
522 pub fn resolve_session(
523 &self,
524 requested: Option<&str>,
525 create_if_missing: bool,
526 ) -> Result<PathBuf> {
527 self.ensure_root()?;
528 if let Some(value) = requested {
529 let path = PathBuf::from(value);
530 let session_dir = if path.exists() || value.contains('/') {
531 path
532 } else {
533 self.root.join("sessions").join(value)
534 };
535 if session_dir.exists() || create_if_missing {
536 self.ensure_session_dirs(&session_dir)?;
537 return Ok(session_dir);
538 }
539 return Err(FerrisError::new(
540 ErrorKind::Storage,
541 format!("session not found: {}", session_dir.display()),
542 ));
543 }
544
545 if let Some(latest) = self.latest_session()? {
546 return Ok(latest);
547 }
548
549 if create_if_missing {
550 return self.create_session();
551 }
552
553 Err(FerrisError::new(
554 ErrorKind::Storage,
555 "no existing session; run ferrisgrid observe first or pass --session",
556 ))
557 }
558
559 pub fn create_session(&self) -> Result<PathBuf> {
560 self.ensure_root()?;
561 let session_id = new_session_id();
562 let session_dir = self.root.join("sessions").join(session_id);
563 self.ensure_session_dirs(&session_dir)?;
564 Ok(session_dir)
565 }
566
567 pub fn latest_session(&self) -> Result<Option<PathBuf>> {
568 let sessions_dir = self.root.join("sessions");
569 if !sessions_dir.exists() {
570 return Ok(None);
571 }
572 let mut entries = Vec::new();
573 for entry in fs::read_dir(sessions_dir)? {
574 let entry = entry?;
575 if entry.file_type()?.is_dir() {
576 entries.push(entry.path());
577 }
578 }
579 entries.sort();
580 Ok(entries.pop())
581 }
582
583 pub fn next_step(&self, session_dir: &Path) -> Result<u32> {
584 let frames = session_dir.join("frames");
585 fs::create_dir_all(&frames)?;
586 let mut max_step = 0;
587 for entry in fs::read_dir(frames)? {
588 let entry = entry?;
589 if !entry.file_type()?.is_dir() {
590 continue;
591 }
592 if let Some(name) = entry.file_name().to_str() {
593 if let Ok(step) = name.parse::<u32>() {
594 max_step = max_step.max(step);
595 }
596 }
597 }
598 Ok(max_step + 1)
599 }
600
601 pub fn frame_dir(&self, session_dir: &Path, step: u32) -> Result<PathBuf> {
602 let dir = session_dir.join("frames").join(format!("{step:06}"));
603 fs::create_dir_all(&dir)?;
604 Ok(dir)
605 }
606
607 pub fn write_manifest_if_missing(&self, session_dir: &Path) -> Result<()> {
608 let manifest = session_dir.join("manifest.md");
609 if !manifest.exists() {
610 fs::write(
611 manifest,
612 format!(
613 "## FerrisGrid Session\n- session_id: {}\n- created_at_unix_ms: {}\n",
614 session_dir
615 .file_name()
616 .and_then(|value| value.to_str())
617 .unwrap_or("unknown"),
618 unix_millis()
619 ),
620 )?;
621 }
622 Ok(())
623 }
624
625 pub fn append_event(&self, session_dir: &Path, line: impl AsRef<str>) -> Result<()> {
626 let mut file = OpenOptions::new()
627 .create(true)
628 .append(true)
629 .open(session_dir.join("events.md"))?;
630 writeln!(file, "- {}", line.as_ref())?;
631 Ok(())
632 }
633
634 pub fn write_action_files(
635 &self,
636 session_dir: &Path,
637 step: u32,
638 request: &str,
639 parsed: &str,
640 result: &str,
641 ) -> Result<()> {
642 let actions = session_dir.join("actions");
643 fs::create_dir_all(&actions)?;
644 fs::write(
645 actions.join(format!("{step:06}.md")),
646 format!(
647 "## FerrisGrid Action\n- step: {step}\n\n### Request\n```text\n{}\n```\n\n### Parsed\n```text\n{}\n```\n\n### Result\n```text\n{}\n```\n",
648 request.trim(),
649 parsed.trim(),
650 result.trim()
651 ),
652 )?;
653 Ok(())
654 }
655
656 fn ensure_session_dirs(&self, session_dir: &Path) -> Result<()> {
657 fs::create_dir_all(session_dir.join("frames"))?;
658 self.write_manifest_if_missing(session_dir)?;
659 Ok(())
660 }
661}
662
663pub fn observe(request: ObserveRequest, capture: &dyn CaptureBackend) -> Result<ObserveResult> {
664 let store = SessionStore::new(request.output_dir);
665 let session_dir = store.resolve_session(request.session.as_deref(), true)?;
666 let step = store.next_step(&session_dir)?;
667 let frame_dir = store.frame_dir(&session_dir, step)?;
668 let target = match request.screen_id {
669 Some(id) => CaptureTarget::Screen(resolve_primary_alias(&id, &capture.list_screens()?)),
670 None => CaptureTarget::All,
671 };
672 let screens = match capture.capture(
673 target,
674 &frame_dir,
675 &request.format,
676 request.grid_overlay,
677 request.image_size_limit,
678 ) {
679 Ok(screens) => screens,
680 Err(error) => {
681 remove_empty_dir(&frame_dir);
682 return Err(error);
683 }
684 };
685 store.append_event(
686 &session_dir,
687 format!(
688 "{} frame_captured step={} screens={}",
689 unix_millis(),
690 step,
691 screens.len()
692 ),
693 )?;
694 Ok(ObserveResult {
695 session_dir,
696 step,
697 coordinate_mode: CoordinateMode::Normalized1000,
698 image_size_limit: request.image_size_limit,
699 screens,
700 })
701}
702
703pub fn act(
704 request: ActRequest,
705 capture: &dyn CaptureBackend,
706 input: &dyn InputBackend,
707) -> std::result::Result<ActResult, ActionErrorResult> {
708 match act_inner(request, capture, input) {
709 Ok(result) => Ok(result),
710 Err((error, context)) => Err(renderable_error(error, context)),
711 }
712}
713
714fn act_inner(
715 request: ActRequest,
716 capture: &dyn CaptureBackend,
717 input: &dyn InputBackend,
718) -> std::result::Result<ActResult, (FerrisError, ErrorContext)> {
719 let store = SessionStore::new(request.output_dir);
720 let session_dir = store
721 .resolve_session(request.session.as_deref(), false)
722 .map_err(|error| (error, ErrorContext::default()))?;
723 let step = store
724 .next_step(&session_dir)
725 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
726 let action = parse_action_block(&request.input_markdown)
727 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
728
729 if action.status == ActionStatus::Done || action.status == ActionStatus::Fail {
730 let result = if action.status == ActionStatus::Done {
731 "done"
732 } else {
733 "fail"
734 };
735 store
736 .write_action_files(
737 &session_dir,
738 step,
739 &request.input_markdown,
740 &format!("{action:?}"),
741 result,
742 )
743 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
744 return Ok(ActResult {
745 session_dir,
746 step,
747 action_summary: result.to_string(),
748 wait_after_ms: 0,
749 result: result.to_string(),
750 dry_run: request.dry_run,
751 image_size_limit: request.image_size_limit,
752 screens: Vec::new(),
753 });
754 }
755
756 let kind = action.kind.clone().ok_or_else(|| {
757 (
758 FerrisError::new(
759 ErrorKind::Protocol,
760 "status action requires an action field",
761 ),
762 ErrorContext::with_session(session_dir.clone()),
763 )
764 })?;
765
766 validate_policy(&kind)
767 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
768 let screens = capture
769 .list_screens()
770 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
771 let requested_screen_id = kind
772 .screen_id()
773 .or(request.default_screen_id.as_deref())
774 .filter(|_| kind.accepts_screen_id());
775 let resolved_screen = if kind.requires_screen_id() || requested_screen_id.is_some() {
776 resolve_action_screen(requested_screen_id, &screens).map_err(|error| {
777 let mut context = ErrorContext::with_session(session_dir.clone());
778 context.available_screens = capture_latest_screens(
779 &store,
780 &session_dir,
781 step,
782 capture,
783 &request.format,
784 request.image_size_limit,
785 )
786 .unwrap_or_default();
787 (error, context)
788 })?
789 } else {
790 None
791 };
792
793 let resolved_kind = kind.with_screen_id(
794 resolved_screen
795 .as_ref()
796 .map(|screen| screen.screen_id.clone()),
797 );
798 let native = to_native_action(&resolved_kind, resolved_screen)
799 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
800
801 let execution = if request.dry_run {
802 InputExecution {
803 summary: "dry_run".to_string(),
804 }
805 } else {
806 input
807 .execute(&native)
808 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?
809 };
810
811 let wait_after_ms = action.wait_after_ms.unwrap_or(0);
812 if wait_after_ms > 0 && !request.dry_run {
813 thread::sleep(Duration::from_millis(wait_after_ms));
814 }
815
816 let frame_dir = store
817 .frame_dir(&session_dir, step)
818 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
819 let target = match resolved_screen {
820 Some(screen) => CaptureTarget::Screen(screen.screen_id.clone()),
821 None => CaptureTarget::All,
822 };
823 let captured = match capture.capture(
824 target,
825 &frame_dir,
826 &request.format,
827 request.grid_overlay,
828 request.image_size_limit,
829 ) {
830 Ok(captured) => captured,
831 Err(error) => {
832 remove_empty_dir(&frame_dir);
833 return Err((error, ErrorContext::with_session(session_dir.clone())));
834 }
835 };
836 let summary = action_summary(&resolved_kind);
837 let parsed_summary = action_summary_with_wait_after(&resolved_kind, wait_after_ms);
838 let result_text = if request.dry_run {
839 "dry_run"
840 } else {
841 "success"
842 };
843 store
844 .write_action_files(
845 &session_dir,
846 step,
847 &request.input_markdown,
848 &parsed_summary,
849 &execution.summary,
850 )
851 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
852 store
853 .append_event(
854 &session_dir,
855 format!(
856 "{} action_executed step={} action={} wait_after_ms={} result={}",
857 unix_millis(),
858 step,
859 summary,
860 wait_after_ms,
861 result_text
862 ),
863 )
864 .map_err(|error| (error, ErrorContext::with_session(session_dir.clone())))?;
865
866 Ok(ActResult {
867 session_dir,
868 step,
869 action_summary: summary,
870 wait_after_ms,
871 result: result_text.to_string(),
872 dry_run: request.dry_run,
873 image_size_limit: request.image_size_limit,
874 screens: captured,
875 })
876}
877
878#[derive(Default)]
879struct ErrorContext {
880 session_dir: Option<PathBuf>,
881 step: Option<u32>,
882 available_screens: Vec<CapturedScreen>,
883}
884
885impl ErrorContext {
886 fn with_session(session_dir: PathBuf) -> Self {
887 Self {
888 session_dir: Some(session_dir),
889 step: None,
890 available_screens: Vec::new(),
891 }
892 }
893}
894
895fn renderable_error(error: FerrisError, context: ErrorContext) -> ActionErrorResult {
896 ActionErrorResult {
897 session_dir: context.session_dir,
898 step: context.step,
899 error_type: error.kind.as_str().to_string(),
900 reason: error.message,
901 available_screens: context.available_screens,
902 }
903}
904
905fn capture_latest_screens(
906 store: &SessionStore,
907 session_dir: &Path,
908 step: u32,
909 capture: &dyn CaptureBackend,
910 format: &ImageFormat,
911 image_size_limit: ImageSizeLimit,
912) -> Result<Vec<CapturedScreen>> {
913 let frame_dir = store.frame_dir(session_dir, step)?;
914 capture.capture(
915 CaptureTarget::All,
916 &frame_dir,
917 format,
918 true,
919 image_size_limit,
920 )
921}
922
923fn remove_empty_dir(path: &Path) {
924 let _ = fs::remove_dir(path);
925}
926
927pub fn render_observation(result: &ObserveResult) -> String {
928 let mut out = String::new();
929 out.push_str("## FerrisGrid Observation\n");
930 out.push_str(&format!("- session: {}\n", result.session_dir.display()));
931 out.push_str(&format!("- step: {}\n", result.step));
932 out.push_str(&format!(
933 "- coordinate_mode: {}\n",
934 result.coordinate_mode.as_str()
935 ));
936 out.push_str(&format!(
937 "- image_size_limit: {}\n",
938 result.image_size_limit.description()
939 ));
940 out.push_str("- coordinate_range: x=0..1000 y=0..1000 origin=top_left scope=screen_local\n");
941 out.push_str(
942 "- action_coordinates: use these x/y values with ferrisgrid act; include screen_id when more than one screen is listed\n",
943 );
944 out.push_str(&format!("- screens: {}\n", result.screens.len()));
945 for screen in &result.screens {
946 out.push_str(&format!(
947 "- screen: {} primary={} image={}x{} native={}x{} origin={},{} coords=x:0..1000,y:0..1000 screenshot={} metadata={}\n",
948 screen.screen.screen_id,
949 screen.screen.is_primary,
950 screen.image_width,
951 screen.image_height,
952 screen.screen.native_width,
953 screen.screen.native_height,
954 screen.screen.origin_x,
955 screen.screen.origin_y,
956 screen.screenshot_path.display(),
957 screen.metadata_path.display()
958 ));
959 out.push_str(&format!(
960 "- map: {} image_x=round(x/1000*{}) image_y=round(y/1000*{}) native_x={}+round(x/1000*{}) native_y={}+round(y/1000*{})\n",
961 screen.screen.screen_id,
962 screen.image_width.saturating_sub(1),
963 screen.image_height.saturating_sub(1),
964 screen.screen.origin_x,
965 screen.screen.native_width,
966 screen.screen.origin_y,
967 screen.screen.native_height
968 ));
969 }
970 out
971}
972
973pub fn render_action_result(result: &ActResult) -> String {
974 let mut out = String::new();
975 out.push_str("## FerrisGrid Action Result\n");
976 out.push_str(&format!("- session: {}\n", result.session_dir.display()));
977 out.push_str(&format!("- step: {}\n", result.step));
978 out.push_str(&format!("- action: {}\n", result.action_summary));
979 if result.wait_after_ms > 0 {
980 out.push_str(&format!("- wait_after_ms: {}\n", result.wait_after_ms));
981 }
982 out.push_str(&format!("- result: {}\n", result.result));
983 out.push_str(&format!("- screens: {}\n", result.screens.len()));
984 if result.screens.is_empty() {
985 out.push_str("- note: no post-action screenshot captured for terminal status\n");
986 return out;
987 }
988 out.push_str("- coordinate_mode: normalized-1000\n");
989 out.push_str(&format!(
990 "- image_size_limit: {}\n",
991 result.image_size_limit.description()
992 ));
993 out.push_str("- coordinate_range: x=0..1000 y=0..1000 origin=top_left scope=screen_local\n");
994 for screen in &result.screens {
995 out.push_str(&format!(
996 "- screen: {} primary={} image={}x{} native={}x{} origin={},{} coords=x:0..1000,y:0..1000 screenshot={} metadata={}\n",
997 screen.screen.screen_id,
998 screen.screen.is_primary,
999 screen.image_width,
1000 screen.image_height,
1001 screen.screen.native_width,
1002 screen.screen.native_height,
1003 screen.screen.origin_x,
1004 screen.screen.origin_y,
1005 screen.screenshot_path.display(),
1006 screen.metadata_path.display()
1007 ));
1008 out.push_str(&format!(
1009 "- map: {} image_x=round(x/1000*{}) image_y=round(y/1000*{}) native_x={}+round(x/1000*{}) native_y={}+round(y/1000*{})\n",
1010 screen.screen.screen_id,
1011 screen.image_width.saturating_sub(1),
1012 screen.image_height.saturating_sub(1),
1013 screen.screen.origin_x,
1014 screen.screen.native_width,
1015 screen.screen.origin_y,
1016 screen.screen.native_height
1017 ));
1018 }
1019 out
1020}
1021
1022pub fn render_action_error(error: &ActionErrorResult) -> String {
1023 let mut out = String::new();
1024 out.push_str("## FerrisGrid Action Error\n");
1025 out.push_str(&format!("- type: {}\n", error.error_type));
1026 out.push_str("- result: rejected\n");
1027 out.push_str(&format!("- reason: {}\n", error.reason));
1028 if let Some(session) = &error.session_dir {
1029 out.push_str(&format!("- session: {}\n", session.display()));
1030 }
1031 for screen in &error.available_screens {
1032 out.push_str(&format!(
1033 "- available_screen: {} coords=x:0..1000,y:0..1000 screenshot={} metadata={}\n",
1034 screen.screen.screen_id,
1035 screen.screenshot_path.display(),
1036 screen.metadata_path.display()
1037 ));
1038 }
1039 out
1040}
1041
1042pub fn render_doctor(report: &DoctorReport) -> String {
1043 let mut out = String::new();
1044 out.push_str("## FerrisGrid Doctor\n");
1045 out.push_str(&format!("- os: {}\n", report.os));
1046 out.push_str(&format!("- capture: {}\n", report.capture));
1047 out.push_str(&format!("- input: {}\n", report.input));
1048 out.push_str(&format!("- output_directory: {}\n", report.output_dir));
1049 out.push_str(&format!("- screens: {}\n", report.screens.len()));
1050 for screen in &report.screens {
1051 out.push_str(&format!(
1052 "- screen: {} primary={} origin={},{} native={}x{} scale={}\n",
1053 screen.screen_id,
1054 screen.is_primary,
1055 screen.origin_x,
1056 screen.origin_y,
1057 screen.native_width,
1058 screen.native_height,
1059 screen.scale_factor
1060 ));
1061 }
1062 out.push_str(&format!("- ffmpeg: {}\n", report.ffmpeg));
1063 out
1064}
1065
1066pub fn parse_action_block(markdown: &str) -> Result<AgentAction> {
1067 let trimmed = markdown.trim();
1068 if trimmed.is_empty() {
1069 return Err(FerrisError::new(ErrorKind::Protocol, "empty action input"));
1070 }
1071 if trimmed.starts_with('{') || trimmed.starts_with('[') {
1072 return Err(FerrisError::new(
1073 ErrorKind::Protocol,
1074 "JSON action input is not supported; use compact Markdown",
1075 ));
1076 }
1077
1078 let mut fields = BTreeMap::new();
1079 for line in trimmed.lines() {
1080 let line = line.trim();
1081 if line.is_empty() || line.starts_with('#') {
1082 continue;
1083 }
1084 let Some((key, value)) = line.split_once(':') else {
1085 return Err(FerrisError::new(
1086 ErrorKind::Protocol,
1087 format!("invalid action line, expected key: value: {line}"),
1088 ));
1089 };
1090 fields.insert(key.trim().to_string(), value.trim().to_string());
1091 }
1092 if fields.is_empty() {
1093 return Err(FerrisError::new(
1094 ErrorKind::Protocol,
1095 "action input must contain compact Markdown key/value lines",
1096 ));
1097 }
1098
1099 let status = match fields.get("status").map(String::as_str).unwrap_or("action") {
1100 "action" => ActionStatus::Action,
1101 "done" => ActionStatus::Done,
1102 "fail" => ActionStatus::Fail,
1103 other => {
1104 return Err(FerrisError::new(
1105 ErrorKind::Protocol,
1106 format!("unsupported status: {other}"),
1107 ));
1108 }
1109 };
1110 let confidence = match fields.get("confidence") {
1111 Some(value) => Some(parse_f32(value, "confidence")?),
1112 None => None,
1113 };
1114 let reason = fields.get("reason").cloned();
1115 let wait_after_ms = match fields.get("wait_after_ms") {
1116 Some(value) => Some(parse_u64(value, "wait_after_ms")?),
1117 None => None,
1118 };
1119 validate_wait_after(wait_after_ms)?;
1120 let kind = if status == ActionStatus::Action {
1121 Some(parse_action_kind(&fields)?)
1122 } else {
1123 None
1124 };
1125
1126 Ok(AgentAction {
1127 status,
1128 kind,
1129 wait_after_ms,
1130 confidence,
1131 reason,
1132 })
1133}
1134
1135fn validate_wait_after(wait_after_ms: Option<u64>) -> Result<()> {
1136 if let Some(wait_after_ms) = wait_after_ms {
1137 if wait_after_ms > 30_000 {
1138 return Err(FerrisError::new(
1139 ErrorKind::Protocol,
1140 "wait_after_ms exceeds 30000 ms",
1141 ));
1142 }
1143 }
1144 Ok(())
1145}
1146
1147fn parse_action_kind(fields: &BTreeMap<String, String>) -> Result<ActionKind> {
1148 let action = required(fields, "action")?;
1149 let screen_id = fields.get("screen_id").cloned();
1150 match action.as_str() {
1151 "click" => Ok(ActionKind::Click {
1152 screen_id,
1153 x: parse_i32_required(fields, "x")?,
1154 y: parse_i32_required(fields, "y")?,
1155 button: parse_button(fields.get("button").map(String::as_str).unwrap_or("left"))?,
1156 }),
1157 "double_click" => Ok(ActionKind::DoubleClick {
1158 screen_id,
1159 x: parse_i32_required(fields, "x")?,
1160 y: parse_i32_required(fields, "y")?,
1161 button: parse_button(fields.get("button").map(String::as_str).unwrap_or("left"))?,
1162 }),
1163 "right_click" => Ok(ActionKind::RightClick {
1164 screen_id,
1165 x: parse_i32_required(fields, "x")?,
1166 y: parse_i32_required(fields, "y")?,
1167 }),
1168 "move_mouse" => Ok(ActionKind::MoveMouse {
1169 screen_id,
1170 x: parse_i32_required(fields, "x")?,
1171 y: parse_i32_required(fields, "y")?,
1172 }),
1173 "drag" => Ok(ActionKind::Drag {
1174 screen_id,
1175 from_x: parse_i32_required(fields, "from_x")?,
1176 from_y: parse_i32_required(fields, "from_y")?,
1177 to_x: parse_i32_required(fields, "to_x")?,
1178 to_y: parse_i32_required(fields, "to_y")?,
1179 duration_ms: parse_u64(
1180 fields
1181 .get("duration_ms")
1182 .map(String::as_str)
1183 .unwrap_or("450"),
1184 "duration_ms",
1185 )?,
1186 button: parse_button(fields.get("button").map(String::as_str).unwrap_or("left"))?,
1187 }),
1188 "scroll" => Ok(ActionKind::Scroll {
1189 screen_id,
1190 x: parse_i32_optional(fields.get("x").map(String::as_str), "x")?,
1191 y: parse_i32_optional(fields.get("y").map(String::as_str), "y")?,
1192 delta_x: parse_i32_optional(fields.get("delta_x").map(String::as_str), "delta_x")?
1193 .unwrap_or(0),
1194 delta_y: parse_i32_required(fields, "delta_y")?,
1195 }),
1196 "type" => Ok(ActionKind::Type {
1197 text: required(fields, "text")?,
1198 }),
1199 "press_key" => Ok(ActionKind::PressKey {
1200 key: required(fields, "key")?,
1201 }),
1202 "hotkey" => Ok(ActionKind::Hotkey {
1203 keys: required(fields, "keys")?
1204 .split('+')
1205 .map(str::trim)
1206 .filter(|value| !value.is_empty())
1207 .map(ToOwned::to_owned)
1208 .collect(),
1209 }),
1210 "wait" => Ok(ActionKind::Wait {
1211 duration_ms: parse_u64(&required(fields, "duration_ms")?, "duration_ms")?,
1212 }),
1213 other => Err(FerrisError::new(
1214 ErrorKind::Protocol,
1215 format!("unknown action: {other}"),
1216 )),
1217 }
1218}
1219
1220fn validate_policy(action: &ActionKind) -> Result<()> {
1221 match action {
1222 ActionKind::Click { x, y, .. }
1223 | ActionKind::DoubleClick { x, y, .. }
1224 | ActionKind::RightClick { x, y, .. }
1225 | ActionKind::MoveMouse { x, y, .. } => validate_agent_point(*x, *y),
1226 ActionKind::Drag {
1227 from_x,
1228 from_y,
1229 to_x,
1230 to_y,
1231 duration_ms,
1232 ..
1233 } => {
1234 validate_agent_point(*from_x, *from_y)?;
1235 validate_agent_point(*to_x, *to_y)?;
1236 if *duration_ms > 5_000 {
1237 return Err(FerrisError::new(
1238 ErrorKind::Protocol,
1239 "drag duration exceeds 5000 ms",
1240 ));
1241 }
1242 Ok(())
1243 }
1244 ActionKind::Scroll {
1245 x,
1246 y,
1247 delta_x,
1248 delta_y,
1249 ..
1250 } => {
1251 match (x, y) {
1252 (Some(x), Some(y)) => validate_agent_point(*x, *y)?,
1253 (None, None) => {}
1254 _ => {
1255 return Err(FerrisError::new(
1256 ErrorKind::Protocol,
1257 "scroll x and y must be supplied together",
1258 ));
1259 }
1260 }
1261 if delta_x.abs() > 2_000 || delta_y.abs() > 2_000 {
1262 return Err(FerrisError::new(
1263 ErrorKind::Protocol,
1264 "scroll delta exceeds 2000",
1265 ));
1266 }
1267 Ok(())
1268 }
1269 ActionKind::Type { text } => {
1270 if text.chars().count() > 500 {
1271 return Err(FerrisError::new(
1272 ErrorKind::Protocol,
1273 "typed text exceeds 500 characters",
1274 ));
1275 }
1276 Ok(())
1277 }
1278 ActionKind::Hotkey { keys } => {
1279 if keys.is_empty() || keys.len() > 4 {
1280 return Err(FerrisError::new(
1281 ErrorKind::Protocol,
1282 "hotkey must contain 1 to 4 keys",
1283 ));
1284 }
1285 Ok(())
1286 }
1287 ActionKind::PressKey { key } => {
1288 if key.trim().is_empty() {
1289 return Err(FerrisError::new(ErrorKind::Protocol, "key is required"));
1290 }
1291 Ok(())
1292 }
1293 ActionKind::Wait { duration_ms } => {
1294 if *duration_ms > 30_000 {
1295 return Err(FerrisError::new(
1296 ErrorKind::Protocol,
1297 "wait duration exceeds 30000 ms",
1298 ));
1299 }
1300 Ok(())
1301 }
1302 }
1303}
1304
1305fn validate_agent_point(x: i32, y: i32) -> Result<()> {
1306 if !(0..=1000).contains(&x) || !(0..=1000).contains(&y) {
1307 return Err(FerrisError::new(
1308 ErrorKind::Coordinate,
1309 "coordinates must be within 0..1000",
1310 ));
1311 }
1312 Ok(())
1313}
1314
1315fn resolve_action_screen<'a>(
1316 screen_id: Option<&str>,
1317 screens: &'a [ScreenInfo],
1318) -> Result<Option<&'a ScreenInfo>> {
1319 if screens.is_empty() {
1320 return Err(FerrisError::new(ErrorKind::Capture, "no screens available"));
1321 }
1322 if let Some(id) = screen_id {
1323 let id = resolve_primary_alias(id, screens);
1324 return screens
1325 .iter()
1326 .find(|screen| screen.screen_id == id)
1327 .map(Some)
1328 .ok_or_else(|| {
1329 FerrisError::new(ErrorKind::Coordinate, format!("unknown screen_id: {id}"))
1330 });
1331 }
1332 if screens.len() == 1 {
1333 return Ok(screens.first());
1334 }
1335 Err(FerrisError::new(
1336 ErrorKind::Coordinate,
1337 "screen_id is required because multiple screens are active",
1338 ))
1339}
1340
1341fn resolve_primary_alias(id: &str, screens: &[ScreenInfo]) -> String {
1342 if id == "primary" {
1343 if let Some(primary) = screens.iter().find(|screen| screen.is_primary) {
1344 return primary.screen_id.clone();
1345 }
1346 }
1347 id.to_string()
1348}
1349
1350fn to_native_action(action: &ActionKind, screen: Option<&ScreenInfo>) -> Result<NativeAction> {
1351 match action {
1352 ActionKind::Click { x, y, button, .. } => {
1353 let (x, y) = map_point(
1354 *x,
1355 *y,
1356 screen.ok_or_else(|| {
1357 FerrisError::new(ErrorKind::Coordinate, "screen_id required for click")
1358 })?,
1359 )?;
1360 Ok(NativeAction::Click {
1361 x,
1362 y,
1363 button: *button,
1364 })
1365 }
1366 ActionKind::DoubleClick { x, y, button, .. } => {
1367 let (x, y) = map_point(
1368 *x,
1369 *y,
1370 screen.ok_or_else(|| {
1371 FerrisError::new(ErrorKind::Coordinate, "screen_id required for double_click")
1372 })?,
1373 )?;
1374 Ok(NativeAction::DoubleClick {
1375 x,
1376 y,
1377 button: *button,
1378 })
1379 }
1380 ActionKind::RightClick { x, y, .. } => {
1381 let (x, y) = map_point(
1382 *x,
1383 *y,
1384 screen.ok_or_else(|| {
1385 FerrisError::new(ErrorKind::Coordinate, "screen_id required for right_click")
1386 })?,
1387 )?;
1388 Ok(NativeAction::RightClick { x, y })
1389 }
1390 ActionKind::MoveMouse { x, y, .. } => {
1391 let (x, y) = map_point(
1392 *x,
1393 *y,
1394 screen.ok_or_else(|| {
1395 FerrisError::new(ErrorKind::Coordinate, "screen_id required for move_mouse")
1396 })?,
1397 )?;
1398 Ok(NativeAction::MoveMouse { x, y })
1399 }
1400 ActionKind::Drag {
1401 from_x,
1402 from_y,
1403 to_x,
1404 to_y,
1405 duration_ms,
1406 button,
1407 ..
1408 } => {
1409 let screen = screen.ok_or_else(|| {
1410 FerrisError::new(ErrorKind::Coordinate, "screen_id required for drag")
1411 })?;
1412 let (from_x, from_y) = map_point(*from_x, *from_y, screen)?;
1413 let (to_x, to_y) = map_point(*to_x, *to_y, screen)?;
1414 Ok(NativeAction::Drag {
1415 from_x,
1416 from_y,
1417 to_x,
1418 to_y,
1419 duration_ms: *duration_ms,
1420 button: *button,
1421 })
1422 }
1423 ActionKind::Scroll {
1424 x,
1425 y,
1426 delta_x,
1427 delta_y,
1428 ..
1429 } => {
1430 let point = match (x, y, screen) {
1431 (Some(x), Some(y), Some(screen)) => Some(map_point(*x, *y, screen)?),
1432 _ => None,
1433 };
1434 Ok(NativeAction::Scroll {
1435 x: point.map(|value| value.0),
1436 y: point.map(|value| value.1),
1437 delta_x: *delta_x,
1438 delta_y: *delta_y,
1439 })
1440 }
1441 ActionKind::Type { text } => Ok(NativeAction::Type { text: text.clone() }),
1442 ActionKind::PressKey { key } => Ok(NativeAction::PressKey { key: key.clone() }),
1443 ActionKind::Hotkey { keys } => Ok(NativeAction::Hotkey { keys: keys.clone() }),
1444 ActionKind::Wait { duration_ms } => Ok(NativeAction::Wait {
1445 duration_ms: *duration_ms,
1446 }),
1447 }
1448}
1449
1450pub fn map_point(agent_x: i32, agent_y: i32, screen: &ScreenInfo) -> Result<(i32, i32)> {
1451 validate_agent_point(agent_x, agent_y)?;
1452 let native_x =
1453 screen.origin_x + ((agent_x as f64 / 1000.0) * screen.native_width as f64).round() as i32;
1454 let native_y =
1455 screen.origin_y + ((agent_y as f64 / 1000.0) * screen.native_height as f64).round() as i32;
1456 let max_x = screen.origin_x + screen.native_width.saturating_sub(1) as i32;
1457 let max_y = screen.origin_y + screen.native_height.saturating_sub(1) as i32;
1458 Ok((
1459 native_x.clamp(screen.origin_x, max_x),
1460 native_y.clamp(screen.origin_y, max_y),
1461 ))
1462}
1463
1464fn action_summary(action: &ActionKind) -> String {
1465 match action {
1466 ActionKind::Click {
1467 screen_id,
1468 x,
1469 y,
1470 button,
1471 } => format!(
1472 "click screen_id={} x={} y={} button={}",
1473 screen_id.as_deref().unwrap_or(""),
1474 x,
1475 y,
1476 button.as_str()
1477 ),
1478 ActionKind::DoubleClick {
1479 screen_id,
1480 x,
1481 y,
1482 button,
1483 } => format!(
1484 "double_click screen_id={} x={} y={} button={}",
1485 screen_id.as_deref().unwrap_or(""),
1486 x,
1487 y,
1488 button.as_str()
1489 ),
1490 ActionKind::RightClick { screen_id, x, y } => format!(
1491 "right_click screen_id={} x={} y={}",
1492 screen_id.as_deref().unwrap_or(""),
1493 x,
1494 y
1495 ),
1496 ActionKind::MoveMouse { screen_id, x, y } => format!(
1497 "move_mouse screen_id={} x={} y={}",
1498 screen_id.as_deref().unwrap_or(""),
1499 x,
1500 y
1501 ),
1502 ActionKind::Drag {
1503 screen_id,
1504 from_x,
1505 from_y,
1506 to_x,
1507 to_y,
1508 duration_ms,
1509 button,
1510 } => format!(
1511 "drag screen_id={} from_x={} from_y={} to_x={} to_y={} duration_ms={} button={}",
1512 screen_id.as_deref().unwrap_or(""),
1513 from_x,
1514 from_y,
1515 to_x,
1516 to_y,
1517 duration_ms,
1518 button.as_str()
1519 ),
1520 ActionKind::Scroll {
1521 screen_id,
1522 x,
1523 y,
1524 delta_x,
1525 delta_y,
1526 } => format!(
1527 "scroll screen_id={} x={} y={} delta_x={} delta_y={}",
1528 screen_id.as_deref().unwrap_or(""),
1529 x.map(|v| v.to_string()).unwrap_or_default(),
1530 y.map(|v| v.to_string()).unwrap_or_default(),
1531 delta_x,
1532 delta_y
1533 ),
1534 ActionKind::Type { .. } => "type text=<redacted>".to_string(),
1535 ActionKind::PressKey { key } => format!("press_key key={key}"),
1536 ActionKind::Hotkey { keys } => format!("hotkey keys={}", keys.join("+")),
1537 ActionKind::Wait { duration_ms } => format!("wait duration_ms={duration_ms}"),
1538 }
1539}
1540
1541fn action_summary_with_wait_after(action: &ActionKind, wait_after_ms: u64) -> String {
1542 let summary = action_summary(action);
1543 if wait_after_ms == 0 {
1544 summary
1545 } else {
1546 format!("{summary}\nwait_after_ms={wait_after_ms}")
1547 }
1548}
1549
1550fn required(fields: &BTreeMap<String, String>, key: &str) -> Result<String> {
1551 fields
1552 .get(key)
1553 .cloned()
1554 .filter(|value| !value.is_empty())
1555 .ok_or_else(|| FerrisError::new(ErrorKind::Protocol, format!("{key} is required")))
1556}
1557
1558fn parse_i32_required(fields: &BTreeMap<String, String>, key: &str) -> Result<i32> {
1559 parse_i32(&required(fields, key)?, key)
1560}
1561
1562fn parse_i32_optional(value: Option<&str>, key: &str) -> Result<Option<i32>> {
1563 value.map(|value| parse_i32(value, key)).transpose()
1564}
1565
1566fn parse_i32(value: &str, key: &str) -> Result<i32> {
1567 value
1568 .parse::<i32>()
1569 .map_err(|_| FerrisError::new(ErrorKind::Protocol, format!("{key} must be an integer")))
1570}
1571
1572fn parse_u64(value: &str, key: &str) -> Result<u64> {
1573 value
1574 .parse::<u64>()
1575 .map_err(|_| FerrisError::new(ErrorKind::Protocol, format!("{key} must be an integer")))
1576}
1577
1578fn parse_f32(value: &str, key: &str) -> Result<f32> {
1579 value
1580 .parse::<f32>()
1581 .map_err(|_| FerrisError::new(ErrorKind::Protocol, format!("{key} must be a number")))
1582}
1583
1584fn parse_button(value: &str) -> Result<MouseButton> {
1585 match value {
1586 "left" => Ok(MouseButton::Left),
1587 "right" => Ok(MouseButton::Right),
1588 "middle" => Ok(MouseButton::Middle),
1589 other => Err(FerrisError::new(
1590 ErrorKind::Protocol,
1591 format!("unsupported mouse button: {other}"),
1592 )),
1593 }
1594}
1595
1596fn new_session_id() -> String {
1597 format!("{}-{}", unix_millis(), std::process::id())
1598}
1599
1600fn unix_millis() -> u128 {
1601 SystemTime::now()
1602 .duration_since(UNIX_EPOCH)
1603 .unwrap_or_default()
1604 .as_millis()
1605}
1606
1607#[cfg(test)]
1608mod tests {
1609 use super::*;
1610 use std::sync::atomic::{AtomicU64, Ordering};
1611 use std::time::{SystemTime, UNIX_EPOCH};
1612
1613 static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
1614
1615 struct TestCaptureBackend;
1616
1617 impl CaptureBackend for TestCaptureBackend {
1618 fn name(&self) -> &'static str {
1619 "test"
1620 }
1621
1622 fn list_screens(&self) -> Result<Vec<ScreenInfo>> {
1623 Ok(vec![
1624 screen(),
1625 ScreenInfo {
1626 screen_id: "screen-2".to_string(),
1627 name: "Test 2".to_string(),
1628 is_primary: false,
1629 origin_x: 3024,
1630 origin_y: 0,
1631 native_width: 2560,
1632 native_height: 1440,
1633 scale_factor: 1.0,
1634 },
1635 ])
1636 }
1637
1638 fn capture(
1639 &self,
1640 target: CaptureTarget,
1641 frame_dir: &Path,
1642 format: &ImageFormat,
1643 _grid_overlay: bool,
1644 _image_size_limit: ImageSizeLimit,
1645 ) -> Result<Vec<CapturedScreen>> {
1646 let screens = self.list_screens()?;
1647 let selected: Vec<ScreenInfo> = match target {
1648 CaptureTarget::All => screens,
1649 CaptureTarget::Screen(id) => screens
1650 .into_iter()
1651 .filter(|screen| screen.screen_id == id)
1652 .collect(),
1653 };
1654 Ok(selected
1655 .into_iter()
1656 .map(|screen| CapturedScreen {
1657 screenshot_path: frame_dir.join(format!(
1658 "{}.{}",
1659 screen.screen_id,
1660 format.extension()
1661 )),
1662 metadata_path: frame_dir.join(format!("{}.meta.md", screen.screen_id)),
1663 image_width: 800,
1664 image_height: 520,
1665 screen,
1666 })
1667 .collect())
1668 }
1669 }
1670
1671 struct TestInputBackend;
1672
1673 impl InputBackend for TestInputBackend {
1674 fn name(&self) -> &'static str {
1675 "test"
1676 }
1677
1678 fn capabilities(&self) -> InputCapabilities {
1679 InputCapabilities {
1680 can_mouse: true,
1681 can_keyboard: true,
1682 }
1683 }
1684
1685 fn execute(&self, action: &NativeAction) -> Result<InputExecution> {
1686 Ok(InputExecution {
1687 summary: format!("{action:?}"),
1688 })
1689 }
1690 }
1691
1692 fn screen() -> ScreenInfo {
1693 ScreenInfo {
1694 screen_id: "screen-1".to_string(),
1695 name: "Test".to_string(),
1696 is_primary: true,
1697 origin_x: 0,
1698 origin_y: 0,
1699 native_width: 3024,
1700 native_height: 1964,
1701 scale_factor: 2.0,
1702 }
1703 }
1704
1705 fn temp_output_dir(name: &str) -> PathBuf {
1706 let nonce = SystemTime::now()
1707 .duration_since(UNIX_EPOCH)
1708 .unwrap()
1709 .as_nanos();
1710 let counter = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
1711 std::env::temp_dir().join(format!(
1712 "ferrisgrid-core-test-{name}-{}-{nonce}-{counter}",
1713 std::process::id()
1714 ))
1715 }
1716
1717 fn create_test_session(output_dir: &Path) {
1718 let store = SessionStore::new(output_dir);
1719 let session = store.create_session().unwrap();
1720 let frame_dir = store.frame_dir(&session, 1).unwrap();
1721 fs::write(frame_dir.join("screen-1.jpg"), "test").unwrap();
1722 }
1723
1724 fn test_act_request(output_dir: PathBuf, input_markdown: &str) -> ActRequest {
1725 ActRequest {
1726 output_dir,
1727 session: None,
1728 default_screen_id: None,
1729 input_markdown: input_markdown.to_string(),
1730 dry_run: true,
1731 format: ImageFormat::Jpg,
1732 grid_overlay: false,
1733 image_size_limit: ImageSizeLimit::FixedMaxEdge(800),
1734 }
1735 }
1736
1737 #[test]
1738 fn maps_normalized_center_to_native_center() {
1739 assert_eq!(map_point(500, 500, &screen()).unwrap(), (1512, 982));
1740 }
1741
1742 #[test]
1743 fn rejects_out_of_bounds_coordinates() {
1744 assert!(map_point(1001, 500, &screen()).is_err());
1745 }
1746
1747 #[test]
1748 fn parses_click_action_block() {
1749 let action = parse_action_block(
1750 "status: action\naction: click\nscreen_id: screen-1\nx: 742\ny: 611\nbutton: left\n",
1751 )
1752 .unwrap();
1753 assert_eq!(action.status, ActionStatus::Action);
1754 assert!(matches!(action.kind, Some(ActionKind::Click { .. })));
1755 assert_eq!(action.wait_after_ms, None);
1756 }
1757
1758 #[test]
1759 fn parses_wait_after_ms() {
1760 let action = parse_action_block(
1761 "status: action\naction: click\nscreen_id: screen-1\nx: 742\ny: 611\nbutton: left\nwait_after_ms: 750\n",
1762 )
1763 .unwrap();
1764 assert_eq!(action.wait_after_ms, Some(750));
1765 }
1766
1767 #[test]
1768 fn rejects_excessive_wait_after_ms() {
1769 let error = parse_action_block(
1770 "status: action\naction: click\nscreen_id: screen-1\nx: 742\ny: 611\nwait_after_ms: 30001\n",
1771 )
1772 .unwrap_err();
1773 assert_eq!(error.kind, ErrorKind::Protocol);
1774 assert!(error.message.contains("wait_after_ms"));
1775 }
1776
1777 #[test]
1778 fn rejects_json_action_input() {
1779 assert!(parse_action_block("{\"action\":\"click\"}").is_err());
1780 }
1781
1782 #[test]
1783 fn rejects_missing_screen_in_multi_screen_context() {
1784 let screens = vec![
1785 screen(),
1786 ScreenInfo {
1787 screen_id: "screen-2".to_string(),
1788 is_primary: false,
1789 ..screen()
1790 },
1791 ];
1792 assert!(resolve_action_screen(None, &screens).is_err());
1793 }
1794
1795 #[test]
1796 fn multi_screen_wait_does_not_require_screen_id() {
1797 let output_dir = temp_output_dir("wait-no-screen");
1798 create_test_session(&output_dir);
1799 let result = act(
1800 test_act_request(output_dir.clone(), "action: wait\nduration_ms: 1\n"),
1801 &TestCaptureBackend,
1802 &TestInputBackend,
1803 )
1804 .unwrap();
1805
1806 assert_eq!(result.action_summary, "wait duration_ms=1");
1807 assert_eq!(result.screens.len(), 2);
1808 let _ = fs::remove_dir_all(output_dir);
1809 }
1810
1811 #[test]
1812 fn default_screen_id_disambiguates_pointer_action() {
1813 let output_dir = temp_output_dir("default-screen");
1814 create_test_session(&output_dir);
1815 let mut request = test_act_request(output_dir.clone(), "action: click\nx: 500\ny: 500\n");
1816 request.default_screen_id = Some("screen-1".to_string());
1817
1818 let result = act(request, &TestCaptureBackend, &TestInputBackend).unwrap();
1819
1820 assert!(result.action_summary.contains("click screen_id=screen-1"));
1821 assert_eq!(result.screens.len(), 1);
1822 assert_eq!(result.screens[0].screen.screen_id, "screen-1");
1823 let _ = fs::remove_dir_all(output_dir);
1824 }
1825
1826 #[test]
1827 fn rejects_partial_scroll_point() {
1828 let action = parse_action_block("action: scroll\nx: 500\ndelta_y: -120\n").unwrap();
1829
1830 let error = validate_policy(&action.kind.unwrap()).unwrap_err();
1831
1832 assert_eq!(error.kind, ErrorKind::Protocol);
1833 assert!(error.message.contains("x and y"));
1834 }
1835
1836 #[test]
1837 fn terminal_action_result_reports_no_screens() {
1838 let rendered = render_action_result(&ActResult {
1839 session_dir: PathBuf::from(".ferrisgrid/sessions/test"),
1840 step: 2,
1841 action_summary: "done".to_string(),
1842 wait_after_ms: 0,
1843 result: "done".to_string(),
1844 dry_run: false,
1845 image_size_limit: ImageSizeLimit::FixedMaxEdge(800),
1846 screens: Vec::new(),
1847 });
1848
1849 assert!(rendered.contains("- screens: 0"));
1850 assert!(rendered.contains("no post-action screenshot"));
1851 assert!(!rendered.contains("- coordinate_mode:"));
1852 }
1853}