#[cfg(test)]
use crate::session::PROTOCOL_VERSION;
use crate::session::unix_ms;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::{
error::Error,
fmt::{self, Display, Formatter},
fs::{self, File, OpenOptions},
io::{self, BufRead, BufReader, Read, Write},
net::TcpStream,
path::{Path, PathBuf},
time::{Duration, Instant},
};
#[derive(Clone, Debug)]
pub struct AgentClientConfig {
pub protocol_file: PathBuf,
pub timeout: Duration,
pub transcript_file: Option<PathBuf>,
pub ocr: OcrOptions,
}
impl Default for AgentClientConfig {
fn default() -> Self {
Self {
protocol_file: std::env::var_os("BEVY_FEEDBACK_PROTOCOL")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("target/agent-feedback/agent-feedback.json")),
timeout: Duration::from_secs(10),
transcript_file: std::env::var_os("BEVY_FEEDBACK_TRANSCRIPT").map(PathBuf::from),
ocr: OcrOptions::default(),
}
}
}
#[derive(Clone, Debug)]
pub struct OcrOptions {
pub tesseract: PathBuf,
pub language: String,
pub timeout: Duration,
}
impl Default for OcrOptions {
fn default() -> Self {
Self {
tesseract: std::env::var_os("BEVY_FEEDBACK_TESSERACT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from("tesseract")),
language: "eng".to_string(),
timeout: Duration::from_secs(5),
}
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum CaptureCompletion {
ScreenshotCaptured,
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum CaptureWindowMode {
Windowed,
BorderlessFullscreen,
Fullscreen,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct CaptureWindowInfo {
pub logical_width: f32,
pub logical_height: f32,
pub physical_width: u32,
pub physical_height: u32,
pub scale_factor: f32,
pub cursor_position: Option<[f32; 2]>,
pub focused: bool,
pub visible: bool,
pub mode: CaptureWindowMode,
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct Capture {
pub sequence: u64,
pub path: PathBuf,
pub label: Option<String>,
pub requested_frame: u64,
pub completed_frame: u64,
pub image_width: u32,
pub image_height: u32,
pub window_at_request: CaptureWindowInfo,
pub window_at_completion: Option<CaptureWindowInfo>,
pub completion: CaptureCompletion,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct Region {
pub x: u32,
pub y: u32,
pub width: u32,
pub height: u32,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AgentCapabilities {
pub max_wait_frames: u16,
pub max_abort_predicates: usize,
pub deterministic_time: bool,
pub max_time_advance_steps: u16,
pub max_time_advance: Duration,
}
pub struct AgentClient {
stream: TcpStream,
reader: BufReader<TcpStream>,
next_id: u64,
timeout: Duration,
transcript: Option<File>,
last_capture: Option<Capture>,
ocr: OcrOptions,
capabilities: AgentCapabilities,
last_observation: Option<ObservedPredicate>,
}
#[derive(Debug)]
pub enum ClientError {
Io(String),
Json(String),
Protocol(String),
Command {
code: String,
message: String,
context: Option<Value>,
},
OcrUnavailable(String),
Ocr(String),
Assertion(String),
}
impl Display for ClientError {
fn fmt(&self, formatter: &mut Formatter<'_>) -> fmt::Result {
match self {
Self::Io(message)
| Self::Json(message)
| Self::Protocol(message)
| Self::OcrUnavailable(message)
| Self::Ocr(message)
| Self::Assertion(message) => formatter.write_str(message),
Self::Command {
code,
message,
context,
} => {
write!(formatter, "command failed [{code}]: {message}")?;
if let Some(context) = context {
write!(formatter, "; context={context}")?;
}
Ok(())
}
}
}
}
impl Error for ClientError {}
impl ClientError {
fn is_semantic_wait_failure(&self) -> bool {
matches!(
self,
Self::Command { code, .. }
if code == "predicate_timeout" || code == "predicate_aborted"
)
}
fn attach_failure_capture(&mut self, capture: &Capture) {
let Self::Command { context, .. } = self else {
return;
};
let context = context.get_or_insert_with(|| json!({}));
if let Some(object) = context.as_object_mut()
&& let Ok(capture) = serde_json::to_value(capture)
{
object.insert("failure_capture".to_string(), capture);
}
}
}
impl From<io::Error> for ClientError {
fn from(error: io::Error) -> Self {
Self::Io(error.to_string())
}
}
impl From<serde_json::Error> for ClientError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error.to_string())
}
}
impl AgentClient {
pub fn connect(protocol_file: impl AsRef<Path>) -> Result<Self, ClientError> {
Self::with_config(AgentClientConfig {
protocol_file: protocol_file.as_ref().to_path_buf(),
..Default::default()
})
}
pub fn with_config(config: AgentClientConfig) -> Result<Self, ClientError> {
let protocol = read_protocol(&config.protocol_file)?;
if protocol.max_wait_frames == 0 {
return Err(ClientError::Protocol(
"protocol advertises zero max_wait_frames".to_string(),
));
}
if protocol.max_abort_predicates == 0 {
return Err(ClientError::Protocol(
"protocol advertises zero max_abort_predicates".to_string(),
));
}
if protocol.max_time_advance_steps == 0 {
return Err(ClientError::Protocol(
"protocol advertises zero max_time_advance_steps".to_string(),
));
}
let max_time_advance = Duration::try_from_secs_f64(protocol.max_time_advance_seconds)
.map_err(|_| {
ClientError::Protocol(
"protocol max_time_advance_seconds must be finite and positive".to_string(),
)
})?;
if max_time_advance.is_zero() {
return Err(ClientError::Protocol(
"protocol advertises zero max_time_advance_seconds".to_string(),
));
}
let stream = TcpStream::connect_timeout(&protocol.socket_addr, config.timeout)
.map_err(|error| socket_error(error, &protocol.socket_addr))?;
stream.set_read_timeout(Some(config.timeout))?;
stream.set_write_timeout(Some(config.timeout))?;
let reader = BufReader::new(stream.try_clone()?);
let transcript = match &config.transcript_file {
Some(path) => {
if let Some(parent) = path.parent()
&& !parent.as_os_str().is_empty()
{
fs::create_dir_all(parent)?;
}
Some(OpenOptions::new().create(true).append(true).open(path)?)
}
None => None,
};
Ok(Self {
stream,
reader,
next_id: 1,
timeout: config.timeout,
transcript,
last_capture: None,
ocr: config.ocr,
capabilities: AgentCapabilities {
max_wait_frames: protocol.max_wait_frames,
max_abort_predicates: protocol.max_abort_predicates,
deterministic_time: protocol.deterministic_time,
max_time_advance_steps: protocol.max_time_advance_steps,
max_time_advance,
},
last_observation: None,
})
}
pub fn request(&mut self, mut request: Value) -> Result<Value, ClientError> {
let Some(object) = request.as_object_mut() else {
return Err(ClientError::Protocol(
"request must be a JSON object".to_string(),
));
};
object
.entry("id".to_string())
.or_insert_with(|| Value::from(self.next_id));
self.next_id = self.next_id.saturating_add(1);
let line = serde_json::to_string(&request)?;
let ts_ms = u64::try_from(unix_ms()).unwrap_or(u64::MAX);
let start = Instant::now();
writeln!(self.stream, "{line}")?;
self.stream.flush()?;
let mut response = String::new();
if let Err(error) = self.reader.read_line(&mut response) {
return Err(self.read_error(error));
}
if response.is_empty() {
return Err(ClientError::Io(
"agent socket closed before response".to_string(),
));
}
let response: Value = serde_json::from_str(&response)?;
if let Some(transcript) = &mut self.transcript {
serde_json::to_writer(
&mut *transcript,
&json!({
"ts_ms": ts_ms,
"duration_ms": u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
"request": request,
"response": response,
}),
)?;
writeln!(transcript)?;
transcript.flush()?;
}
self.record_response_context(&response);
if response["ok"] == Value::Bool(true) {
return Ok(response);
}
let code = response["error"]["code"]
.as_str()
.unwrap_or("command_failed")
.to_string();
let mut message = response["error"]["message"]
.as_str()
.unwrap_or("game returned an error")
.to_string();
if code == "timeout" {
message = self.with_last_capture(message);
}
let context = response["error"].get("context").cloned();
Err(ClientError::Command {
code,
message,
context,
})
}
pub fn replay_jsonl(&mut self, path: impl AsRef<Path>) -> Result<Vec<Value>, ClientError> {
let file = File::open(path)?;
let lines = BufReader::new(file)
.lines()
.collect::<Result<Vec<_>, _>>()?;
let mut responses = Vec::new();
for line in lines {
if line.trim().is_empty() {
continue;
}
let value: Value = serde_json::from_str(&line)?;
let request = value.get("request").cloned().unwrap_or(value);
responses.push(self.request(request)?);
}
Ok(responses)
}
pub fn window_info(&mut self) -> Result<Value, ClientError> {
self.request(json!({"command": "window_info"}))
}
pub fn cursor_move(&mut self, x: f32, y: f32) -> Result<Value, ClientError> {
self.request(json!({"command": "cursor_move", "x": x, "y": y}))
}
pub fn key_down(&mut self, key: &str) -> Result<Value, ClientError> {
self.request(json!({"command": "key_down", "key": key}))
}
pub fn key_up(&mut self, key: &str) -> Result<Value, ClientError> {
self.request(json!({"command": "key_up", "key": key}))
}
pub fn mouse_down(&mut self, button: &str) -> Result<Value, ClientError> {
self.request(json!({"command": "mouse_down", "button": button}))
}
pub fn mouse_up(&mut self, button: &str) -> Result<Value, ClientError> {
self.request(json!({"command": "mouse_up", "button": button}))
}
pub fn scroll(&mut self, lines: f32) -> Result<Value, ClientError> {
self.request(json!({"command": "scroll", "lines": lines}))
}
pub fn click(&mut self, x: f32, y: f32, button: &str) -> Result<Value, ClientError> {
self.request(json!({"command": "click", "x": x, "y": y, "button": button}))
}
pub fn drag(
&mut self,
button: &str,
from: [f32; 2],
to: [f32; 2],
steps: u16,
frames: u16,
) -> Result<Value, ClientError> {
self.request(json!({
"command": "drag",
"button": button,
"from": from,
"to": to,
"steps": steps,
"frames": frames,
}))
}
pub fn key_tap(&mut self, key: &str) -> Result<Value, ClientError> {
self.request(json!({"command": "key_tap", "key": key}))
}
pub fn key_hold(&mut self, key: &str, frames: u16) -> Result<Value, ClientError> {
self.request(json!({"command": "key_hold", "key": key, "frames": frames}))
}
pub fn release_all_inputs(&mut self) -> Result<Value, ClientError> {
self.request(json!({"command": "release_all_inputs"}))
}
pub fn shutdown(&mut self) -> Result<Value, ClientError> {
self.request(json!({"command": "shutdown"}))
}
pub fn wait_for_disconnect(&mut self) -> Result<(), ClientError> {
let mut byte = [0_u8; 1];
match self.reader.read(&mut byte) {
Ok(0) => Ok(()),
Ok(_) => Err(ClientError::Protocol(
"unexpected protocol data after shutdown acknowledgment".to_string(),
)),
Err(error) => Err(error.into()),
}
}
fn record_response_context(&mut self, response: &Value) {
let capture_value = response["result"]
.get("capture")
.or_else(|| response["result"].get("latest_capture"))
.or_else(|| response["error"]["context"].get("latest_capture"));
if let Some(capture) =
capture_value.and_then(|value| serde_json::from_value::<Capture>(value.clone()).ok())
{
self.last_capture = Some(capture);
}
let observation_value = response["result"]
.get("details")
.or_else(|| response["error"]["context"].get("observed_predicate"));
if let Some(observation) = observation_value
.and_then(|value| serde_json::from_value::<ObservedPredicate>(value.clone()).ok())
{
self.last_observation = Some(observation);
}
}
fn read_error(&self, error: io::Error) -> ClientError {
if error.kind() == io::ErrorKind::TimedOut || error.kind() == io::ErrorKind::WouldBlock {
ClientError::Io(self.with_last_capture(format!(
"agent request timed out after {} ms",
self.timeout.as_millis()
)))
} else {
ClientError::Io(error.to_string())
}
}
fn with_last_capture(&self, message: String) -> String {
match &self.last_capture {
Some(capture) => format!("{message}; last captured frame: {}", capture.path.display()),
None => message,
}
}
fn validate_wait_limit(&self, name: &str, requested: u64) -> Result<(), ClientError> {
let supported = u64::from(self.capabilities.max_wait_frames);
if requested > supported {
return Err(ClientError::Protocol(format!(
"{name}={requested} exceeds server limit {supported}; configure AgentFeedbackConfig.max_wait_frames or issue explicit bounded requests"
)));
}
Ok(())
}
}
impl Drop for AgentClient {
fn drop(&mut self) {
let _ = writeln!(
self.stream,
"{}",
json!({"id": self.next_id, "command": "release_all_inputs"})
);
let _ = self.stream.flush();
}
}
fn capture_from_response(response: &Value) -> Result<Capture, ClientError> {
serde_json::from_value(response["result"]["capture"].clone()).map_err(|error| {
ClientError::Protocol(format!("invalid capture metadata ({error}): {response}"))
})
}
mod image_assertions;
#[cfg(test)]
use image_assertions::run_tesseract;
mod protocol_file;
use protocol_file::{read_protocol, socket_error};
mod diagnostics;
pub use diagnostics::{
ComparisonOperator, ObservedPredicate, Predicate, PredicateOutcome, ResolvedTargetKind,
TargetBounds, TargetInfo, TargetKind, TargetSelector,
};
mod timing;
#[cfg(test)]
mod tests;