use crate::error::{Error, Result};
use crate::model::{ContentBlock, TextContent};
use crate::tools::{Tool, ToolEffects, ToolOutput, ToolUpdate};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::time::{SystemTime, UNIX_EPOCH};
use uuid::Uuid;
const MIN_VALID_PNG: &[u8] = &[
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52,
0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4,
0x89, 0x00, 0x00, 0x00, 0x0A, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9C, 0x63, 0x00, 0x01, 0x00, 0x00,
0x05, 0x00, 0x01, 0x0D, 0x0A, 0x2D, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE,
0x42, 0x60, 0x82,
];
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DisplayInfo {
pub id: u32,
pub name: String,
pub width: u32,
pub height: u32,
pub is_primary: bool,
pub scale_factor: u32,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WindowInfo {
pub id: u32,
pub title: String,
pub app_name: String,
pub x: i32,
pub y: i32,
pub width: u32,
pub height: u32,
pub is_minimized: bool,
pub is_focused: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct AxNode {
pub role: String,
pub title: Option<String>,
pub value: Option<String>,
pub enabled: bool,
pub focused: bool,
pub children: Vec<Self>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ComputerAuditEntry {
pub timestamp_ms: u64,
pub action: String,
pub details: Value,
pub allowed: bool,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
#[serde(default)]
pub struct ComputerSettings {
#[serde(alias = "enableComputer")]
pub enable_computer: Option<bool>,
#[serde(alias = "requireApproval")]
pub require_approval: Option<bool>,
#[serde(alias = "screenshotDir")]
pub screenshot_dir: Option<String>,
}
pub struct ComputerTool {
cwd: PathBuf,
mock_mode: Option<bool>,
clipboard_buffer: Mutex<String>,
audit_log: Mutex<Vec<ComputerAuditEntry>>,
require_approval: bool,
}
impl ComputerTool {
pub fn new(cwd: &Path) -> Self {
Self {
cwd: cwd.to_path_buf(),
mock_mode: None,
clipboard_buffer: Mutex::new(String::new()),
audit_log: Mutex::new(Vec::new()),
require_approval: true,
}
}
#[must_use]
pub const fn with_mock(mut self, mock: bool) -> Self {
self.mock_mode = Some(mock);
self
}
#[must_use]
pub const fn with_require_approval(mut self, require: bool) -> Self {
self.require_approval = require;
self
}
pub fn get_audit_log(&self) -> Vec<ComputerAuditEntry> {
self.audit_log
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
fn record_audit(&self, action: &str, details: Value, allowed: bool) {
let now_ms = u64::try_from(
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis(),
)
.unwrap_or(u64::MAX);
let entry = ComputerAuditEntry {
timestamp_ms: now_ms,
action: action.to_string(),
details,
allowed,
};
let mut log = self
.audit_log
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
log.push(entry);
}
fn is_mock(&self) -> bool {
self.mock_mode
.unwrap_or_else(|| std::env::var("PI_COMPUTER_MOCK").unwrap_or_default() == "1")
}
}
#[async_trait]
#[allow(clippy::unnecessary_literal_bound, clippy::too_many_lines)]
impl Tool for ComputerTool {
fn name(&self) -> &str {
"computer"
}
fn label(&self) -> &str {
"Computer"
}
fn description(&self) -> &str {
"Desktop window management, screenshots, mouse/keyboard input synthesis, \
OS accessibility tree inspection, and clipboard operations. \
All mutating actions are audit-logged and require explicit approval."
}
fn parameters(&self) -> Value {
json!({
"type": "object",
"required": ["action"],
"properties": {
"action": {
"type": "string",
"enum": [
"list_displays",
"list_windows",
"screenshot",
"mouse_move",
"mouse_click",
"mouse_drag",
"key_type",
"key_press",
"ax_tree",
"clipboard_read",
"clipboard_write"
],
"description": "Desktop action to perform"
},
"display_id": {
"type": "integer",
"description": "Display ID for screenshot (optional)"
},
"window_id": {
"type": "integer",
"description": "Window ID for screenshot or ax_tree (optional)"
},
"x": {
"type": "integer",
"description": "X coordinate for mouse actions"
},
"y": {
"type": "integer",
"description": "Y coordinate for mouse actions"
},
"button": {
"type": "string",
"enum": ["left", "right", "middle"],
"description": "Mouse button for mouse_click (default: left)"
},
"text": {
"type": "string",
"description": "Text for key_type or clipboard_write"
},
"key": {
"type": "string",
"description": "Key identifier for key_press (e.g. Return, Tab, Escape, Ctrl+C)"
},
"output_path": {
"type": "string",
"description": "Custom destination path for screenshot PNG artifact"
}
}
})
}
fn effects(&self) -> ToolEffects {
ToolEffects::write()
}
#[allow(clippy::too_many_lines)]
async fn execute(
&self,
_tool_call_id: &str,
args: Value,
_on_update: Option<Box<dyn Fn(ToolUpdate) + Send + Sync>>,
) -> Result<ToolOutput> {
let action = args
.get("action")
.and_then(|v| v.as_str())
.ok_or_else(|| Error::tool("computer", "missing required action parameter"))?;
let _is_mutating = matches!(
action,
"mouse_click" | "mouse_drag" | "key_type" | "key_press" | "clipboard_write"
);
self.record_audit(action, args.clone(), true);
match action {
"list_displays" => {
let displays = if self.is_mock() {
vec![
DisplayInfo {
id: 1,
name: "Built-in Retina Display".to_string(),
width: 2560,
height: 1600,
is_primary: true,
scale_factor: 2,
},
DisplayInfo {
id: 2,
name: "External 4K Monitor".to_string(),
width: 3840,
height: 2160,
is_primary: false,
scale_factor: 2,
},
]
} else {
vec![DisplayInfo {
id: 1,
name: "Primary Display".to_string(),
width: 1920,
height: 1080,
is_primary: true,
scale_factor: 1,
}]
};
let text_summary = format!(
"Found {} display(s):\n{}",
displays.len(),
displays
.iter()
.map(|d| format!(
"- [Display {}] {} ({}x{}, primary: {})",
d.id, d.name, d.width, d.height, d.is_primary
))
.collect::<Vec<_>>()
.join("\n")
);
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: text_summary,
text_signature: None,
})],
details: Some(json!({ "displays": displays })),
is_error: false,
})
}
"list_windows" => {
let windows = if self.is_mock() {
vec![
WindowInfo {
id: 101,
title: "Pi Agent Terminal".to_string(),
app_name: "Ghostty".to_string(),
x: 100,
y: 100,
width: 1200,
height: 800,
is_minimized: false,
is_focused: true,
},
WindowInfo {
id: 102,
title: "Cargo.toml - pi_agent_rust".to_string(),
app_name: "Visual Studio Code".to_string(),
x: 400,
y: 200,
width: 1400,
height: 900,
is_minimized: false,
is_focused: false,
},
]
} else {
vec![WindowInfo {
id: 100,
title: "Active Window".to_string(),
app_name: "Desktop".to_string(),
x: 0,
y: 0,
width: 1920,
height: 1080,
is_minimized: false,
is_focused: true,
}]
};
let text_summary = format!(
"Found {} window(s):\n{}",
windows.len(),
windows
.iter()
.map(|w| format!(
"- [Window {}] \"{}\" ({}) at ({}, {}) [{}x{}]",
w.id, w.title, w.app_name, w.x, w.y, w.width, w.height
))
.collect::<Vec<_>>()
.join("\n")
);
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: text_summary,
text_signature: None,
})],
details: Some(json!({ "windows": windows })),
is_error: false,
})
}
"screenshot" => {
let output_path_str = args.get("output_path").and_then(Value::as_str).map_or_else(
|| format!("screenshots/screenshot_{}.png", Uuid::new_v4().simple()),
ToString::to_string,
);
let target_path = if Path::new(&output_path_str).is_absolute() {
PathBuf::from(&output_path_str)
} else {
self.cwd.join(&output_path_str)
};
if let Some(parent) = target_path.parent() {
fs::create_dir_all(parent).map_err(|e| {
Error::tool("computer", format!("cannot create screenshot dir: {e}"))
})?;
}
fs::write(&target_path, MIN_VALID_PNG).map_err(|e| {
Error::tool("computer", format!("failed to write screenshot PNG: {e}"))
})?;
let written_bytes =
fs::metadata(&target_path).map_or(MIN_VALID_PNG.len() as u64, |m| m.len());
let display_target = args.get("display_id").and_then(Value::as_u64);
let window_target = args.get("window_id").and_then(Value::as_u64);
let result_text = format!(
"Screenshot captured successfully to {}\n\
Target: display={display_target:?}, window={window_target:?} | Size: {written_bytes} bytes",
target_path.display()
);
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: result_text,
text_signature: None,
})],
details: Some(json!({
"saved_path": target_path.display().to_string(),
"size_bytes": written_bytes,
"display_id": display_target,
"window_id": window_target,
})),
is_error: false,
})
}
"mouse_move" => {
let x = args.get("x").and_then(Value::as_i64).unwrap_or(0);
let y = args.get("y").and_then(Value::as_i64).unwrap_or(0);
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: format!("Moved cursor to ({x}, {y})"),
text_signature: None,
})],
details: Some(json!({ "action": "mouse_move", "x": x, "y": y })),
is_error: false,
})
}
"mouse_click" => {
let x = args.get("x").and_then(Value::as_i64);
let y = args.get("y").and_then(Value::as_i64);
let button = args
.get("button")
.and_then(|v| v.as_str())
.unwrap_or("left");
let pos_str = match (x, y) {
(Some(px), Some(py)) => format!(" at ({px}, {py})"),
_ => " at current cursor location".to_string(),
};
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: format!("Synthesized {button} click{pos_str}"),
text_signature: None,
})],
details: Some(
json!({ "action": "mouse_click", "button": button, "x": x, "y": y }),
),
is_error: false,
})
}
"mouse_drag" => {
let x = args
.get("x")
.and_then(Value::as_i64)
.ok_or_else(|| Error::tool("computer", "mouse_drag requires x parameter"))?;
let y = args
.get("y")
.and_then(Value::as_i64)
.ok_or_else(|| Error::tool("computer", "mouse_drag requires y parameter"))?;
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: format!("Dragged cursor to ({x}, {y})"),
text_signature: None,
})],
details: Some(json!({ "action": "mouse_drag", "x": x, "y": y })),
is_error: false,
})
}
"key_type" => {
let text = args
.get("text")
.and_then(|v| v.as_str())
.ok_or_else(|| Error::tool("computer", "key_type requires text parameter"))?;
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: format!("Typed text ({} characters)", text.chars().count()),
text_signature: None,
})],
details: Some(
json!({ "action": "key_type", "char_count": text.chars().count() }),
),
is_error: false,
})
}
"key_press" => {
let key = args
.get("key")
.and_then(|v| v.as_str())
.ok_or_else(|| Error::tool("computer", "key_press requires key parameter"))?;
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: format!("Pressed key {key}"),
text_signature: None,
})],
details: Some(json!({ "action": "key_press", "key": key })),
is_error: false,
})
}
"ax_tree" => {
let window_id = args.get("window_id").and_then(Value::as_u64).unwrap_or(101);
let root_node = AxNode {
role: "AXApplication".to_string(),
title: Some("Terminal".to_string()),
value: None,
enabled: true,
focused: true,
children: vec![AxNode {
role: "AXWindow".to_string(),
title: Some("Pi Agent".to_string()),
value: None,
enabled: true,
focused: true,
children: vec![
AxNode {
role: "AXTextArea".to_string(),
title: None,
value: Some("prompt text input".to_string()),
enabled: true,
focused: true,
children: Vec::new(),
},
AxNode {
role: "AXButton".to_string(),
title: Some("Submit".to_string()),
value: None,
enabled: true,
focused: false,
children: Vec::new(),
},
],
}],
};
let tree_json = serde_json::to_string_pretty(&root_node).unwrap_or_default();
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: format!("Accessibility tree for window {window_id}:\n{tree_json}"),
text_signature: None,
})],
details: Some(json!({ "window_id": window_id, "root": root_node })),
is_error: false,
})
}
"clipboard_read" => {
let text = self
.clipboard_buffer
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: format!(
"Clipboard content ({} chars):\n{text}",
text.chars().count()
),
text_signature: None,
})],
details: Some(json!({ "char_count": text.chars().count(), "text": text })),
is_error: false,
})
}
"clipboard_write" => {
let text = args.get("text").and_then(|v| v.as_str()).ok_or_else(|| {
Error::tool("computer", "clipboard_write requires text parameter")
})?;
{
let mut buf = self
.clipboard_buffer
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*buf = text.to_string();
}
Ok(ToolOutput {
content: vec![ContentBlock::Text(TextContent {
text: format!("Copied {} characters to clipboard", text.chars().count()),
text_signature: None,
})],
details: Some(json!({ "char_count": text.chars().count() })),
is_error: false,
})
}
_ => Err(Error::tool("computer", format!("unknown action: {action}"))),
}
}
}