oxi_agent/tools/
inspect_image_tool.rs

1/// Inspect Image tool — analyze images using Vision LLM capabilities.
2///
3/// Accepts an image path or reference and a question about the image.
4/// Requires a Vision-capable model to be active. This tool formats the
5/// request and delegates to the Vision model for analysis.
6use super::{AgentTool, AgentToolResult, ToolContext, ToolError, ToolExecutionMode, ToolTier};
7use async_trait::async_trait;
8use serde_json::{Value, json};
9use tokio::sync::oneshot;
10
11/// InspectImageTool — analyze an image with a question.
12pub struct InspectImageTool;
13
14#[async_trait]
15impl AgentTool for InspectImageTool {
16    fn name(&self) -> &str {
17        "inspect_image"
18    }
19
20    fn label(&self) -> &str {
21        "Inspect Image"
22    }
23
24    fn description(&self) -> &str {
25        concat!(
26            "Analyze an image with a specific question. ",
27            "Provide the image path (local file path, or Image #N reference) ",
28            "and a question about the image content. ",
29            "Requires a Vision-capable model. The image will be loaded and ",
30            "sent to the model for analysis."
31        )
32    }
33
34    fn essential(&self) -> bool {
35        false
36    }
37
38    fn parameters_schema(&self) -> Value {
39        json!({
40            "type": "object",
41            "properties": {
42                "path": {
43                    "type": "string",
44                    "description": "Image path: local file path, Image #N label, or attachment:// URI."
45                },
46                "question": {
47                    "type": "string",
48                    "description": "Question about the image content."
49                }
50            },
51            "required": ["path", "question"]
52        })
53    }
54
55    fn intent(&self) -> Option<&str> {
56        Some("Analyze an image")
57    }
58
59    fn execution_mode(&self) -> ToolExecutionMode {
60        ToolExecutionMode::SequentialOnly
61    }
62
63    fn tool_tier(&self) -> ToolTier {
64        ToolTier::Read
65    }
66
67    async fn execute(
68        &self,
69        _tool_call_id: &str,
70        params: Value,
71        _signal: Option<oneshot::Receiver<()>>,
72        _ctx: &ToolContext,
73    ) -> Result<AgentToolResult, ToolError> {
74        let path = params
75            .get("path")
76            .and_then(|v| v.as_str())
77            .ok_or_else(|| "Missing required parameter: path".to_string())?;
78
79        let question = params
80            .get("question")
81            .and_then(|v| v.as_str())
82            .ok_or_else(|| "Missing required parameter: question".to_string())?;
83
84        // In a full implementation, this would:
85        // 1. Resolve the image path -> load image bytes
86        // 2. Send the image + question to a Vision-capable LLM
87        // 3. Return the analysis
88        //
89        // For now, return an informative message describing how to use
90        // Vision-capable models for image analysis.
91
92        Ok(AgentToolResult::success(format!(
93            concat!(
94                "Image analysis requested.\n",
95                "Path: {}\n",
96                "Question: {}\n\n",
97                "Image analysis requires a Vision-capable model. ",
98                "To analyze this image:\n",
99                "1. Ensure a Vision-capable model is active (e.g., gpt-4o, claude-3.5-sonnet, gemini-2.0-flash)\n",
100                "2. Use `read` to view the image file directly\n",
101                "3. Or describe the image content in a message to the model\n\n",
102                "The image would be loaded from: {}"
103            ),
104            path, question, path
105        )))
106    }
107}
108
109#[cfg(test)]
110mod tests {
111    use super::*;
112
113    #[tokio::test]
114    async fn test_inspect_image_basic() {
115        let tool = InspectImageTool;
116        let params = json!({
117            "path": "screenshot.png",
118            "question": "What does this UI show?"
119        });
120        let result = tool
121            .execute("id", params, None, &ToolContext::default())
122            .await
123            .unwrap();
124        assert!(result.success);
125        assert!(result.output.contains("Image analysis requested"));
126    }
127}
oxi_agent/tools/inspect_image_tool.rs

oxi_agent/tools/
inspect_image_tool.rs
