microclaw_engine/tools/
describe_image.rs

1use async_trait::async_trait;
2use serde_json::{json, Value};
3use std::path::PathBuf;
4
5use crate::internal::tool_runtime::media_client::{load_bytes_from_location, MediaClient};
6use microclaw_core::llm_types::ToolDefinition;
7
8use super::{schema_object, Tool, ToolResult};
9use crate::config::{Config, MediaConfig, VisionConfig};
10
11/// OpenAI-compatible vision tool.
12///
13/// Calls `/v1/chat/completions` with a single image + instruction message.
14/// Accepts a local path (inside working_dir), a public URL, or a `data:` URI.
15/// Returns the model's text answer. Disabled by default.
16pub struct DescribeImageTool {
17    working_dir: PathBuf,
18    allowed_read_dirs: Vec<PathBuf>,
19    cfg: VisionConfig,
20    media: MediaConfig,
21    openai_api_key: Option<String>,
22    openai_base_url: Option<String>,
23    timeout_secs: u64,
24    /// Optional `aux_models.vision` override, applied over `cfg.model` when set.
25    vision_aux: Option<String>,
26}
27
28impl DescribeImageTool {
29    pub fn new(config: &Config) -> Self {
30        Self {
31            working_dir: PathBuf::from(&config.working_dir),
32            allowed_read_dirs: config
33                .media
34                .allowed_read_dirs
35                .iter()
36                .map(PathBuf::from)
37                .collect(),
38            cfg: config.media.vision.clone(),
39            media: config.media.clone(),
40            openai_api_key: config.openai_api_key.clone(),
41            openai_base_url: config.openai_base_url.clone(),
42            timeout_secs: config.tool_timeout_secs("describe_image", 60),
43            vision_aux: config.aux_models.vision_model().map(|s| s.to_string()),
44        }
45    }
46
47    fn client(&self) -> Result<MediaClient, String> {
48        let key = self
49            .media
50            .resolve_api_key(self.openai_api_key.as_deref())
51            .ok_or_else(|| {
52                "describe_image requires an API key (media.api_key, \
53                 MICROCLAW_OPENAI_API_KEY, OPENAI_API_KEY, or top-level \
54                 openai_api_key)."
55                    .to_string()
56            })?;
57        let base = self.media.resolve_base_url(self.openai_base_url.as_deref());
58        MediaClient::new(base, key, self.timeout_secs)
59    }
60}
61
62#[async_trait]
63impl Tool for DescribeImageTool {
64    fn name(&self) -> &str {
65        "describe_image"
66    }
67
68    fn definition(&self) -> ToolDefinition {
69        ToolDefinition {
70            name: self.name().into(),
71            description: "Describe / analyze / OCR an image via an OpenAI-compatible \
72                vision-capable chat model. Accepts a file path under the working \
73                directory, a public URL, or a `data:` URI. Returns the model's text \
74                answer. Disabled by default; requires operator opt-in via \
75                `media.vision.enabled`."
76                .into(),
77            input_schema: schema_object(
78                json!({
79                    "image": {
80                        "type": "string",
81                        "description": "Image source: local file path (inside working_dir), https:// URL, or data: URI."
82                    },
83                    "prompt": {
84                        "type": "string",
85                        "description": "Optional instruction (defaults to 'Describe this image in detail.'). Use to ask for OCR, specific analysis, etc."
86                    },
87                    "model": {
88                        "type": "string",
89                        "description": "Optional model override (e.g. gpt-4o, gpt-4o-mini)."
90                    }
91                }),
92                &["image"],
93            ),
94        }
95    }
96
97    async fn execute(&self, input: Value) -> ToolResult {
98        if !self.cfg.enabled {
99            return ToolResult::error(
100                "describe_image is disabled. Set media.vision.enabled=true to enable.".into(),
101            );
102        }
103        let image = match input.get("image").and_then(|v| v.as_str()) {
104            Some(v) if !v.trim().is_empty() => v.trim().to_string(),
105            _ => return ToolResult::error("Missing parameter: image".into()),
106        };
107        let prompt = input
108            .get("prompt")
109            .and_then(|v| v.as_str())
110            .unwrap_or("Describe this image in detail.")
111            .trim()
112            .to_string();
113        // Resolution: explicit tool input > aux_models.vision > media.vision.model.
114        let model = input
115            .get("model")
116            .and_then(|v| v.as_str())
117            .map(|s| s.to_string())
118            .or_else(|| self.vision_aux.clone())
119            .unwrap_or_else(|| self.cfg.model.clone());
120
121        let client = match self.client() {
122            Ok(c) => c,
123            Err(e) => return ToolResult::error(e),
124        };
125
126        // Build an image_url content block. For local files we inline as
127        // data: URI; for remote URLs we pass the URL through (the provider
128        // will fetch it). We still call SSRF check on remote URLs through
129        // `load_bytes_from_location` to normalize behavior.
130        let image_url: String = if image.starts_with("http://") || image.starts_with("https://") {
131            // Remote URLs: run SSRF check via load_bytes then re-encode as
132            // data: URI so provider always sees inline bytes (avoids cases
133            // where the provider can't reach the URL).
134            let (bytes, mime) = match load_bytes_from_location(
135                client.client(),
136                &image,
137                &self.working_dir,
138                &self.allowed_read_dirs,
139            )
140            .await
141            {
142                Ok(v) => v,
143                Err(e) => return ToolResult::error(format!("image fetch failed: {e}")),
144            };
145            let mime = mime.unwrap_or_else(|| "image/png".to_string());
146            use base64::Engine;
147            let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
148            format!("data:{mime};base64,{b64}")
149        } else if image.starts_with("data:") {
150            image.clone()
151        } else {
152            let (bytes, mime) = match load_bytes_from_location(
153                client.client(),
154                &image,
155                &self.working_dir,
156                &self.allowed_read_dirs,
157            )
158            .await
159            {
160                Ok(v) => v,
161                Err(e) => return ToolResult::error(format!("image read failed: {e}")),
162            };
163            let mime = mime.unwrap_or_else(|| "image/png".to_string());
164            use base64::Engine;
165            let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
166            format!("data:{mime};base64,{b64}")
167        };
168
169        let body = json!({
170            "model": model,
171            "max_tokens": self.cfg.max_tokens,
172            "messages": [
173                {
174                    "role": "user",
175                    "content": [
176                        { "type": "text", "text": prompt },
177                        { "type": "image_url", "image_url": { "url": image_url } }
178                    ]
179                }
180            ]
181        });
182
183        let resp = match client
184            .post_json("chat/completions")
185            .json(&body)
186            .send()
187            .await
188        {
189            Ok(r) => r,
190            Err(e) => return ToolResult::error(format!("vision API request failed: {e}")),
191        };
192        if !resp.status().is_success() {
193            let status = resp.status();
194            let text = resp.text().await.unwrap_or_default();
195            return ToolResult::error(format!("vision API HTTP {status}: {text}"));
196        }
197        let parsed: Value = match resp.json().await {
198            Ok(v) => v,
199            Err(e) => return ToolResult::error(format!("invalid JSON from vision API: {e}")),
200        };
201
202        let content = parsed
203            .get("choices")
204            .and_then(|v| v.as_array())
205            .and_then(|arr| arr.first())
206            .and_then(|c| c.get("message"))
207            .and_then(|m| m.get("content"));
208
209        let text = match content {
210            Some(Value::String(s)) => s.clone(),
211            Some(Value::Array(blocks)) => blocks
212                .iter()
213                .filter_map(|b| b.get("text").and_then(|t| t.as_str()))
214                .collect::<Vec<_>>()
215                .join("\n"),
216            _ => {
217                return ToolResult::error(
218                    "vision API response missing choices[0].message.content".into(),
219                );
220            }
221        };
222
223        ToolResult::success(text)
224    }
225}
226
227#[cfg(test)]
228mod tests {
229    // Full e2e with a mock chat/completions server lives in the integration
230    // test file; module-level tests here just sanity-check struct wiring
231    // once the rest of the PR compiles.
232}
microclaw_engine/tools/describe_image.rs

microclaw_engine/tools/
describe_image.rs
