car_server_core/assistant/
vision_tools.rs

1//! Vision tools for the general assistant — "CAR can see": read and understand
2//! image files. The CONSUMER counterpart to the media generators
3//! ([`MediaTools`](super::media_tools), [`StudioMediaTools`](super::studio_tools)):
4//! a text-only agent (Claude Code, Codex) can neither generate an image nor read
5//! one back.
6//!
7//! Backed by [`car_vision`] — Apple Vision on macOS, with a cross-platform
8//! Tesseract-CLI fallback for OCR. These tools are **read-only**: they take an
9//! image path (clamped under the working root, so the agent can't read arbitrary
10//! host files) and return text/labels. They never write and never mutate, so
11//! they carry no `mutating` flag.
12
13use std::path::PathBuf;
14
15use async_trait::async_trait;
16use car_engine::ToolExecutor;
17use serde_json::{json, Value};
18
19use crate::coder::policy::stays_under;
20
21/// Host-side image understanding, backed by `car_vision`.
22pub struct VisionTools {
23    root: PathBuf,
24}
25
26/// Every vision schema that can be model-facing on a capable host. Runtime
27/// advertisement still filters this catalog through [`car_vision::is_available`].
28pub(super) fn catalog_tool_defs() -> Vec<Value> {
29    // OCR works via the Apple Vision shim OR the Tesseract fallback.
30    let mut defs = vec![json!({
31        "name": "read_image_text",
32        "description": "Read the text in an image file (OCR) — a screenshot, a scan, a photo \
33            of a document or sign, or a generated image that contains words. Returns the \
34            recognized text. Give a path to an image under the working directory.",
35        "parameters": {
36            "type": "object",
37            "properties": {
38                "image_path": {
39                    "type": "string",
40                    "description": "Path to the image, relative to the working directory."
41                }
42            },
43            "required": ["image_path"]
44        }
45    })];
46    // Classification is cross-platform; the first off-macOS call may download
47    // the default model, but the schema itself is stable.
48    defs.push(json!({
49        "name": "classify_image",
50        "description": "Identify what an image depicts — returns ranked content labels \
51            with confidence (e.g. \"golden retriever\", \"airliner\", \"beach\"). Give a \
52            path to an image under the working directory. Use this to check what a \
53            generated or downloaded image actually shows.",
54        "parameters": {
55            "type": "object",
56            "properties": {
57                "image_path": {
58                    "type": "string",
59                    "description": "Path to the image, relative to the working directory."
60                },
61                "top_k": {
62                    "type": "integer",
63                    "description": "How many labels to return (default 5, max 20)."
64                }
65            },
66            "required": ["image_path"]
67        }
68    }));
69    defs
70}
71
72impl VisionTools {
73    pub fn new(root: PathBuf) -> Self {
74        Self { root }
75    }
76
77    /// Advertised tool defs — empty when no vision backend is reachable on this
78    /// host (no Apple Vision shim AND no Tesseract), so a tool that would only
79    /// error is never offered.
80    pub fn tool_defs(&self) -> Vec<Value> {
81        if !car_vision::is_available() {
82            return Vec::new();
83        }
84        catalog_tool_defs()
85    }
86
87    /// Resolve + validate a caller-supplied image path: it must stay under the
88    /// working root (so the agent can't read arbitrary host files) and exist.
89    /// Read-only — no directory creation.
90    fn resolve_input(&self, params: &Value) -> Result<PathBuf, String> {
91        let rel = params
92            .get("image_path")
93            .and_then(|v| v.as_str())
94            .map(str::trim)
95            .filter(|s| !s.is_empty())
96            .ok_or("requires a non-empty `image_path`")?;
97        if !stays_under(&self.root, rel) {
98            return Err(format!("image_path '{rel}' escapes the working directory"));
99        }
100        let abs = self.root.join(rel);
101        if !abs.exists() {
102            return Err(format!("image_path '{rel}' does not exist"));
103        }
104        Ok(abs)
105    }
106
107    async fn run_read_image_text(&self, params: &Value) -> Result<Value, String> {
108        let path = self.resolve_input(params)?;
109        // The FFI/CLI OCR call is synchronous and CPU-bound — run it off the
110        // async worker.
111        let observations = tokio::task::spawn_blocking(move || {
112            car_vision::ocr::recognize(&path, &car_vision::ocr::OcrConfig::default())
113        })
114        .await
115        .map_err(|e| format!("OCR task join: {e}"))?
116        .map_err(|e| format!("OCR failed: {e}"))?;
117
118        let text = observations
119            .iter()
120            .map(|o| o.text.as_str())
121            .collect::<Vec<_>>()
122            .join("\n");
123        Ok(json!({ "text": text, "line_count": observations.len() }))
124    }
125
126    async fn run_classify_image(&self, params: &Value) -> Result<Value, String> {
127        let path = self.resolve_input(params)?;
128        let top_k = params
129            .get("top_k")
130            .and_then(Value::as_u64)
131            .unwrap_or(5)
132            .clamp(1, 20) as usize;
133
134        let labels =
135            tokio::task::spawn_blocking(move || car_vision::classify::classify(&path, top_k))
136                .await
137                .map_err(|e| format!("classify task join: {e}"))?
138                .map_err(|e| format!("image classification failed: {e}"))?;
139
140        let labels_json: Vec<Value> = labels
141            .iter()
142            .map(|c| json!({ "label": c.identifier, "confidence": c.confidence }))
143            .collect();
144        Ok(json!({ "labels": labels_json }))
145    }
146}
147
148#[async_trait]
149impl ToolExecutor for VisionTools {
150    async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
151        match tool {
152            "read_image_text" => self.run_read_image_text(params).await,
153            "classify_image" => self.run_classify_image(params).await,
154            // The prefix must be exactly "unknown tool" so the ChainedDelegate
155            // falls through to the next executor.
156            other => Err(format!("unknown tool: '{other}'")),
157        }
158    }
159}
160
161#[cfg(test)]
162mod tests {
163    use super::*;
164
165    #[tokio::test]
166    async fn unknown_tool_falls_through() {
167        let v = VisionTools::new(std::env::temp_dir());
168        let err = v.execute("nope", &json!({})).await.unwrap_err();
169        assert!(err.starts_with("unknown tool"), "{err}");
170    }
171
172    #[tokio::test]
173    async fn rejects_missing_and_escaping_paths() {
174        let dir = tempfile::tempdir().unwrap();
175        let v = VisionTools::new(dir.path().to_path_buf());
176        // Escapes the root.
177        assert!(v
178            .execute("read_image_text", &json!({"image_path": "../secret.png"}))
179            .await
180            .unwrap_err()
181            .contains("escapes"));
182        // Under root but does not exist.
183        assert!(v
184            .execute("read_image_text", &json!({"image_path": "nope.png"}))
185            .await
186            .unwrap_err()
187            .contains("does not exist"));
188        // Empty.
189        assert!(v
190            .execute("classify_image", &json!({"image_path": "  "}))
191            .await
192            .unwrap_err()
193            .contains("non-empty"));
194    }
195}
car_server_core/assistant/vision_tools.rs

car_server_core/assistant/
vision_tools.rs
