1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
//! Vision tools for the general assistant — "CAR can see": read and understand
//! image files. The CONSUMER counterpart to the media generators
//! ([`MediaTools`](super::media_tools), [`StudioMediaTools`](super::studio_tools)):
//! a text-only agent (Claude Code, Codex) can neither generate an image nor read
//! one back.
//!
//! Backed by [`car_vision`] — Apple Vision on macOS, with a cross-platform
//! Tesseract-CLI fallback for OCR. These tools are **read-only**: they take an
//! image path (clamped under the working root, so the agent can't read arbitrary
//! host files) and return text/labels. They never write and never mutate, so
//! they carry no `mutating` flag.
use std::path::PathBuf;
use async_trait::async_trait;
use car_engine::ToolExecutor;
use serde_json::{json, Value};
use crate::coder::policy::stays_under;
/// Host-side image understanding, backed by `car_vision`.
pub struct VisionTools {
root: PathBuf,
}
impl VisionTools {
pub fn new(root: PathBuf) -> Self {
Self { root }
}
/// Advertised tool defs — empty when no vision backend is reachable on this
/// host (no Apple Vision shim AND no Tesseract), so a tool that would only
/// error is never offered.
pub fn tool_defs(&self) -> Vec<Value> {
if !car_vision::is_available() {
return Vec::new();
}
// OCR works via the Apple Vision shim OR the Tesseract fallback, so it's
// advertised whenever any vision backend is present.
let mut defs = vec![json!({
"name": "read_image_text",
"description": "Read the text in an image file (OCR) — a screenshot, a scan, a photo \
of a document or sign, or a generated image that contains words. Returns the \
recognized text. Give a path to an image under the working directory.",
"parameters": {
"type": "object",
"properties": {
"image_path": {
"type": "string",
"description": "Path to the image, relative to the working directory."
}
},
"required": ["image_path"]
}
})];
// Image classification works on every platform: Apple Vision on macOS
// (built-in, zero config), and a default MobileNetV2 elsewhere —
// auto-downloaded + cached on first use (car-vision::classify_download),
// or an explicit CAR_VISION_CLASSIFY_MODEL override. So it's advertised
// unconditionally; the first off-macOS call fetches ~14 MB once (or
// returns a clean error if offline with no cache), like whisper STT.
defs.push(json!({
"name": "classify_image",
"description": "Identify what an image depicts — returns ranked content labels \
with confidence (e.g. \"golden retriever\", \"airliner\", \"beach\"). Give a \
path to an image under the working directory. Use this to check what a \
generated or downloaded image actually shows.",
"parameters": {
"type": "object",
"properties": {
"image_path": {
"type": "string",
"description": "Path to the image, relative to the working directory."
},
"top_k": {
"type": "integer",
"description": "How many labels to return (default 5, max 20)."
}
},
"required": ["image_path"]
}
}));
defs
}
/// Resolve + validate a caller-supplied image path: it must stay under the
/// working root (so the agent can't read arbitrary host files) and exist.
/// Read-only — no directory creation.
fn resolve_input(&self, params: &Value) -> Result<PathBuf, String> {
let rel = params
.get("image_path")
.and_then(|v| v.as_str())
.map(str::trim)
.filter(|s| !s.is_empty())
.ok_or("requires a non-empty `image_path`")?;
if !stays_under(&self.root, rel) {
return Err(format!("image_path '{rel}' escapes the working directory"));
}
let abs = self.root.join(rel);
if !abs.exists() {
return Err(format!("image_path '{rel}' does not exist"));
}
Ok(abs)
}
async fn run_read_image_text(&self, params: &Value) -> Result<Value, String> {
let path = self.resolve_input(params)?;
// The FFI/CLI OCR call is synchronous and CPU-bound — run it off the
// async worker.
let observations = tokio::task::spawn_blocking(move || {
car_vision::ocr::recognize(&path, &car_vision::ocr::OcrConfig::default())
})
.await
.map_err(|e| format!("OCR task join: {e}"))?
.map_err(|e| format!("OCR failed: {e}"))?;
let text = observations
.iter()
.map(|o| o.text.as_str())
.collect::<Vec<_>>()
.join("\n");
Ok(json!({ "text": text, "line_count": observations.len() }))
}
async fn run_classify_image(&self, params: &Value) -> Result<Value, String> {
let path = self.resolve_input(params)?;
let top_k = params
.get("top_k")
.and_then(Value::as_u64)
.unwrap_or(5)
.clamp(1, 20) as usize;
let labels =
tokio::task::spawn_blocking(move || car_vision::classify::classify(&path, top_k))
.await
.map_err(|e| format!("classify task join: {e}"))?
.map_err(|e| format!("image classification failed: {e}"))?;
let labels_json: Vec<Value> = labels
.iter()
.map(|c| json!({ "label": c.identifier, "confidence": c.confidence }))
.collect();
Ok(json!({ "labels": labels_json }))
}
}
#[async_trait]
impl ToolExecutor for VisionTools {
async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
match tool {
"read_image_text" => self.run_read_image_text(params).await,
"classify_image" => self.run_classify_image(params).await,
// The prefix must be exactly "unknown tool" so the ChainedDelegate
// falls through to the next executor.
other => Err(format!("unknown tool: '{other}'")),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn unknown_tool_falls_through() {
let v = VisionTools::new(std::env::temp_dir());
let err = v.execute("nope", &json!({})).await.unwrap_err();
assert!(err.starts_with("unknown tool"), "{err}");
}
#[tokio::test]
async fn rejects_missing_and_escaping_paths() {
let dir = tempfile::tempdir().unwrap();
let v = VisionTools::new(dir.path().to_path_buf());
// Escapes the root.
assert!(v
.execute("read_image_text", &json!({"image_path": "../secret.png"}))
.await
.unwrap_err()
.contains("escapes"));
// Under root but does not exist.
assert!(v
.execute("read_image_text", &json!({"image_path": "nope.png"}))
.await
.unwrap_err()
.contains("does not exist"));
// Empty.
assert!(v
.execute("classify_image", &json!({"image_path": " "}))
.await
.unwrap_err()
.contains("non-empty"));
}
}