car-server-core 0.34.0

//! Vision tools for the general assistant — "CAR can see": read and understand
//! image files. The CONSUMER counterpart to the media generators
//! ([`MediaTools`](super::media_tools), [`StudioMediaTools`](super::studio_tools)):
//! a text-only agent (Claude Code, Codex) can neither generate an image nor read
//! one back.
//!
//! Backed by [`car_vision`] — Apple Vision on macOS, with a cross-platform
//! Tesseract-CLI fallback for OCR. These tools are **read-only**: they take an
//! image path (clamped under the working root, so the agent can't read arbitrary
//! host files) and return text/labels. They never write and never mutate, so
//! they carry no `mutating` flag.

use std::path::PathBuf;

use async_trait::async_trait;
use car_engine::ToolExecutor;
use serde_json::{json, Value};

use crate::coder::policy::stays_under;

/// Host-side image understanding, backed by `car_vision`.
pub struct VisionTools {
    root: PathBuf,
}

impl VisionTools {
    pub fn new(root: PathBuf) -> Self {
        Self { root }
    }

    /// Advertised tool defs — empty when no vision backend is reachable on this
    /// host (no Apple Vision shim AND no Tesseract), so a tool that would only
    /// error is never offered.
    pub fn tool_defs(&self) -> Vec<Value> {
        if !car_vision::is_available() {
            return Vec::new();
        }
        // OCR works via the Apple Vision shim OR the Tesseract fallback, so it's
        // advertised whenever any vision backend is present.
        let mut defs = vec![json!({
            "name": "read_image_text",
            "description": "Read the text in an image file (OCR) — a screenshot, a scan, a photo \
                of a document or sign, or a generated image that contains words. Returns the \
                recognized text. Give a path to an image under the working directory.",
            "parameters": {
                "type": "object",
                "properties": {
                    "image_path": {
                        "type": "string",
                        "description": "Path to the image, relative to the working directory."
                    }
                },
                "required": ["image_path"]
            }
        })];
        // Classification / barcodes are Apple-Vision-only (macOS shim); the
        // Tesseract fallback covers OCR alone. Advertise them only on macOS, and
        // they still return a clean error if the shim wasn't compiled in.
        if cfg!(target_os = "macos") {
            defs.push(json!({
                "name": "classify_image",
                "description": "Identify what an image depicts — returns ranked content labels \
                    with confidence (e.g. \"golden retriever\", \"airliner\", \"beach\"). Give a \
                    path to an image under the working directory. Use this to check what a \
                    generated or downloaded image actually shows.",
                "parameters": {
                    "type": "object",
                    "properties": {
                        "image_path": {
                            "type": "string",
                            "description": "Path to the image, relative to the working directory."
                        },
                        "top_k": {
                            "type": "integer",
                            "description": "How many labels to return (default 5, max 20)."
                        }
                    },
                    "required": ["image_path"]
                }
            }));
        }
        defs
    }

    /// Resolve + validate a caller-supplied image path: it must stay under the
    /// working root (so the agent can't read arbitrary host files) and exist.
    /// Read-only — no directory creation.
    fn resolve_input(&self, params: &Value) -> Result<PathBuf, String> {
        let rel = params
            .get("image_path")
            .and_then(|v| v.as_str())
            .map(str::trim)
            .filter(|s| !s.is_empty())
            .ok_or("requires a non-empty `image_path`")?;
        if !stays_under(&self.root, rel) {
            return Err(format!("image_path '{rel}' escapes the working directory"));
        }
        let abs = self.root.join(rel);
        if !abs.exists() {
            return Err(format!("image_path '{rel}' does not exist"));
        }
        Ok(abs)
    }

    async fn run_read_image_text(&self, params: &Value) -> Result<Value, String> {
        let path = self.resolve_input(params)?;
        // The FFI/CLI OCR call is synchronous and CPU-bound — run it off the
        // async worker.
        let observations = tokio::task::spawn_blocking(move || {
            car_vision::ocr::recognize(&path, &car_vision::ocr::OcrConfig::default())
        })
        .await
        .map_err(|e| format!("OCR task join: {e}"))?
        .map_err(|e| format!("OCR failed: {e}"))?;

        let text = observations
            .iter()
            .map(|o| o.text.as_str())
            .collect::<Vec<_>>()
            .join("\n");
        Ok(json!({ "text": text, "line_count": observations.len() }))
    }

    async fn run_classify_image(&self, params: &Value) -> Result<Value, String> {
        let path = self.resolve_input(params)?;
        let top_k = params
            .get("top_k")
            .and_then(Value::as_u64)
            .unwrap_or(5)
            .clamp(1, 20) as usize;

        let labels =
            tokio::task::spawn_blocking(move || car_vision::classify::classify(&path, top_k))
                .await
                .map_err(|e| format!("classify task join: {e}"))?
                .map_err(|e| format!("image classification failed: {e}"))?;

        let labels_json: Vec<Value> = labels
            .iter()
            .map(|c| json!({ "label": c.identifier, "confidence": c.confidence }))
            .collect();
        Ok(json!({ "labels": labels_json }))
    }
}

#[async_trait]
impl ToolExecutor for VisionTools {
    async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
        match tool {
            "read_image_text" => self.run_read_image_text(params).await,
            "classify_image" => self.run_classify_image(params).await,
            // The prefix must be exactly "unknown tool" so the ChainedDelegate
            // falls through to the next executor.
            other => Err(format!("unknown tool: '{other}'")),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn unknown_tool_falls_through() {
        let v = VisionTools::new(std::env::temp_dir());
        let err = v.execute("nope", &json!({})).await.unwrap_err();
        assert!(err.starts_with("unknown tool"), "{err}");
    }

    #[tokio::test]
    async fn rejects_missing_and_escaping_paths() {
        let dir = tempfile::tempdir().unwrap();
        let v = VisionTools::new(dir.path().to_path_buf());
        // Escapes the root.
        assert!(v
            .execute("read_image_text", &json!({"image_path": "../secret.png"}))
            .await
            .unwrap_err()
            .contains("escapes"));
        // Under root but does not exist.
        assert!(v
            .execute("read_image_text", &json!({"image_path": "nope.png"}))
            .await
            .unwrap_err()
            .contains("does not exist"));
        // Empty.
        assert!(v
            .execute("classify_image", &json!({"image_path": "  "}))
            .await
            .unwrap_err()
            .contains("non-empty"));
    }
}