open_agent/types/
image.rs

1/// Image detail level for vision API calls.
2///
3/// Controls the resolution and token cost of image processing.
4///
5/// # Token Costs Vary by Model ⚠️
6///
7/// **OpenAI Vision API** (reference values):
8/// - `Low`: ~85 tokens (512x512 max resolution)
9/// - `High`: Variable tokens based on image dimensions
10/// - `Auto`: Model decides (balanced default)
11///
12/// **Local models** (llama.cpp, Ollama, vLLM):
13/// - May have **completely different** token calculations
14/// - Some models don't charge tokens for images at all
15/// - The `ImageDetail` setting may be ignored entirely
16///
17/// **Anthropic messages protocol**: the API has no equivalent field, so the setting is
18/// dropped in translation rather than mapped onto something approximate.
19///
20/// **Recommendation:** Always benchmark your specific model to understand
21/// actual token consumption. Do not rely on OpenAI's values for capacity planning
22/// with local models.
23///
24/// # Examples
25///
26/// ```
27/// use open_agent::ImageDetail;
28///
29/// let detail = ImageDetail::High;
30/// assert_eq!(detail.to_string(), "high");
31/// ```
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
33#[serde(rename_all = "lowercase")]
34#[derive(Default)]
35pub enum ImageDetail {
36    /// Low resolution (512x512), fixed 85 tokens
37    Low,
38    /// High resolution, variable tokens based on dimensions
39    High,
40    /// Automatic selection (default)
41    #[default]
42    Auto,
43}
44
45impl std::fmt::Display for ImageDetail {
46    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
47        match self {
48            ImageDetail::Low => write!(f, "low"),
49            ImageDetail::High => write!(f, "high"),
50            ImageDetail::Auto => write!(f, "auto"),
51        }
52    }
53}
54
55/// Image content block for vision-capable models.
56///
57/// Supports both URL-based images and base64-encoded images.
58///
59/// # Examples
60///
61/// ```
62/// use open_agent::{ImageBlock, ImageDetail};
63///
64/// // From URL
65/// let image = ImageBlock::from_url("https://example.com/image.jpg")?;
66///
67/// // From base64 (use properly formatted base64)
68/// let image = ImageBlock::from_base64("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", "image/png")?;
69///
70/// // With detail level
71/// let image = ImageBlock::from_url("https://example.com/image.jpg")?
72///     .with_detail(ImageDetail::High);
73/// # Ok::<(), open_agent::Error>(())
74/// ```
75#[derive(Debug, Clone, Serialize, Deserialize)]
76pub struct ImageBlock {
77    url: String,
78    #[serde(default)]
79    detail: ImageDetail,
80}
81
82impl ImageBlock {
83    /// Creates a new image block from a URL.
84    ///
85    /// # Arguments
86    ///
87    /// * `url` - The image URL (must be HTTP, HTTPS, or data URI)
88    ///
89    /// # Errors
90    ///
91    /// Returns `Error::InvalidInput` if:
92    /// - URL is empty
93    /// - URL contains control characters (newline, tab, null, etc.)
94    /// - URL scheme is not `http://`, `https://`, or `data:`
95    /// - Data URI is malformed (missing MIME type or base64 encoding)
96    /// - Data URI base64 portion has invalid characters, length, or padding
97    ///
98    /// # Warnings
99    ///
100    /// - Logs a warning to stderr if URL exceeds 2000 characters
101    ///
102    /// # Example
103    ///
104    /// ```
105    /// use open_agent::ImageBlock;
106    ///
107    /// let image = ImageBlock::from_url("https://example.com/cat.jpg")?;
108    /// assert_eq!(image.url(), "https://example.com/cat.jpg");
109    /// # Ok::<(), open_agent::Error>(())
110    /// ```
111    pub fn from_url(url: impl Into<String>) -> crate::Result<Self> {
112        let url = url.into();
113
114        // Validate URL is not empty
115        if url.is_empty() {
116            return Err(crate::Error::invalid_input("Image URL cannot be empty"));
117        }
118
119        // Check for control characters in URL
120        if url.contains(char::is_control) {
121            return Err(crate::Error::invalid_input(
122                "Image URL contains invalid control characters",
123            ));
124        }
125
126        // Warn about very long URLs (>2000 chars)
127        if url.len() > 2000 {
128            eprintln!(
129                "WARNING: Very long image URL ({} chars). \
130                 Some APIs may have URL length limits.",
131                url.len()
132            );
133        }
134
135        // Validate URL scheme
136        if url.starts_with("http://") || url.starts_with("https://") {
137            // Valid HTTP/HTTPS URL
138            Ok(Self {
139                url,
140                detail: ImageDetail::default(),
141            })
142        } else if let Some(mime_part) = url.strip_prefix("data:") {
143            // Validate data URI format: data:MIME;base64,DATA
144            if !url.contains(";base64,") {
145                return Err(crate::Error::invalid_input(
146                    "Data URI must be in format: data:image/TYPE;base64,DATA",
147                ));
148            }
149
150            // Extract MIME type from data:MIME;base64,DATA
151            let mime_type = if let Some(semicolon_pos) = mime_part.find(';') {
152                &mime_part[..semicolon_pos]
153            } else {
154                return Err(crate::Error::invalid_input(
155                    "Malformed data URI: missing MIME type",
156                ));
157            };
158
159            if mime_type.is_empty() || !mime_type.starts_with("image/") {
160                return Err(crate::Error::invalid_input(
161                    "Data URI MIME type must start with 'image/'",
162                ));
163            }
164
165            // Extract and validate base64 data portion
166            if let Some(base64_start_pos) = url.find(";base64,") {
167                let base64_data = &url[base64_start_pos + 8..]; // Skip ";base64,"
168
169                // Validate base64 data using same rules as from_base64()
170                // Check data is not empty
171                if base64_data.is_empty() {
172                    return Err(crate::Error::invalid_input(
173                        "Data URI base64 data cannot be empty",
174                    ));
175                }
176
177                // Check character set
178                if !base64_data
179                    .chars()
180                    .all(|c| c.is_alphanumeric() || c == '+' || c == '/' || c == '=')
181                {
182                    return Err(crate::Error::invalid_input(
183                        "Data URI base64 data contains invalid characters. Valid characters: A-Z, a-z, 0-9, +, /, =",
184                    ));
185                }
186
187                // Check length (must be multiple of 4)
188                if base64_data.len() % 4 != 0 {
189                    return Err(crate::Error::invalid_input(
190                        "Data URI base64 data has invalid length (must be multiple of 4)",
191                    ));
192                }
193
194                // Validate padding
195                let equals_count = base64_data.chars().filter(|c| *c == '=').count();
196                if equals_count > 2 {
197                    return Err(crate::Error::invalid_input(
198                        "Data URI base64 data has invalid padding (max 2 '=' characters allowed)",
199                    ));
200                }
201                // Padding must be at the end
202                if equals_count > 0 {
203                    let trimmed = base64_data.trim_end_matches('=');
204                    if trimmed.len() + equals_count != base64_data.len() {
205                        return Err(crate::Error::invalid_input(
206                            "Data URI base64 padding characters must be at the end",
207                        ));
208                    }
209                }
210            }
211
212            Ok(Self {
213                url,
214                detail: ImageDetail::default(),
215            })
216        } else {
217            Err(crate::Error::invalid_input(
218                "Image URL must start with http://, https://, or data:",
219            ))
220        }
221    }
222
223    /// Creates a new image block from base64-encoded data.
224    ///
225    /// # Arguments
226    ///
227    /// * `base64_data` - The base64-encoded image data
228    /// * `mime_type` - The MIME type (e.g., "image/jpeg", "image/png")
229    ///
230    /// # Errors
231    ///
232    /// Returns `Error::InvalidInput` if:
233    /// - Base64 data is empty
234    /// - Base64 contains invalid characters (only A-Z, a-z, 0-9, +, /, = allowed)
235    /// - Base64 length is not a multiple of 4
236    /// - Base64 has invalid padding (more than 2 '=' characters or not at end)
237    /// - MIME type is empty
238    /// - MIME type does not start with "image/"
239    /// - MIME type contains injection characters (;, \\n, \\r, ,)
240    ///
241    /// # Warnings
242    ///
243    /// - Logs a warning to stderr if base64 data exceeds 10MB (~7.5MB decoded)
244    ///
245    /// # Example
246    ///
247    /// ```
248    /// use open_agent::ImageBlock;
249    ///
250    /// let base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
251    /// let image = ImageBlock::from_base64(base64, "image/png")?;
252    /// assert!(image.url().starts_with("data:image/png;base64,"));
253    /// # Ok::<(), open_agent::Error>(())
254    /// ```
255    pub fn from_base64(
256        base64_data: impl AsRef<str>,
257        mime_type: impl AsRef<str>,
258    ) -> crate::Result<Self> {
259        let data = base64_data.as_ref();
260        let mime = mime_type.as_ref();
261
262        // Validate base64 data is not empty
263        if data.is_empty() {
264            return Err(crate::Error::invalid_input(
265                "Base64 image data cannot be empty",
266            ));
267        }
268
269        // Validate base64 character set (alphanumeric + +/=)
270        // This catches common errors like spaces, special characters, etc.
271        if !data
272            .chars()
273            .all(|c| c.is_alphanumeric() || c == '+' || c == '/' || c == '=')
274        {
275            return Err(crate::Error::invalid_input(
276                "Base64 data contains invalid characters. Valid characters: A-Z, a-z, 0-9, +, /, =",
277            ));
278        }
279
280        // Validate base64 padding (length must be multiple of 4)
281        if data.len() % 4 != 0 {
282            return Err(crate::Error::invalid_input(
283                "Base64 data has invalid length (must be multiple of 4)",
284            ));
285        }
286
287        // Validate padding characters only appear at the end (max 2)
288        let equals_count = data.chars().filter(|c| *c == '=').count();
289        if equals_count > 2 {
290            return Err(crate::Error::invalid_input(
291                "Base64 data has invalid padding (max 2 '=' characters allowed)",
292            ));
293        }
294        if equals_count > 0 {
295            // Padding must be at the end
296            let trimmed = data.trim_end_matches('=');
297            if trimmed.len() + equals_count != data.len() {
298                return Err(crate::Error::invalid_input(
299                    "Base64 padding characters must be at the end",
300                ));
301            }
302        }
303
304        // Validate MIME type is not empty
305        if mime.is_empty() {
306            return Err(crate::Error::invalid_input("MIME type cannot be empty"));
307        }
308
309        // Validate MIME type starts with "image/"
310        if !mime.starts_with("image/") {
311            return Err(crate::Error::invalid_input(
312                "MIME type must start with 'image/' (e.g., 'image/png', 'image/jpeg')",
313            ));
314        }
315
316        // Check for MIME type injection characters
317        if mime.contains([';', ',', '\n', '\r']) {
318            return Err(crate::Error::invalid_input(
319                "MIME type contains invalid characters (;, \\n, \\r not allowed)",
320            ));
321        }
322
323        // Warn about extremely large base64 data (>10MB)
324        if data.len() > 10_000_000 {
325            eprintln!(
326                "WARNING: Very large base64 image data ({} chars, ~{:.1}MB). \
327                 This may exceed API limits or cause performance issues.",
328                data.len(),
329                (data.len() as f64 * 0.75) / 1_000_000.0
330            );
331        }
332
333        let url = format!("data:{};base64,{}", mime, data);
334        Ok(Self {
335            url,
336            detail: ImageDetail::default(),
337        })
338    }
339
340    /// Creates a new image block from a local file path.
341    ///
342    /// This is a convenience method that reads the file from disk, encodes it as
343    /// base64, and creates an ImageBlock with a data URI. The MIME type is inferred
344    /// from the file extension.
345    ///
346    /// # Arguments
347    ///
348    /// * `path` - Path to the image file on the local filesystem
349    ///
350    /// # Errors
351    ///
352    /// Returns `Error::InvalidInput` if:
353    /// - File cannot be read
354    /// - File extension is missing or unsupported
355    /// - File is too large (>10MB warning)
356    ///
357    /// # Supported Formats
358    ///
359    /// - `.jpg`, `.jpeg` → `image/jpeg`
360    /// - `.png` → `image/png`
361    /// - `.gif` → `image/gif`
362    /// - `.webp` → `image/webp`
363    /// - `.bmp` → `image/bmp`
364    /// - `.svg` → `image/svg+xml`
365    ///
366    /// # Example
367    ///
368    /// ```no_run
369    /// use open_agent::ImageBlock;
370    ///
371    /// let image = ImageBlock::from_file_path("/path/to/photo.jpg")?;
372    /// # Ok::<(), open_agent::Error>(())
373    /// ```
374    ///
375    /// # Security Note
376    ///
377    /// This method reads files from the local filesystem. Ensure the path comes from
378    /// a trusted source to prevent unauthorized file access.
379    pub fn from_file_path(path: impl AsRef<std::path::Path>) -> crate::Result<Self> {
380        use base64::{Engine as _, engine::general_purpose};
381
382        let path = path.as_ref();
383
384        // Read file bytes
385        let bytes = std::fs::read(path).map_err(|e| {
386            crate::Error::invalid_input(format!(
387                "Failed to read image file '{}': {}",
388                path.display(),
389                e
390            ))
391        })?;
392
393        // Determine MIME type from file extension
394        let mime_type = match path.extension().and_then(|e| e.to_str()) {
395            Some("jpg") | Some("jpeg") => "image/jpeg",
396            Some("png") => "image/png",
397            Some("gif") => "image/gif",
398            Some("webp") => "image/webp",
399            Some("bmp") => "image/bmp",
400            Some("svg") => "image/svg+xml",
401            Some(ext) => {
402                return Err(crate::Error::invalid_input(format!(
403                    "Unsupported image file extension: .{}. Supported: jpg, jpeg, png, gif, webp, bmp, svg",
404                    ext
405                )));
406            }
407            None => {
408                return Err(crate::Error::invalid_input(
409                    "Image file path must have a file extension (e.g., .jpg, .png)",
410                ));
411            }
412        };
413
414        // Encode to base64
415        let base64_data = general_purpose::STANDARD.encode(&bytes);
416
417        // Use existing from_base64 method for validation
418        Self::from_base64(&base64_data, mime_type)
419    }
420
421    /// Sets the image detail level.
422    ///
423    /// # Example
424    ///
425    /// ```
426    /// use open_agent::{ImageBlock, ImageDetail};
427    ///
428    /// let image = ImageBlock::from_url("https://example.com/image.jpg")?
429    ///     .with_detail(ImageDetail::High);
430    /// # Ok::<(), open_agent::Error>(())
431    /// ```
432    pub fn with_detail(mut self, detail: ImageDetail) -> Self {
433        self.detail = detail;
434        self
435    }
436
437    /// Returns the image URL (or data URI for base64 images).
438    pub fn url(&self) -> &str {
439        &self.url
440    }
441
442    /// Returns the image detail level.
443    pub fn detail(&self) -> ImageDetail {
444        self.detail
445    }
446}
open_agent/types/image.rs

open_agent/types/
image.rs
