open_agent/types/
image.rs

1/// Image detail level for vision API calls.
2///
3/// Controls the resolution and token cost of image processing.
4///
5/// # Token Costs Vary by Model ⚠️
6///
7/// **OpenAI Vision API** (reference values):
8/// - `Low`: ~85 tokens (512x512 max resolution)
9/// - `High`: Variable tokens based on image dimensions
10/// - `Auto`: Model decides (balanced default)
11///
12/// **Local models** (llama.cpp, Ollama, vLLM):
13/// - May have **completely different** token calculations
14/// - Some models don't charge tokens for images at all
15/// - The `ImageDetail` setting may be ignored entirely
16///
17/// **Recommendation:** Always benchmark your specific model to understand
18/// actual token consumption. Do not rely on OpenAI's values for capacity planning
19/// with local models.
20///
21/// # Examples
22///
23/// ```
24/// use open_agent::ImageDetail;
25///
26/// let detail = ImageDetail::High;
27/// assert_eq!(detail.to_string(), "high");
28/// ```
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
30#[serde(rename_all = "lowercase")]
31#[derive(Default)]
32pub enum ImageDetail {
33    /// Low resolution (512x512), fixed 85 tokens
34    Low,
35    /// High resolution, variable tokens based on dimensions
36    High,
37    /// Automatic selection (default)
38    #[default]
39    Auto,
40}
41
42impl std::fmt::Display for ImageDetail {
43    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
44        match self {
45            ImageDetail::Low => write!(f, "low"),
46            ImageDetail::High => write!(f, "high"),
47            ImageDetail::Auto => write!(f, "auto"),
48        }
49    }
50}
51
52/// Image content block for vision-capable models.
53///
54/// Supports both URL-based images and base64-encoded images.
55///
56/// # Examples
57///
58/// ```
59/// use open_agent::{ImageBlock, ImageDetail};
60///
61/// // From URL
62/// let image = ImageBlock::from_url("https://example.com/image.jpg")?;
63///
64/// // From base64 (use properly formatted base64)
65/// let image = ImageBlock::from_base64("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", "image/png")?;
66///
67/// // With detail level
68/// let image = ImageBlock::from_url("https://example.com/image.jpg")?
69///     .with_detail(ImageDetail::High);
70/// # Ok::<(), open_agent::Error>(())
71/// ```
72#[derive(Debug, Clone, Serialize, Deserialize)]
73pub struct ImageBlock {
74    url: String,
75    #[serde(default)]
76    detail: ImageDetail,
77}
78
79impl ImageBlock {
80    /// Creates a new image block from a URL.
81    ///
82    /// # Arguments
83    ///
84    /// * `url` - The image URL (must be HTTP, HTTPS, or data URI)
85    ///
86    /// # Errors
87    ///
88    /// Returns `Error::InvalidInput` if:
89    /// - URL is empty
90    /// - URL contains control characters (newline, tab, null, etc.)
91    /// - URL scheme is not `http://`, `https://`, or `data:`
92    /// - Data URI is malformed (missing MIME type or base64 encoding)
93    /// - Data URI base64 portion has invalid characters, length, or padding
94    ///
95    /// # Warnings
96    ///
97    /// - Logs a warning to stderr if URL exceeds 2000 characters
98    ///
99    /// # Example
100    ///
101    /// ```
102    /// use open_agent::ImageBlock;
103    ///
104    /// let image = ImageBlock::from_url("https://example.com/cat.jpg")?;
105    /// assert_eq!(image.url(), "https://example.com/cat.jpg");
106    /// # Ok::<(), open_agent::Error>(())
107    /// ```
108    pub fn from_url(url: impl Into<String>) -> crate::Result<Self> {
109        let url = url.into();
110
111        // Validate URL is not empty
112        if url.is_empty() {
113            return Err(crate::Error::invalid_input("Image URL cannot be empty"));
114        }
115
116        // Check for control characters in URL
117        if url.contains(char::is_control) {
118            return Err(crate::Error::invalid_input(
119                "Image URL contains invalid control characters",
120            ));
121        }
122
123        // Warn about very long URLs (>2000 chars)
124        if url.len() > 2000 {
125            eprintln!(
126                "WARNING: Very long image URL ({} chars). \
127                 Some APIs may have URL length limits.",
128                url.len()
129            );
130        }
131
132        // Validate URL scheme
133        if url.starts_with("http://") || url.starts_with("https://") {
134            // Valid HTTP/HTTPS URL
135            Ok(Self {
136                url,
137                detail: ImageDetail::default(),
138            })
139        } else if let Some(mime_part) = url.strip_prefix("data:") {
140            // Validate data URI format: data:MIME;base64,DATA
141            if !url.contains(";base64,") {
142                return Err(crate::Error::invalid_input(
143                    "Data URI must be in format: data:image/TYPE;base64,DATA",
144                ));
145            }
146
147            // Extract MIME type from data:MIME;base64,DATA
148            let mime_type = if let Some(semicolon_pos) = mime_part.find(';') {
149                &mime_part[..semicolon_pos]
150            } else {
151                return Err(crate::Error::invalid_input(
152                    "Malformed data URI: missing MIME type",
153                ));
154            };
155
156            if mime_type.is_empty() || !mime_type.starts_with("image/") {
157                return Err(crate::Error::invalid_input(
158                    "Data URI MIME type must start with 'image/'",
159                ));
160            }
161
162            // Extract and validate base64 data portion
163            if let Some(base64_start_pos) = url.find(";base64,") {
164                let base64_data = &url[base64_start_pos + 8..]; // Skip ";base64,"
165
166                // Validate base64 data using same rules as from_base64()
167                // Check data is not empty
168                if base64_data.is_empty() {
169                    return Err(crate::Error::invalid_input(
170                        "Data URI base64 data cannot be empty",
171                    ));
172                }
173
174                // Check character set
175                if !base64_data
176                    .chars()
177                    .all(|c| c.is_alphanumeric() || c == '+' || c == '/' || c == '=')
178                {
179                    return Err(crate::Error::invalid_input(
180                        "Data URI base64 data contains invalid characters. Valid characters: A-Z, a-z, 0-9, +, /, =",
181                    ));
182                }
183
184                // Check length (must be multiple of 4)
185                if base64_data.len() % 4 != 0 {
186                    return Err(crate::Error::invalid_input(
187                        "Data URI base64 data has invalid length (must be multiple of 4)",
188                    ));
189                }
190
191                // Validate padding
192                let equals_count = base64_data.chars().filter(|c| *c == '=').count();
193                if equals_count > 2 {
194                    return Err(crate::Error::invalid_input(
195                        "Data URI base64 data has invalid padding (max 2 '=' characters allowed)",
196                    ));
197                }
198                // Padding must be at the end
199                if equals_count > 0 {
200                    let trimmed = base64_data.trim_end_matches('=');
201                    if trimmed.len() + equals_count != base64_data.len() {
202                        return Err(crate::Error::invalid_input(
203                            "Data URI base64 padding characters must be at the end",
204                        ));
205                    }
206                }
207            }
208
209            Ok(Self {
210                url,
211                detail: ImageDetail::default(),
212            })
213        } else {
214            Err(crate::Error::invalid_input(
215                "Image URL must start with http://, https://, or data:",
216            ))
217        }
218    }
219
220    /// Creates a new image block from base64-encoded data.
221    ///
222    /// # Arguments
223    ///
224    /// * `base64_data` - The base64-encoded image data
225    /// * `mime_type` - The MIME type (e.g., "image/jpeg", "image/png")
226    ///
227    /// # Errors
228    ///
229    /// Returns `Error::InvalidInput` if:
230    /// - Base64 data is empty
231    /// - Base64 contains invalid characters (only A-Z, a-z, 0-9, +, /, = allowed)
232    /// - Base64 length is not a multiple of 4
233    /// - Base64 has invalid padding (more than 2 '=' characters or not at end)
234    /// - MIME type is empty
235    /// - MIME type does not start with "image/"
236    /// - MIME type contains injection characters (;, \\n, \\r, ,)
237    ///
238    /// # Warnings
239    ///
240    /// - Logs a warning to stderr if base64 data exceeds 10MB (~7.5MB decoded)
241    ///
242    /// # Example
243    ///
244    /// ```
245    /// use open_agent::ImageBlock;
246    ///
247    /// let base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
248    /// let image = ImageBlock::from_base64(base64, "image/png")?;
249    /// assert!(image.url().starts_with("data:image/png;base64,"));
250    /// # Ok::<(), open_agent::Error>(())
251    /// ```
252    pub fn from_base64(
253        base64_data: impl AsRef<str>,
254        mime_type: impl AsRef<str>,
255    ) -> crate::Result<Self> {
256        let data = base64_data.as_ref();
257        let mime = mime_type.as_ref();
258
259        // Validate base64 data is not empty
260        if data.is_empty() {
261            return Err(crate::Error::invalid_input(
262                "Base64 image data cannot be empty",
263            ));
264        }
265
266        // Validate base64 character set (alphanumeric + +/=)
267        // This catches common errors like spaces, special characters, etc.
268        if !data
269            .chars()
270            .all(|c| c.is_alphanumeric() || c == '+' || c == '/' || c == '=')
271        {
272            return Err(crate::Error::invalid_input(
273                "Base64 data contains invalid characters. Valid characters: A-Z, a-z, 0-9, +, /, =",
274            ));
275        }
276
277        // Validate base64 padding (length must be multiple of 4)
278        if data.len() % 4 != 0 {
279            return Err(crate::Error::invalid_input(
280                "Base64 data has invalid length (must be multiple of 4)",
281            ));
282        }
283
284        // Validate padding characters only appear at the end (max 2)
285        let equals_count = data.chars().filter(|c| *c == '=').count();
286        if equals_count > 2 {
287            return Err(crate::Error::invalid_input(
288                "Base64 data has invalid padding (max 2 '=' characters allowed)",
289            ));
290        }
291        if equals_count > 0 {
292            // Padding must be at the end
293            let trimmed = data.trim_end_matches('=');
294            if trimmed.len() + equals_count != data.len() {
295                return Err(crate::Error::invalid_input(
296                    "Base64 padding characters must be at the end",
297                ));
298            }
299        }
300
301        // Validate MIME type is not empty
302        if mime.is_empty() {
303            return Err(crate::Error::invalid_input("MIME type cannot be empty"));
304        }
305
306        // Validate MIME type starts with "image/"
307        if !mime.starts_with("image/") {
308            return Err(crate::Error::invalid_input(
309                "MIME type must start with 'image/' (e.g., 'image/png', 'image/jpeg')",
310            ));
311        }
312
313        // Check for MIME type injection characters
314        if mime.contains([';', ',', '\n', '\r']) {
315            return Err(crate::Error::invalid_input(
316                "MIME type contains invalid characters (;, \\n, \\r not allowed)",
317            ));
318        }
319
320        // Warn about extremely large base64 data (>10MB)
321        if data.len() > 10_000_000 {
322            eprintln!(
323                "WARNING: Very large base64 image data ({} chars, ~{:.1}MB). \
324                 This may exceed API limits or cause performance issues.",
325                data.len(),
326                (data.len() as f64 * 0.75) / 1_000_000.0
327            );
328        }
329
330        let url = format!("data:{};base64,{}", mime, data);
331        Ok(Self {
332            url,
333            detail: ImageDetail::default(),
334        })
335    }
336
337    /// Creates a new image block from a local file path.
338    ///
339    /// This is a convenience method that reads the file from disk, encodes it as
340    /// base64, and creates an ImageBlock with a data URI. The MIME type is inferred
341    /// from the file extension.
342    ///
343    /// # Arguments
344    ///
345    /// * `path` - Path to the image file on the local filesystem
346    ///
347    /// # Errors
348    ///
349    /// Returns `Error::InvalidInput` if:
350    /// - File cannot be read
351    /// - File extension is missing or unsupported
352    /// - File is too large (>10MB warning)
353    ///
354    /// # Supported Formats
355    ///
356    /// - `.jpg`, `.jpeg` → `image/jpeg`
357    /// - `.png` → `image/png`
358    /// - `.gif` → `image/gif`
359    /// - `.webp` → `image/webp`
360    /// - `.bmp` → `image/bmp`
361    /// - `.svg` → `image/svg+xml`
362    ///
363    /// # Example
364    ///
365    /// ```no_run
366    /// use open_agent::ImageBlock;
367    ///
368    /// let image = ImageBlock::from_file_path("/path/to/photo.jpg")?;
369    /// # Ok::<(), open_agent::Error>(())
370    /// ```
371    ///
372    /// # Security Note
373    ///
374    /// This method reads files from the local filesystem. Ensure the path comes from
375    /// a trusted source to prevent unauthorized file access.
376    pub fn from_file_path(path: impl AsRef<std::path::Path>) -> crate::Result<Self> {
377        use base64::{Engine as _, engine::general_purpose};
378
379        let path = path.as_ref();
380
381        // Read file bytes
382        let bytes = std::fs::read(path).map_err(|e| {
383            crate::Error::invalid_input(format!(
384                "Failed to read image file '{}': {}",
385                path.display(),
386                e
387            ))
388        })?;
389
390        // Determine MIME type from file extension
391        let mime_type = match path.extension().and_then(|e| e.to_str()) {
392            Some("jpg") | Some("jpeg") => "image/jpeg",
393            Some("png") => "image/png",
394            Some("gif") => "image/gif",
395            Some("webp") => "image/webp",
396            Some("bmp") => "image/bmp",
397            Some("svg") => "image/svg+xml",
398            Some(ext) => {
399                return Err(crate::Error::invalid_input(format!(
400                    "Unsupported image file extension: .{}. Supported: jpg, jpeg, png, gif, webp, bmp, svg",
401                    ext
402                )));
403            }
404            None => {
405                return Err(crate::Error::invalid_input(
406                    "Image file path must have a file extension (e.g., .jpg, .png)",
407                ));
408            }
409        };
410
411        // Encode to base64
412        let base64_data = general_purpose::STANDARD.encode(&bytes);
413
414        // Use existing from_base64 method for validation
415        Self::from_base64(&base64_data, mime_type)
416    }
417
418    /// Sets the image detail level.
419    ///
420    /// # Example
421    ///
422    /// ```
423    /// use open_agent::{ImageBlock, ImageDetail};
424    ///
425    /// let image = ImageBlock::from_url("https://example.com/image.jpg")?
426    ///     .with_detail(ImageDetail::High);
427    /// # Ok::<(), open_agent::Error>(())
428    /// ```
429    pub fn with_detail(mut self, detail: ImageDetail) -> Self {
430        self.detail = detail;
431        self
432    }
433
434    /// Returns the image URL (or data URI for base64 images).
435    pub fn url(&self) -> &str {
436        &self.url
437    }
438
439    /// Returns the image detail level.
440    pub fn detail(&self) -> ImageDetail {
441        self.detail
442    }
443}
open_agent/types/image.rs

open_agent/types/
image.rs
