Skip to main content

host/
view_image.rs

1//! `rlm.view_image` host side, ported from ultron/rlm/view-image.ts over utils/image-process.ts,
2//! utils/image-resize-core.ts and utils/mime.ts: content-sniffed type, conversion of BMP to PNG, auto-resize to
3//! the model's limits (Lanczos3, then the smaller of PNG and JPEG under the byte budget, then 3/4 steps), and the
4//! kernel's short description. Photon (WASM) is replaced by the `image` crate, so resized or converted bytes differ
5//! from TypeScript's; dimensions, type choice and limits do not.
6
7use image::codecs::jpeg::JpegEncoder;
8use image::imageops::FilterType;
9use image::{DynamicImage, ImageDecoder, ImageFormat, ImageReader};
10use serde_json::{Map, Value, json};
11use std::io::Cursor;
12
13pub const VIEW_IMAGE_REQUEST: &str = "rlm.view_image";
14pub const MAX_IMAGES_PER_CELL: usize = 8;
15pub const MAX_VIEW_IMAGE_SOURCE_BYTES: u64 = 64 * 1024 * 1024;
16const LOW_DETAIL_MAX_SIDE: u32 = 512;
17const PNG_SIGNATURE: [u8; 8] = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
18
19/// A model's image resize limits (`inputLimits.images.resize`); unset fields take the defaults.
20#[derive(Clone, Copy, Debug, Default)]
21pub struct ResizeLimits {
22    pub max_width: Option<u32>,
23    pub max_height: Option<u32>,
24    pub max_bytes: Option<f64>,
25    pub jpeg_quality: Option<u8>,
26}
27
28/// The model a cell's images go to.
29#[derive(Clone, Debug)]
30pub struct ImageModel {
31    pub provider: String,
32    pub id: String,
33    pub accepts_images: bool,
34    pub resize: Option<ResizeLimits>,
35}
36
37/// Pi's `images.autoResize` setting (default on) and the current model.
38#[derive(Clone, Debug)]
39pub struct ViewImageOptions {
40    pub auto_resize: bool,
41    pub model: Option<ImageModel>,
42}
43
44impl Default for ViewImageOptions {
45    fn default() -> Self {
46        ViewImageOptions {
47            auto_resize: true,
48            model: None,
49        }
50    }
51}
52
53#[derive(Clone, Debug, PartialEq, Eq)]
54pub struct CellImage {
55    pub mime_type: &'static str,
56    pub data: Vec<u8>,
57}
58
59/// The images one cell attached, in call order.
60#[derive(Default)]
61pub struct CellImages {
62    pub images: Vec<CellImage>,
63    pub options: ViewImageOptions,
64}
65
66fn u32_be(b: &[u8], at: usize) -> u64 {
67    (0..4).fold(0, |n, i| (n << 8) | u64::from(*b.get(at + i).unwrap_or(&0)))
68}
69
70fn u32_le(b: &[u8], at: usize) -> u64 {
71    (0..4)
72        .rev()
73        .fold(0, |n, i| (n << 8) | u64::from(*b.get(at + i).unwrap_or(&0)))
74}
75
76fn u16_le(b: &[u8], at: usize) -> u64 {
77    u64::from(*b.get(at).unwrap_or(&0)) | (u64::from(*b.get(at + 1).unwrap_or(&0)) << 8)
78}
79
80fn ascii_at(b: &[u8], at: usize, text: &str) -> bool {
81    b.get(at..at + text.len()) == Some(text.as_bytes())
82}
83
84fn animated_png(b: &[u8]) -> bool {
85    let mut offset = PNG_SIGNATURE.len();
86    while offset + 8 <= b.len() {
87        if ascii_at(b, offset + 4, "acTL") {
88            return true;
89        }
90        if ascii_at(b, offset + 4, "IDAT") {
91            return false;
92        }
93        let next = offset as u64 + 12 + u32_be(b, offset);
94        if next > b.len() as u64 {
95            return false;
96        }
97        offset = next as usize;
98    }
99    false
100}
101
102fn bmp(b: &[u8]) -> bool {
103    if b.len() < 26 {
104        return false;
105    }
106    let (size, pixels, dib) = (u32_le(b, 2), u32_le(b, 10), u32_le(b, 14));
107    if (size != 0 && size < 26) || pixels < 14 + dib || (size != 0 && pixels >= size) {
108        return false;
109    }
110    let (planes, bits) = match dib {
111        12 => (u16_le(b, 22), u16_le(b, 24)),
112        40..=124 if b.len() >= 30 => (u16_le(b, 26), u16_le(b, 28)),
113        _ => return false,
114    };
115    planes == 1 && [1, 4, 8, 16, 24, 32].contains(&bits)
116}
117
118/// The image type by content (utils/mime.ts detectSupportedImageMimeType).
119pub fn detect_image_mime_type(b: &[u8]) -> Option<&'static str> {
120    if b.starts_with(&[0xff, 0xd8, 0xff]) {
121        return (b.get(3) != Some(&0xf7)).then_some("image/jpeg");
122    }
123    if b.starts_with(&PNG_SIGNATURE) {
124        let png = b.len() >= 16 && u32_be(b, 8) == 13 && ascii_at(b, 12, "IHDR");
125        return (png && !animated_png(b)).then_some("image/png");
126    }
127    if ascii_at(b, 0, "GIF87a") || ascii_at(b, 0, "GIF89a") {
128        return Some("image/gif");
129    }
130    if ascii_at(b, 0, "RIFF") && ascii_at(b, 8, "WEBP") {
131        return Some("image/webp");
132    }
133    (ascii_at(b, 0, "BM") && bmp(b)).then_some("image/bmp")
134}
135
136/// Header dimensions for the inline types (png, jpeg, gif, webp); None for others or unreadable headers.
137fn dimensions(bytes: &[u8], mime: &str) -> Option<(u32, u32)> {
138    let format = match mime {
139        "image/png" => ImageFormat::Png,
140        "image/jpeg" => ImageFormat::Jpeg,
141        "image/gif" => ImageFormat::Gif,
142        "image/webp" => ImageFormat::WebP,
143        _ => return None,
144    };
145    ImageReader::with_format(Cursor::new(bytes), format)
146        .into_dimensions()
147        .ok()
148}
149
150/// Decode with the EXIF orientation applied (utils/exif-orientation.ts).
151fn decode(bytes: &[u8]) -> Option<DynamicImage> {
152    let mut decoder = ImageReader::new(Cursor::new(bytes))
153        .with_guessed_format()
154        .ok()?
155        .into_decoder()
156        .ok()?;
157    let orientation = decoder.orientation().ok();
158    let mut image = DynamicImage::from_decoder(decoder).ok()?;
159    if let Some(orientation) = orientation {
160        image.apply_orientation(orientation);
161    }
162    Some(image)
163}
164
165fn encode_png(image: &DynamicImage) -> Option<Vec<u8>> {
166    let mut out = Cursor::new(Vec::new());
167    image.to_rgba8().write_to(&mut out, ImageFormat::Png).ok()?;
168    Some(out.into_inner())
169}
170
171fn encode_jpeg(image: &DynamicImage, quality: u8) -> Option<Vec<u8>> {
172    let mut out = Vec::new();
173    image
174        .to_rgb8()
175        .write_with_encoder(JpegEncoder::new_with_quality(&mut out, quality))
176        .ok()?;
177    Some(out)
178}
179
180fn base64_len(bytes: usize) -> f64 {
181    bytes.div_ceil(3) as f64 * 4.0
182}
183
184/// resizeImageInProcess: None when the image cannot be decoded or brought under `max_bytes`.
185fn resize(bytes: &[u8], mime: &'static str, limits: ResizeLimits) -> Option<CellImage> {
186    let max_width = limits.max_width.unwrap_or(2000);
187    let max_height = limits.max_height.unwrap_or(2000);
188    let max_bytes = limits.max_bytes.unwrap_or(4.5 * 1024.0 * 1024.0);
189    let quality = limits.jpeg_quality.unwrap_or(80);
190    let image = decode(bytes)?;
191    let (width, height) = (image.width(), image.height());
192    if width <= max_width && height <= max_height && base64_len(bytes.len()) < max_bytes {
193        return Some(CellImage {
194            mime_type: mime,
195            data: bytes.to_vec(),
196        });
197    }
198    let (mut w, mut h) = (width, height);
199    if w > max_width {
200        h = jsrt::number::round(f64::from(h) * f64::from(max_width) / f64::from(w)) as u32;
201        w = max_width;
202    }
203    if h > max_height {
204        w = jsrt::number::round(f64::from(w) * f64::from(max_height) / f64::from(h)) as u32;
205        h = max_height;
206    }
207    let mut qualities = vec![quality];
208    for q in [85, 70, 55, 40] {
209        if !qualities.contains(&q) {
210            qualities.push(q);
211        }
212    }
213    loop {
214        let resized = image.resize_exact(w, h, FilterType::Lanczos3);
215        let png = encode_png(&resized)?;
216        if base64_len(png.len()) < max_bytes {
217            return Some(CellImage {
218                mime_type: "image/png",
219                data: png,
220            });
221        }
222        for &q in &qualities {
223            let jpeg = encode_jpeg(&resized, q)?;
224            if base64_len(jpeg.len()) < max_bytes {
225                return Some(CellImage {
226                    mime_type: "image/jpeg",
227                    data: jpeg,
228                });
229            }
230        }
231        if w == 1 && h == 1 {
232            return None;
233        }
234        let next = |side: u32| {
235            if side == 1 {
236                1
237            } else {
238                ((f64::from(side) * 0.75).floor() as u32).max(1)
239            }
240        };
241        let (nw, nh) = (next(w), next(h));
242        if (nw, nh) == (w, h) {
243            return None;
244        }
245        (w, h) = (nw, nh);
246    }
247}
248
249/// `n / unit` with one decimal, as JS `toFixed(1)` rounds it (half up; the quotient is exact).
250fn tenths(n: u64, unit: u64) -> String {
251    let t = (n * 20 + unit) / (2 * unit);
252    format!("{}.{}", t / 10, t % 10)
253}
254
255fn format_bytes(bytes: u64) -> String {
256    if bytes < 1024 {
257        format!("{bytes} B")
258    } else if bytes < 1024 * 1024 {
259        format!("{} KB", tenths(bytes, 1024))
260    } else {
261        format!("{} MB", tenths(bytes, 1024 * 1024))
262    }
263}
264
265fn format_name(mime: &str) -> String {
266    mime.split('/').nth(1).unwrap_or(mime).to_uppercase()
267}
268
269/// Pi's read tool note, said once per cell when the current model takes no image input.
270pub fn non_vision_note(model: Option<&ImageModel>) -> Option<String> {
271    let model = model.filter(|model| !model.accepts_images)?;
272    Some(format!(
273        "[view_image: the current model ({}/{}) does not accept image input, so attached images are omitted from its request. Inspect images with Python instead (for example PIL for size and pixel colors).]",
274        model.provider, model.id
275    ))
276}
277
278impl CellImages {
279    pub fn new(options: ViewImageOptions) -> Self {
280        CellImages {
281            images: Vec::new(),
282            options,
283        }
284    }
285
286    fn limits(&self, detail: Option<&str>) -> ResizeLimits {
287        let base = self.options.model.as_ref().and_then(|model| model.resize);
288        if detail != Some("low") {
289            return base.unwrap_or_default();
290        }
291        let base = base.unwrap_or_default();
292        ResizeLimits {
293            max_width: Some(base.max_width.unwrap_or(LOW_DETAIL_MAX_SIDE).min(LOW_DETAIL_MAX_SIDE)),
294            max_height: Some(base.max_height.unwrap_or(LOW_DETAIL_MAX_SIDE).min(LOW_DETAIL_MAX_SIDE)),
295            ..base
296        }
297    }
298
299    /// Load, normalize and keep one image; the kernel's reply is `{description, note?}`.
300    pub fn attach(&mut self, payload: &Map<String, Value>) -> Result<Value, String> {
301        if self.images.len() >= MAX_IMAGES_PER_CELL {
302            return Err(format!(
303                "view_image: at most {MAX_IMAGES_PER_CELL} images per cell; view the rest in a later cell"
304            ));
305        }
306        let path = match payload.get("path") {
307            Some(Value::String(path)) if !path.is_empty() => path,
308            _ => return Err("view_image: path must be a non-empty string".into()),
309        };
310        let size = std::fs::metadata(path)
311            .map_err(|error| jsrt::fs::error_message(&error, "stat", std::path::Path::new(path)))?
312            .len();
313        if size > MAX_VIEW_IMAGE_SOURCE_BYTES {
314            return Err(format!(
315                "view_image: {} is over the {} limit",
316                format_bytes(size),
317                format_bytes(MAX_VIEW_IMAGE_SOURCE_BYTES)
318            ));
319        }
320        let bytes = std::fs::read(path).map_err(|error| error.to_string())?;
321        let mime = detect_image_mime_type(&bytes)
322            .ok_or("view_image: not a supported image (png, jpeg, gif or webp, detected by content)")?;
323        let original = dimensions(&bytes, mime);
324        let detail = payload.get("detail").and_then(Value::as_str);
325        // normalizeImage: BMP becomes PNG first.
326        let normalized = if mime == "image/bmp" {
327            let png = decode(&bytes)
328                .and_then(|image| encode_png(&image))
329                .ok_or("view_image: Image omitted: could not be converted to a supported inline image format.")?;
330            CellImage {
331                mime_type: "image/png",
332                data: png,
333            }
334        } else {
335            CellImage {
336                mime_type: mime,
337                data: bytes,
338            }
339        };
340        let processed = if self.options.auto_resize {
341            resize(&normalized.data, normalized.mime_type, self.limits(detail))
342                .ok_or("view_image: Image omitted: could not be resized below the inline image size limit.")?
343        } else {
344            normalized
345        };
346        let shown = dimensions(&processed.data, processed.mime_type);
347        let mut description = format!(
348            "image {}: {}{}, {}",
349            self.images.len() + 1,
350            shown.map_or(String::new(), |(w, h)| format!("{w}x{h} ")),
351            format_name(processed.mime_type),
352            format_bytes(processed.data.len() as u64)
353        );
354        match (original, shown) {
355            (Some(original), Some(shown)) if original != shown => {
356                description.push_str(&format!(" (downscaled from {}x{})", original.0, original.1));
357            }
358            _ if processed.mime_type != mime => {
359                description.push_str(&format!(" (converted from {})", format_name(mime)));
360            }
361            _ => {}
362        }
363        self.images.push(processed);
364        Ok(match non_vision_note(self.options.model.as_ref()) {
365            Some(note) => json!({ "description": description, "note": note }),
366            None => json!({ "description": description }),
367        })
368    }
369
370    /// The tool result text for this cell: the non-vision note follows the text when images are attached.
371    pub fn text(&self, text: &str) -> String {
372        match non_vision_note(self.options.model.as_ref()) {
373            Some(note) if !self.images.is_empty() => format!("{text}\n{note}"),
374            _ => text.to_owned(),
375        }
376    }
377}
378
379#[cfg(test)]
380mod tests {
381    /// Literal outputs of the TS formatBytes (`toFixed(1)` rounds the exact quotient half up).
382    #[test]
383    fn host_view_image_formats_bytes_like_js_to_fixed() {
384        let got: Vec<String> = [1023, 1024, 1280, 1382, 1587, 2611, 1_310_720, 67_108_864]
385            .into_iter()
386            .map(super::format_bytes)
387            .collect();
388        assert_eq!(
389            got,
390            [
391                "1023 B", "1.0 KB", "1.3 KB", "1.3 KB", "1.5 KB", "2.5 KB", "1.3 MB", "64.0 MB"
392            ]
393        );
394    }
395}