1use image::codecs::jpeg::JpegEncoder;
8use image::imageops::FilterType;
9use image::{DynamicImage, ImageDecoder, ImageFormat, ImageReader};
10use serde_json::{Map, Value, json};
11use std::io::Cursor;
12
13pub const VIEW_IMAGE_REQUEST: &str = "rlm.view_image";
14pub const MAX_IMAGES_PER_CELL: usize = 8;
15pub const MAX_VIEW_IMAGE_SOURCE_BYTES: u64 = 64 * 1024 * 1024;
16const LOW_DETAIL_MAX_SIDE: u32 = 512;
17const PNG_SIGNATURE: [u8; 8] = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
18
19#[derive(Clone, Copy, Debug, Default)]
21pub struct ResizeLimits {
22 pub max_width: Option<u32>,
23 pub max_height: Option<u32>,
24 pub max_bytes: Option<f64>,
25 pub jpeg_quality: Option<u8>,
26}
27
28#[derive(Clone, Debug)]
30pub struct ImageModel {
31 pub provider: String,
32 pub id: String,
33 pub accepts_images: bool,
34 pub resize: Option<ResizeLimits>,
35}
36
37#[derive(Clone, Debug)]
39pub struct ViewImageOptions {
40 pub auto_resize: bool,
41 pub model: Option<ImageModel>,
42}
43
44impl Default for ViewImageOptions {
45 fn default() -> Self {
46 ViewImageOptions {
47 auto_resize: true,
48 model: None,
49 }
50 }
51}
52
53#[derive(Clone, Debug, PartialEq, Eq)]
54pub struct CellImage {
55 pub mime_type: &'static str,
56 pub data: Vec<u8>,
57}
58
59#[derive(Default)]
61pub struct CellImages {
62 pub images: Vec<CellImage>,
63 pub options: ViewImageOptions,
64}
65
66fn u32_be(b: &[u8], at: usize) -> u64 {
67 (0..4).fold(0, |n, i| (n << 8) | u64::from(*b.get(at + i).unwrap_or(&0)))
68}
69
70fn u32_le(b: &[u8], at: usize) -> u64 {
71 (0..4)
72 .rev()
73 .fold(0, |n, i| (n << 8) | u64::from(*b.get(at + i).unwrap_or(&0)))
74}
75
76fn u16_le(b: &[u8], at: usize) -> u64 {
77 u64::from(*b.get(at).unwrap_or(&0)) | (u64::from(*b.get(at + 1).unwrap_or(&0)) << 8)
78}
79
80fn ascii_at(b: &[u8], at: usize, text: &str) -> bool {
81 b.get(at..at + text.len()) == Some(text.as_bytes())
82}
83
84fn animated_png(b: &[u8]) -> bool {
85 let mut offset = PNG_SIGNATURE.len();
86 while offset + 8 <= b.len() {
87 if ascii_at(b, offset + 4, "acTL") {
88 return true;
89 }
90 if ascii_at(b, offset + 4, "IDAT") {
91 return false;
92 }
93 let next = offset as u64 + 12 + u32_be(b, offset);
94 if next > b.len() as u64 {
95 return false;
96 }
97 offset = next as usize;
98 }
99 false
100}
101
102fn bmp(b: &[u8]) -> bool {
103 if b.len() < 26 {
104 return false;
105 }
106 let (size, pixels, dib) = (u32_le(b, 2), u32_le(b, 10), u32_le(b, 14));
107 if (size != 0 && size < 26) || pixels < 14 + dib || (size != 0 && pixels >= size) {
108 return false;
109 }
110 let (planes, bits) = match dib {
111 12 => (u16_le(b, 22), u16_le(b, 24)),
112 40..=124 if b.len() >= 30 => (u16_le(b, 26), u16_le(b, 28)),
113 _ => return false,
114 };
115 planes == 1 && [1, 4, 8, 16, 24, 32].contains(&bits)
116}
117
118pub fn detect_image_mime_type(b: &[u8]) -> Option<&'static str> {
120 if b.starts_with(&[0xff, 0xd8, 0xff]) {
121 return (b.get(3) != Some(&0xf7)).then_some("image/jpeg");
122 }
123 if b.starts_with(&PNG_SIGNATURE) {
124 let png = b.len() >= 16 && u32_be(b, 8) == 13 && ascii_at(b, 12, "IHDR");
125 return (png && !animated_png(b)).then_some("image/png");
126 }
127 if ascii_at(b, 0, "GIF87a") || ascii_at(b, 0, "GIF89a") {
128 return Some("image/gif");
129 }
130 if ascii_at(b, 0, "RIFF") && ascii_at(b, 8, "WEBP") {
131 return Some("image/webp");
132 }
133 (ascii_at(b, 0, "BM") && bmp(b)).then_some("image/bmp")
134}
135
136fn dimensions(bytes: &[u8], mime: &str) -> Option<(u32, u32)> {
138 let format = match mime {
139 "image/png" => ImageFormat::Png,
140 "image/jpeg" => ImageFormat::Jpeg,
141 "image/gif" => ImageFormat::Gif,
142 "image/webp" => ImageFormat::WebP,
143 _ => return None,
144 };
145 ImageReader::with_format(Cursor::new(bytes), format)
146 .into_dimensions()
147 .ok()
148}
149
150fn decode(bytes: &[u8]) -> Option<DynamicImage> {
152 let mut decoder = ImageReader::new(Cursor::new(bytes))
153 .with_guessed_format()
154 .ok()?
155 .into_decoder()
156 .ok()?;
157 let orientation = decoder.orientation().ok();
158 let mut image = DynamicImage::from_decoder(decoder).ok()?;
159 if let Some(orientation) = orientation {
160 image.apply_orientation(orientation);
161 }
162 Some(image)
163}
164
165fn encode_png(image: &DynamicImage) -> Option<Vec<u8>> {
166 let mut out = Cursor::new(Vec::new());
167 image.to_rgba8().write_to(&mut out, ImageFormat::Png).ok()?;
168 Some(out.into_inner())
169}
170
171fn encode_jpeg(image: &DynamicImage, quality: u8) -> Option<Vec<u8>> {
172 let mut out = Vec::new();
173 image
174 .to_rgb8()
175 .write_with_encoder(JpegEncoder::new_with_quality(&mut out, quality))
176 .ok()?;
177 Some(out)
178}
179
180fn base64_len(bytes: usize) -> f64 {
181 bytes.div_ceil(3) as f64 * 4.0
182}
183
184fn resize(bytes: &[u8], mime: &'static str, limits: ResizeLimits) -> Option<CellImage> {
186 let max_width = limits.max_width.unwrap_or(2000);
187 let max_height = limits.max_height.unwrap_or(2000);
188 let max_bytes = limits.max_bytes.unwrap_or(4.5 * 1024.0 * 1024.0);
189 let quality = limits.jpeg_quality.unwrap_or(80);
190 let image = decode(bytes)?;
191 let (width, height) = (image.width(), image.height());
192 if width <= max_width && height <= max_height && base64_len(bytes.len()) < max_bytes {
193 return Some(CellImage {
194 mime_type: mime,
195 data: bytes.to_vec(),
196 });
197 }
198 let (mut w, mut h) = (width, height);
199 if w > max_width {
200 h = jsrt::number::round(f64::from(h) * f64::from(max_width) / f64::from(w)) as u32;
201 w = max_width;
202 }
203 if h > max_height {
204 w = jsrt::number::round(f64::from(w) * f64::from(max_height) / f64::from(h)) as u32;
205 h = max_height;
206 }
207 let mut qualities = vec![quality];
208 for q in [85, 70, 55, 40] {
209 if !qualities.contains(&q) {
210 qualities.push(q);
211 }
212 }
213 loop {
214 let resized = image.resize_exact(w, h, FilterType::Lanczos3);
215 let png = encode_png(&resized)?;
216 if base64_len(png.len()) < max_bytes {
217 return Some(CellImage {
218 mime_type: "image/png",
219 data: png,
220 });
221 }
222 for &q in &qualities {
223 let jpeg = encode_jpeg(&resized, q)?;
224 if base64_len(jpeg.len()) < max_bytes {
225 return Some(CellImage {
226 mime_type: "image/jpeg",
227 data: jpeg,
228 });
229 }
230 }
231 if w == 1 && h == 1 {
232 return None;
233 }
234 let next = |side: u32| {
235 if side == 1 {
236 1
237 } else {
238 ((f64::from(side) * 0.75).floor() as u32).max(1)
239 }
240 };
241 let (nw, nh) = (next(w), next(h));
242 if (nw, nh) == (w, h) {
243 return None;
244 }
245 (w, h) = (nw, nh);
246 }
247}
248
249fn tenths(n: u64, unit: u64) -> String {
251 let t = (n * 20 + unit) / (2 * unit);
252 format!("{}.{}", t / 10, t % 10)
253}
254
255fn format_bytes(bytes: u64) -> String {
256 if bytes < 1024 {
257 format!("{bytes} B")
258 } else if bytes < 1024 * 1024 {
259 format!("{} KB", tenths(bytes, 1024))
260 } else {
261 format!("{} MB", tenths(bytes, 1024 * 1024))
262 }
263}
264
265fn format_name(mime: &str) -> String {
266 mime.split('/').nth(1).unwrap_or(mime).to_uppercase()
267}
268
269pub fn non_vision_note(model: Option<&ImageModel>) -> Option<String> {
271 let model = model.filter(|model| !model.accepts_images)?;
272 Some(format!(
273 "[view_image: the current model ({}/{}) does not accept image input, so attached images are omitted from its request. Inspect images with Python instead (for example PIL for size and pixel colors).]",
274 model.provider, model.id
275 ))
276}
277
278impl CellImages {
279 pub fn new(options: ViewImageOptions) -> Self {
280 CellImages {
281 images: Vec::new(),
282 options,
283 }
284 }
285
286 fn limits(&self, detail: Option<&str>) -> ResizeLimits {
287 let base = self.options.model.as_ref().and_then(|model| model.resize);
288 if detail != Some("low") {
289 return base.unwrap_or_default();
290 }
291 let base = base.unwrap_or_default();
292 ResizeLimits {
293 max_width: Some(base.max_width.unwrap_or(LOW_DETAIL_MAX_SIDE).min(LOW_DETAIL_MAX_SIDE)),
294 max_height: Some(base.max_height.unwrap_or(LOW_DETAIL_MAX_SIDE).min(LOW_DETAIL_MAX_SIDE)),
295 ..base
296 }
297 }
298
299 pub fn attach(&mut self, payload: &Map<String, Value>) -> Result<Value, String> {
301 if self.images.len() >= MAX_IMAGES_PER_CELL {
302 return Err(format!(
303 "view_image: at most {MAX_IMAGES_PER_CELL} images per cell; view the rest in a later cell"
304 ));
305 }
306 let path = match payload.get("path") {
307 Some(Value::String(path)) if !path.is_empty() => path,
308 _ => return Err("view_image: path must be a non-empty string".into()),
309 };
310 let size = std::fs::metadata(path)
311 .map_err(|error| jsrt::fs::error_message(&error, "stat", std::path::Path::new(path)))?
312 .len();
313 if size > MAX_VIEW_IMAGE_SOURCE_BYTES {
314 return Err(format!(
315 "view_image: {} is over the {} limit",
316 format_bytes(size),
317 format_bytes(MAX_VIEW_IMAGE_SOURCE_BYTES)
318 ));
319 }
320 let bytes = std::fs::read(path).map_err(|error| error.to_string())?;
321 let mime = detect_image_mime_type(&bytes)
322 .ok_or("view_image: not a supported image (png, jpeg, gif or webp, detected by content)")?;
323 let original = dimensions(&bytes, mime);
324 let detail = payload.get("detail").and_then(Value::as_str);
325 let normalized = if mime == "image/bmp" {
327 let png = decode(&bytes)
328 .and_then(|image| encode_png(&image))
329 .ok_or("view_image: Image omitted: could not be converted to a supported inline image format.")?;
330 CellImage {
331 mime_type: "image/png",
332 data: png,
333 }
334 } else {
335 CellImage {
336 mime_type: mime,
337 data: bytes,
338 }
339 };
340 let processed = if self.options.auto_resize {
341 resize(&normalized.data, normalized.mime_type, self.limits(detail))
342 .ok_or("view_image: Image omitted: could not be resized below the inline image size limit.")?
343 } else {
344 normalized
345 };
346 let shown = dimensions(&processed.data, processed.mime_type);
347 let mut description = format!(
348 "image {}: {}{}, {}",
349 self.images.len() + 1,
350 shown.map_or(String::new(), |(w, h)| format!("{w}x{h} ")),
351 format_name(processed.mime_type),
352 format_bytes(processed.data.len() as u64)
353 );
354 match (original, shown) {
355 (Some(original), Some(shown)) if original != shown => {
356 description.push_str(&format!(" (downscaled from {}x{})", original.0, original.1));
357 }
358 _ if processed.mime_type != mime => {
359 description.push_str(&format!(" (converted from {})", format_name(mime)));
360 }
361 _ => {}
362 }
363 self.images.push(processed);
364 Ok(match non_vision_note(self.options.model.as_ref()) {
365 Some(note) => json!({ "description": description, "note": note }),
366 None => json!({ "description": description }),
367 })
368 }
369
370 pub fn text(&self, text: &str) -> String {
372 match non_vision_note(self.options.model.as_ref()) {
373 Some(note) if !self.images.is_empty() => format!("{text}\n{note}"),
374 _ => text.to_owned(),
375 }
376 }
377}
378
379#[cfg(test)]
380mod tests {
381 #[test]
383 fn host_view_image_formats_bytes_like_js_to_fixed() {
384 let got: Vec<String> = [1023, 1024, 1280, 1382, 1587, 2611, 1_310_720, 67_108_864]
385 .into_iter()
386 .map(super::format_bytes)
387 .collect();
388 assert_eq!(
389 got,
390 [
391 "1023 B", "1.0 KB", "1.3 KB", "1.3 KB", "1.5 KB", "2.5 KB", "1.3 MB", "64.0 MB"
392 ]
393 );
394 }
395}