use ai::Model;
use host::view_image::{CellImages, ImageModel, ResizeLimits, VIEW_IMAGE_REQUEST, ViewImageOptions};
use jsrt::sync;
use kernel::{HostHandler, Signal};
use serde_json::{Map, Value};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
pub const CONTEXT_IMAGES_ENV: &str = "ULTRON_CONTEXT_IMAGES";
pub fn image_model(model: &Model) -> ImageModel {
let resize = model
.input_limits
.as_ref()
.and_then(|limits| limits["images"]["resize"].as_object().cloned());
ImageModel {
provider: model.provider.clone(),
id: model.id.clone(),
accepts_images: model.input.iter().any(|i| i == "image"),
resize: resize.map(|r| ResizeLimits {
max_width: r.get("maxWidth").and_then(Value::as_u64).map(|n| n as u32),
max_height: r.get("maxHeight").and_then(Value::as_u64).map(|n| n as u32),
max_bytes: r.get("maxBytes").and_then(Value::as_f64),
jpeg_quality: r.get("jpegQuality").and_then(Value::as_u64).map(|n| n as u8),
}),
}
}
pub struct ViewImages {
cells: Mutex<HashMap<String, CellImages>>,
display: Mutex<HashMap<String, Vec<DisplayImage>>>,
models: Mutex<HashMap<String, Model>>,
default_model: Mutex<Model>,
cwd: PathBuf,
agent_dir: PathBuf,
project_trusted: bool,
}
impl ViewImages {
pub fn new(model: Model, cwd: PathBuf, agent_dir: PathBuf, project_trusted: bool) -> ViewImages {
ViewImages {
cells: Mutex::default(),
display: Mutex::default(),
models: Mutex::default(),
default_model: Mutex::new(model),
cwd,
agent_dir,
project_trusted,
}
}
pub fn set_model(&self, lane: Option<&str>, model: Model) {
match lane {
Some(lane) => {
sync::lock(&self.models).insert(lane.to_string(), model);
}
None => *sync::lock(&self.default_model) = model,
}
}
fn auto_resize(&self) -> bool {
let settings = config::settings::SettingsManager::create(&self.cwd, &self.agent_dir, self.project_trusted);
settings.settings().get("images").and_then(|i| i.get("autoResize")) != Some(&Value::Bool(false))
}
fn options(&self, lane: &str) -> ViewImageOptions {
let model = sync::lock(&self.models)
.get(lane)
.cloned()
.unwrap_or_else(|| sync::lock(&self.default_model).clone());
ViewImageOptions {
auto_resize: self.auto_resize(),
model: Some(image_model(&model)),
}
}
pub fn begin(&self, lane: &str) {
let options = self.options(lane);
sync::lock(&self.cells).insert(lane.to_string(), CellImages::new(options));
sync::lock(&self.display).remove(lane);
}
pub fn display(&self, lane: &str, png: Vec<u8>, line: String) {
sync::lock(&self.display)
.entry(lane.to_string())
.or_default()
.push(DisplayImage { png, line });
}
pub fn take_display(&self, lane: &str) -> Vec<DisplayImage> {
sync::lock(&self.display).remove(lane).unwrap_or_default()
}
pub fn attach(&self, lane: &str, payload: &Map<String, Value>) -> Result<Value, String> {
if !sync::lock(&self.cells).contains_key(lane) {
self.begin(lane);
}
let mut cells = sync::lock(&self.cells);
let images = cells.get_mut(lane).expect("begun");
images.attach(payload)
}
pub fn take(&self, lane: &str) -> Option<CellImages> {
sync::lock(&self.cells).remove(lane)
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DisplayImage {
pub png: Vec<u8>,
pub line: String,
}
pub const MAX_DISPLAY_IMAGE_BYTES: usize = 4 * 1024 * 1024;
pub fn with_display_lines(text: String, shown: &[DisplayImage]) -> String {
shown.iter().fold(text, |text, image| format!("{text}\n{}", image.line))
}
pub fn with_display_details(mut details: Value, shown: &[DisplayImage]) -> Value {
let blocks: Vec<Value> = shown
.iter()
.filter(|image| image.png.len() <= MAX_DISPLAY_IMAGE_BYTES)
.map(|image| {
serde_json::json!({
"type": "image",
"data": jsrt::base64::encode(&image.png),
"mimeType": "image/png",
})
})
.collect();
if blocks.is_empty() {
return details;
}
if !details.is_object() {
details = serde_json::json!({});
}
details["displayImages"] = Value::Array(blocks);
details
}
pub fn cell_content(images: Option<&CellImages>, text: String) -> Vec<ai::UserContent> {
let Some(images) = images.filter(|i| !i.images.is_empty()) else {
return vec![crate::lane::text_block(text)];
};
let mut content = vec![crate::lane::text_block(images.text(&text))];
content.extend(images.images.iter().map(|image| {
ai::UserContent::Image(ai::ImageContent {
data: jsrt::base64::encode(&image.data),
mime_type: image.mime_type.to_string(),
})
}));
content
}
pub struct ViewImageLayer {
pub inner: Arc<dyn HostHandler>,
pub images: Arc<ViewImages>,
pub lane: String,
}
impl HostHandler for ViewImageLayer {
fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
if kind == VIEW_IMAGE_REQUEST {
return self.images.attach(&self.lane, &payload);
}
self.inner.handle(kind, payload, signal)
}
}
pub const DEFAULT_CONTEXT_IMAGES: usize = 8;
pub const OFFLOADED_IMAGE_TEXT: &str =
"[image omitted from context to stay within the image budget; call view_image again to see it]";
pub fn context_image_budget(value: Option<&str>) -> usize {
let raw = value.map(jsrt::string::trim).unwrap_or_default();
if raw.is_empty() {
return DEFAULT_CONTEXT_IMAGES;
}
match jsrt::number::parse(raw) {
n if n.is_finite() && n >= 0.0 && n.fract() == 0.0 => n as usize,
_ => DEFAULT_CONTEXT_IMAGES,
}
}
pub fn images_to_drop(total: usize, max: usize) -> usize {
if max == 0 || total <= max {
return 0;
}
let step = (max / 2).max(1);
total.min((total - step) / step * step)
}
pub fn with_image_budget(messages: &[ai::Message], max: usize) -> Option<Vec<ai::Message>> {
let is_image = |c: &ai::UserContent| matches!(c, ai::UserContent::Image(_));
let total: usize = messages
.iter()
.map(|m| match m {
ai::Message::ToolResult(result) => result.content.iter().filter(|c| is_image(c)).count(),
_ => 0,
})
.sum();
let mut drop = images_to_drop(total, max);
if drop == 0 {
return None;
}
Some(
messages
.iter()
.map(|message| match message {
ai::Message::ToolResult(result) if drop > 0 && result.content.iter().any(is_image) => {
let mut result = result.clone();
for part in &mut result.content {
if drop > 0 && is_image(part) {
drop -= 1;
*part = crate::lane::text_block(OFFLOADED_IMAGE_TEXT.to_string());
}
}
ai::Message::ToolResult(result)
}
other => other.clone(),
})
.collect(),
)
}
pub fn apply_image_budget(messages: &mut Vec<ai::Message>, max: usize) {
if max == 0 {
return;
}
if let Some(kept) = with_image_budget(messages, max) {
*messages = kept;
}
}