1use ai::Model;
8use host::view_image::{CellImages, ImageModel, ResizeLimits, VIEW_IMAGE_REQUEST, ViewImageOptions};
9use jsrt::sync;
10use kernel::{HostHandler, Signal};
11use serde_json::{Map, Value};
12use std::collections::HashMap;
13use std::path::PathBuf;
14use std::sync::{Arc, Mutex};
15
16pub const CONTEXT_IMAGES_ENV: &str = "ULTRON_CONTEXT_IMAGES";
17
18pub fn image_model(model: &Model) -> ImageModel {
20 let resize = model
21 .input_limits
22 .as_ref()
23 .and_then(|limits| limits["images"]["resize"].as_object().cloned());
24 ImageModel {
25 provider: model.provider.clone(),
26 id: model.id.clone(),
27 accepts_images: model.input.iter().any(|i| i == "image"),
28 resize: resize.map(|r| ResizeLimits {
29 max_width: r.get("maxWidth").and_then(Value::as_u64).map(|n| n as u32),
30 max_height: r.get("maxHeight").and_then(Value::as_u64).map(|n| n as u32),
31 max_bytes: r.get("maxBytes").and_then(Value::as_f64),
32 jpeg_quality: r.get("jpegQuality").and_then(Value::as_u64).map(|n| n as u8),
33 }),
34 }
35}
36
37pub struct ViewImages {
39 cells: Mutex<HashMap<String, CellImages>>,
40 display: Mutex<HashMap<String, Vec<DisplayImage>>>,
42 models: Mutex<HashMap<String, Model>>,
43 default_model: Mutex<Model>,
45 cwd: PathBuf,
47 agent_dir: PathBuf,
48 project_trusted: bool,
50}
51
52impl ViewImages {
53 pub fn new(model: Model, cwd: PathBuf, agent_dir: PathBuf, project_trusted: bool) -> ViewImages {
54 ViewImages {
55 cells: Mutex::default(),
56 display: Mutex::default(),
57 models: Mutex::default(),
58 default_model: Mutex::new(model),
59 cwd,
60 agent_dir,
61 project_trusted,
62 }
63 }
64
65 pub fn set_model(&self, lane: Option<&str>, model: Model) {
67 match lane {
68 Some(lane) => {
69 sync::lock(&self.models).insert(lane.to_string(), model);
70 }
71 None => *sync::lock(&self.default_model) = model,
72 }
73 }
74
75 fn auto_resize(&self) -> bool {
77 let settings = config::settings::SettingsManager::create(&self.cwd, &self.agent_dir, self.project_trusted);
78 settings.settings().get("images").and_then(|i| i.get("autoResize")) != Some(&Value::Bool(false))
79 }
80
81 fn options(&self, lane: &str) -> ViewImageOptions {
82 let model = sync::lock(&self.models)
83 .get(lane)
84 .cloned()
85 .unwrap_or_else(|| sync::lock(&self.default_model).clone());
86 ViewImageOptions {
87 auto_resize: self.auto_resize(),
88 model: Some(image_model(&model)),
89 }
90 }
91
92 pub fn begin(&self, lane: &str) {
94 let options = self.options(lane);
95 sync::lock(&self.cells).insert(lane.to_string(), CellImages::new(options));
96 sync::lock(&self.display).remove(lane);
97 }
98
99 pub fn display(&self, lane: &str, png: Vec<u8>, line: String) {
101 sync::lock(&self.display)
102 .entry(lane.to_string())
103 .or_default()
104 .push(DisplayImage { png, line });
105 }
106
107 pub fn take_display(&self, lane: &str) -> Vec<DisplayImage> {
109 sync::lock(&self.display).remove(lane).unwrap_or_default()
110 }
111
112 pub fn attach(&self, lane: &str, payload: &Map<String, Value>) -> Result<Value, String> {
114 if !sync::lock(&self.cells).contains_key(lane) {
115 self.begin(lane);
116 }
117 let mut cells = sync::lock(&self.cells);
118 let images = cells.get_mut(lane).expect("begun");
119 images.attach(payload)
120 }
121
122 pub fn take(&self, lane: &str) -> Option<CellImages> {
124 sync::lock(&self.cells).remove(lane)
125 }
126}
127
128#[derive(Clone, Debug, PartialEq, Eq)]
131pub struct DisplayImage {
132 pub png: Vec<u8>,
133 pub line: String,
134}
135
136pub const MAX_DISPLAY_IMAGE_BYTES: usize = 4 * 1024 * 1024;
138
139pub fn with_display_lines(text: String, shown: &[DisplayImage]) -> String {
141 shown.iter().fold(text, |text, image| format!("{text}\n{}", image.line))
142}
143
144pub fn with_display_details(mut details: Value, shown: &[DisplayImage]) -> Value {
147 let blocks: Vec<Value> = shown
148 .iter()
149 .filter(|image| image.png.len() <= MAX_DISPLAY_IMAGE_BYTES)
150 .map(|image| {
151 serde_json::json!({
152 "type": "image",
153 "data": jsrt::base64::encode(&image.png),
154 "mimeType": "image/png",
155 })
156 })
157 .collect();
158 if blocks.is_empty() {
159 return details;
160 }
161 if !details.is_object() {
162 details = serde_json::json!({});
163 }
164 details["displayImages"] = Value::Array(blocks);
165 details
166}
167
168pub fn cell_content(images: Option<&CellImages>, text: String) -> Vec<ai::UserContent> {
171 let Some(images) = images.filter(|i| !i.images.is_empty()) else {
172 return vec![crate::lane::text_block(text)];
173 };
174 let mut content = vec![crate::lane::text_block(images.text(&text))];
175 content.extend(images.images.iter().map(|image| {
176 ai::UserContent::Image(ai::ImageContent {
177 data: jsrt::base64::encode(&image.data),
178 mime_type: image.mime_type.to_string(),
179 })
180 }));
181 content
182}
183
184pub struct ViewImageLayer {
186 pub inner: Arc<dyn HostHandler>,
187 pub images: Arc<ViewImages>,
188 pub lane: String,
189}
190
191impl HostHandler for ViewImageLayer {
192 fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
193 if kind == VIEW_IMAGE_REQUEST {
194 return self.images.attach(&self.lane, &payload);
195 }
196 self.inner.handle(kind, payload, signal)
197 }
198}
199
200pub const DEFAULT_CONTEXT_IMAGES: usize = 8;
201pub const OFFLOADED_IMAGE_TEXT: &str =
202 "[image omitted from context to stay within the image budget; call view_image again to see it]";
203
204pub fn context_image_budget(value: Option<&str>) -> usize {
206 let raw = value.map(jsrt::string::trim).unwrap_or_default();
207 if raw.is_empty() {
208 return DEFAULT_CONTEXT_IMAGES;
209 }
210 match jsrt::number::parse(raw) {
213 n if n.is_finite() && n >= 0.0 && n.fract() == 0.0 => n as usize,
214 _ => DEFAULT_CONTEXT_IMAGES,
215 }
216}
217
218pub fn images_to_drop(total: usize, max: usize) -> usize {
220 if max == 0 || total <= max {
221 return 0;
222 }
223 let step = (max / 2).max(1);
224 total.min((total - step) / step * step)
225}
226
227pub fn with_image_budget(messages: &[ai::Message], max: usize) -> Option<Vec<ai::Message>> {
230 let is_image = |c: &ai::UserContent| matches!(c, ai::UserContent::Image(_));
231 let total: usize = messages
232 .iter()
233 .map(|m| match m {
234 ai::Message::ToolResult(result) => result.content.iter().filter(|c| is_image(c)).count(),
235 _ => 0,
236 })
237 .sum();
238 let mut drop = images_to_drop(total, max);
239 if drop == 0 {
240 return None;
241 }
242 Some(
243 messages
244 .iter()
245 .map(|message| match message {
246 ai::Message::ToolResult(result) if drop > 0 && result.content.iter().any(is_image) => {
247 let mut result = result.clone();
248 for part in &mut result.content {
249 if drop > 0 && is_image(part) {
250 drop -= 1;
251 *part = crate::lane::text_block(OFFLOADED_IMAGE_TEXT.to_string());
252 }
253 }
254 ai::Message::ToolResult(result)
255 }
256 other => other.clone(),
257 })
258 .collect(),
259 )
260}
261
262pub fn apply_image_budget(messages: &mut Vec<ai::Message>, max: usize) {
264 if max == 0 {
265 return;
266 }
267 if let Some(kept) = with_image_budget(messages, max) {
268 *messages = kept;
269 }
270}