Skip to main content

session_core/
view_images.rs

1//! `view_image` in the session (`ultron/rlm/view-image.ts` as session-worker.ts wires it): a cell's
2//! `rlm.view_image` requests are answered by the cell's own image collector ([`host::view_image::CellImages`],
3//! normalized for the lane's current model and Pi's `images.autoResize` setting), and the `rlm` tool result carries the
4//! images as image blocks after its text. The image budget (`ultron/image-budget.ts`, `ULTRON_CONTEXT_IMAGES`) keeps
5//! the oldest tool-result images of a request out of it as a note.
6
7use ai::Model;
8use host::view_image::{CellImages, ImageModel, ResizeLimits, VIEW_IMAGE_REQUEST, ViewImageOptions};
9use jsrt::sync;
10use kernel::{HostHandler, Signal};
11use serde_json::{Map, Value};
12use std::collections::HashMap;
13use std::path::PathBuf;
14use std::sync::{Arc, Mutex};
15
16pub const CONTEXT_IMAGES_ENV: &str = "ULTRON_CONTEXT_IMAGES";
17
18/// The model a lane's images go to, as the host's normalizer reads it (`input`, `inputLimits.images.resize`).
19pub fn image_model(model: &Model) -> ImageModel {
20    let resize = model
21        .input_limits
22        .as_ref()
23        .and_then(|limits| limits["images"]["resize"].as_object().cloned());
24    ImageModel {
25        provider: model.provider.clone(),
26        id: model.id.clone(),
27        accepts_images: model.input.iter().any(|i| i == "image"),
28        resize: resize.map(|r| ResizeLimits {
29            max_width: r.get("maxWidth").and_then(Value::as_u64).map(|n| n as u32),
30            max_height: r.get("maxHeight").and_then(Value::as_u64).map(|n| n as u32),
31            max_bytes: r.get("maxBytes").and_then(Value::as_f64),
32            jpeg_quality: r.get("jpegQuality").and_then(Value::as_u64).map(|n| n as u8),
33        }),
34    }
35}
36
37/// Every lane's images of the cell it runs now, and the model each lane's results go to.
38pub struct ViewImages {
39    cells: Mutex<HashMap<String, CellImages>>,
40    /// Images each lane's running cell shows the user but not the model (`diagram.mermaid` without `to_model`).
41    display: Mutex<HashMap<String, Vec<DisplayImage>>>,
42    models: Mutex<HashMap<String, Model>>,
43    /// The session's model: a lane without a model of its own sends its results there.
44    default_model: Mutex<Model>,
45    /// Where `images.autoResize` is read (per cell, as TypeScript reads it).
46    cwd: PathBuf,
47    agent_dir: PathBuf,
48    /// The worker's project trust: a trusted project's `.pi/settings.json` is read too.
49    project_trusted: bool,
50}
51
52impl ViewImages {
53    pub fn new(model: Model, cwd: PathBuf, agent_dir: PathBuf, project_trusted: bool) -> ViewImages {
54        ViewImages {
55            cells: Mutex::default(),
56            display: Mutex::default(),
57            models: Mutex::default(),
58            default_model: Mutex::new(model),
59            cwd,
60            agent_dir,
61            project_trusted,
62        }
63    }
64
65    /// The model `lane`'s results go to from now on (`None`: the root lane, whose model is the session's).
66    pub fn set_model(&self, lane: Option<&str>, model: Model) {
67        match lane {
68            Some(lane) => {
69                sync::lock(&self.models).insert(lane.to_string(), model);
70            }
71            None => *sync::lock(&self.default_model) = model,
72        }
73    }
74
75    /// Pi's `images.autoResize` (default on).
76    fn auto_resize(&self) -> bool {
77        let settings = config::settings::SettingsManager::create(&self.cwd, &self.agent_dir, self.project_trusted);
78        settings.settings().get("images").and_then(|i| i.get("autoResize")) != Some(&Value::Bool(false))
79    }
80
81    fn options(&self, lane: &str) -> ViewImageOptions {
82        let model = sync::lock(&self.models)
83            .get(lane)
84            .cloned()
85            .unwrap_or_else(|| sync::lock(&self.default_model).clone());
86        ViewImageOptions {
87            auto_resize: self.auto_resize(),
88            model: Some(image_model(&model)),
89        }
90    }
91
92    /// A cell starts on `lane`: its images start empty.
93    pub fn begin(&self, lane: &str) {
94        let options = self.options(lane);
95        sync::lock(&self.cells).insert(lane.to_string(), CellImages::new(options));
96        sync::lock(&self.display).remove(lane);
97    }
98
99    /// A PNG `lane`'s running cell shows the user only; `line` is what the model reads instead.
100    pub fn display(&self, lane: &str, png: Vec<u8>, line: String) {
101        sync::lock(&self.display)
102            .entry(lane.to_string())
103            .or_default()
104            .push(DisplayImage { png, line });
105    }
106
107    /// The display-only images of `lane`'s cell (its cell ended).
108    pub fn take_display(&self, lane: &str) -> Vec<DisplayImage> {
109        sync::lock(&self.display).remove(lane).unwrap_or_default()
110    }
111
112    /// One `rlm.view_image` request of `lane`'s running cell.
113    pub fn attach(&self, lane: &str, payload: &Map<String, Value>) -> Result<Value, String> {
114        if !sync::lock(&self.cells).contains_key(lane) {
115            self.begin(lane);
116        }
117        let mut cells = sync::lock(&self.cells);
118        let images = cells.get_mut(lane).expect("begun");
119        images.attach(payload)
120    }
121
122    /// The images `lane`'s cell attached (its cell ended).
123    pub fn take(&self, lane: &str) -> Option<CellImages> {
124        sync::lock(&self.cells).remove(lane)
125    }
126}
127
128/// An image for the user only: the TUI draws it from the tool result's `details.displayImages`; the model's copy of
129/// the result has [`DisplayImage::line`] instead.
130#[derive(Clone, Debug, PartialEq, Eq)]
131pub struct DisplayImage {
132    pub png: Vec<u8>,
133    pub line: String,
134}
135
136/// A display-only image larger than this is not kept in the transcript (its line still is).
137pub const MAX_DISPLAY_IMAGE_BYTES: usize = 4 * 1024 * 1024;
138
139/// The result text followed by one line per display-only image.
140pub fn with_display_lines(text: String, shown: &[DisplayImage]) -> String {
141    shown.iter().fold(text, |text, image| format!("{text}\n{}", image.line))
142}
143
144/// `details` with the display-only images as `displayImages`: image blocks the TUI draws after the result's own
145/// content (providers never see `details`).
146pub fn with_display_details(mut details: Value, shown: &[DisplayImage]) -> Value {
147    let blocks: Vec<Value> = shown
148        .iter()
149        .filter(|image| image.png.len() <= MAX_DISPLAY_IMAGE_BYTES)
150        .map(|image| {
151            serde_json::json!({
152                "type": "image",
153                "data": jsrt::base64::encode(&image.png),
154                "mimeType": "image/png",
155            })
156        })
157        .collect();
158    if blocks.is_empty() {
159        return details;
160    }
161    if !details.is_object() {
162        details = serde_json::json!({});
163    }
164    details["displayImages"] = Value::Array(blocks);
165    details
166}
167
168/// The tool result content of a cell (`CellImages.content`): the text (with the non-vision note when images are
169/// attached), then one image block per image.
170pub fn cell_content(images: Option<&CellImages>, text: String) -> Vec<ai::UserContent> {
171    let Some(images) = images.filter(|i| !i.images.is_empty()) else {
172        return vec![crate::lane::text_block(text)];
173    };
174    let mut content = vec![crate::lane::text_block(images.text(&text))];
175    content.extend(images.images.iter().map(|image| {
176        ai::UserContent::Image(ai::ImageContent {
177            data: jsrt::base64::encode(&image.data),
178            mime_type: image.mime_type.to_string(),
179        })
180    }));
181    content
182}
183
184/// `rlm.view_image` answered for the lane's running cell; everything else goes on.
185pub struct ViewImageLayer {
186    pub inner: Arc<dyn HostHandler>,
187    pub images: Arc<ViewImages>,
188    pub lane: String,
189}
190
191impl HostHandler for ViewImageLayer {
192    fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
193        if kind == VIEW_IMAGE_REQUEST {
194            return self.images.attach(&self.lane, &payload);
195        }
196        self.inner.handle(kind, payload, signal)
197    }
198}
199
200pub const DEFAULT_CONTEXT_IMAGES: usize = 8;
201pub const OFFLOADED_IMAGE_TEXT: &str =
202    "[image omitted from context to stay within the image budget; call view_image again to see it]";
203
204/// `contextImageBudget`: `ULTRON_CONTEXT_IMAGES`, default 8; 0 disables the budget.
205pub fn context_image_budget(value: Option<&str>) -> usize {
206    let raw = value.map(jsrt::string::trim).unwrap_or_default();
207    if raw.is_empty() {
208        return DEFAULT_CONTEXT_IMAGES;
209    }
210    // `Number(raw)`: a whole number, possibly written with a fraction of zero, an exponent, or as a hex, octal or
211    // binary literal.
212    match jsrt::number::parse(raw) {
213        n if n.is_finite() && n >= 0.0 && n.fract() == 0.0 => n as usize,
214        _ => DEFAULT_CONTEXT_IMAGES,
215    }
216}
217
218/// `imagesToDrop`: how many of `total` images (oldest first) to drop under budget `max`.
219pub fn images_to_drop(total: usize, max: usize) -> usize {
220    if max == 0 || total <= max {
221        return 0;
222    }
223    let step = (max / 2).max(1);
224    total.min((total - step) / step * step)
225}
226
227/// `withImageBudget`: the messages with the oldest tool-result images replaced by a note; `None` when nothing
228/// changes.
229pub fn with_image_budget(messages: &[ai::Message], max: usize) -> Option<Vec<ai::Message>> {
230    let is_image = |c: &ai::UserContent| matches!(c, ai::UserContent::Image(_));
231    let total: usize = messages
232        .iter()
233        .map(|m| match m {
234            ai::Message::ToolResult(result) => result.content.iter().filter(|c| is_image(c)).count(),
235            _ => 0,
236        })
237        .sum();
238    let mut drop = images_to_drop(total, max);
239    if drop == 0 {
240        return None;
241    }
242    Some(
243        messages
244            .iter()
245            .map(|message| match message {
246                ai::Message::ToolResult(result) if drop > 0 && result.content.iter().any(is_image) => {
247                    let mut result = result.clone();
248                    for part in &mut result.content {
249                        if drop > 0 && is_image(part) {
250                            drop -= 1;
251                            *part = crate::lane::text_block(OFFLOADED_IMAGE_TEXT.to_string());
252                        }
253                    }
254                    ai::Message::ToolResult(result)
255                }
256                other => other.clone(),
257            })
258            .collect(),
259    )
260}
261
262/// The budget applied to a request's messages (`installImageBudget`: off at 0).
263pub fn apply_image_budget(messages: &mut Vec<ai::Message>, max: usize) {
264    if max == 0 {
265        return;
266    }
267    if let Some(kept) = with_image_budget(messages, max) {
268        *messages = kept;
269    }
270}