ultron-session-core 1.0.0

One session's runtime, shared by every mode through the session worker: the rlm lane (model, provider, system prompt, the single rlm tool), the composition root of its host stack, the format-4 run journal, session naming, Pi read views, and the session actor
Documentation
//! `view_image` in the session (`ultron/rlm/view-image.ts` as session-worker.ts wires it): a cell's
//! `rlm.view_image` requests are answered by the cell's own image collector ([`host::view_image::CellImages`],
//! normalized for the lane's current model and Pi's `images.autoResize` setting), and the `rlm` tool result carries the
//! images as image blocks after its text. The image budget (`ultron/image-budget.ts`, `ULTRON_CONTEXT_IMAGES`) keeps
//! the oldest tool-result images of a request out of it as a note.

use ai::Model;
use host::view_image::{CellImages, ImageModel, ResizeLimits, VIEW_IMAGE_REQUEST, ViewImageOptions};
use jsrt::sync;
use kernel::{HostHandler, Signal};
use serde_json::{Map, Value};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};

pub const CONTEXT_IMAGES_ENV: &str = "ULTRON_CONTEXT_IMAGES";

/// The model a lane's images go to, as the host's normalizer reads it (`input`, `inputLimits.images.resize`).
pub fn image_model(model: &Model) -> ImageModel {
    let resize = model
        .input_limits
        .as_ref()
        .and_then(|limits| limits["images"]["resize"].as_object().cloned());
    ImageModel {
        provider: model.provider.clone(),
        id: model.id.clone(),
        accepts_images: model.input.iter().any(|i| i == "image"),
        resize: resize.map(|r| ResizeLimits {
            max_width: r.get("maxWidth").and_then(Value::as_u64).map(|n| n as u32),
            max_height: r.get("maxHeight").and_then(Value::as_u64).map(|n| n as u32),
            max_bytes: r.get("maxBytes").and_then(Value::as_f64),
            jpeg_quality: r.get("jpegQuality").and_then(Value::as_u64).map(|n| n as u8),
        }),
    }
}

/// Every lane's images of the cell it runs now, and the model each lane's results go to.
pub struct ViewImages {
    cells: Mutex<HashMap<String, CellImages>>,
    /// Images each lane's running cell shows the user but not the model (`diagram.mermaid` without `to_model`).
    display: Mutex<HashMap<String, Vec<DisplayImage>>>,
    models: Mutex<HashMap<String, Model>>,
    /// The session's model: a lane without a model of its own sends its results there.
    default_model: Mutex<Model>,
    /// Where `images.autoResize` is read (per cell, as TypeScript reads it).
    cwd: PathBuf,
    agent_dir: PathBuf,
    /// The worker's project trust: a trusted project's `.pi/settings.json` is read too.
    project_trusted: bool,
}

impl ViewImages {
    pub fn new(model: Model, cwd: PathBuf, agent_dir: PathBuf, project_trusted: bool) -> ViewImages {
        ViewImages {
            cells: Mutex::default(),
            display: Mutex::default(),
            models: Mutex::default(),
            default_model: Mutex::new(model),
            cwd,
            agent_dir,
            project_trusted,
        }
    }

    /// The model `lane`'s results go to from now on (`None`: the root lane, whose model is the session's).
    pub fn set_model(&self, lane: Option<&str>, model: Model) {
        match lane {
            Some(lane) => {
                sync::lock(&self.models).insert(lane.to_string(), model);
            }
            None => *sync::lock(&self.default_model) = model,
        }
    }

    /// Pi's `images.autoResize` (default on).
    fn auto_resize(&self) -> bool {
        let settings = config::settings::SettingsManager::create(&self.cwd, &self.agent_dir, self.project_trusted);
        settings.settings().get("images").and_then(|i| i.get("autoResize")) != Some(&Value::Bool(false))
    }

    fn options(&self, lane: &str) -> ViewImageOptions {
        let model = sync::lock(&self.models)
            .get(lane)
            .cloned()
            .unwrap_or_else(|| sync::lock(&self.default_model).clone());
        ViewImageOptions {
            auto_resize: self.auto_resize(),
            model: Some(image_model(&model)),
        }
    }

    /// A cell starts on `lane`: its images start empty.
    pub fn begin(&self, lane: &str) {
        let options = self.options(lane);
        sync::lock(&self.cells).insert(lane.to_string(), CellImages::new(options));
        sync::lock(&self.display).remove(lane);
    }

    /// A PNG `lane`'s running cell shows the user only; `line` is what the model reads instead.
    pub fn display(&self, lane: &str, png: Vec<u8>, line: String) {
        sync::lock(&self.display)
            .entry(lane.to_string())
            .or_default()
            .push(DisplayImage { png, line });
    }

    /// The display-only images of `lane`'s cell (its cell ended).
    pub fn take_display(&self, lane: &str) -> Vec<DisplayImage> {
        sync::lock(&self.display).remove(lane).unwrap_or_default()
    }

    /// One `rlm.view_image` request of `lane`'s running cell.
    pub fn attach(&self, lane: &str, payload: &Map<String, Value>) -> Result<Value, String> {
        if !sync::lock(&self.cells).contains_key(lane) {
            self.begin(lane);
        }
        let mut cells = sync::lock(&self.cells);
        let images = cells.get_mut(lane).expect("begun");
        images.attach(payload)
    }

    /// The images `lane`'s cell attached (its cell ended).
    pub fn take(&self, lane: &str) -> Option<CellImages> {
        sync::lock(&self.cells).remove(lane)
    }
}

/// An image for the user only: the TUI draws it from the tool result's `details.displayImages`; the model's copy of
/// the result has [`DisplayImage::line`] instead.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DisplayImage {
    pub png: Vec<u8>,
    pub line: String,
}

/// A display-only image larger than this is not kept in the transcript (its line still is).
pub const MAX_DISPLAY_IMAGE_BYTES: usize = 4 * 1024 * 1024;

/// The result text followed by one line per display-only image.
pub fn with_display_lines(text: String, shown: &[DisplayImage]) -> String {
    shown.iter().fold(text, |text, image| format!("{text}\n{}", image.line))
}

/// `details` with the display-only images as `displayImages`: image blocks the TUI draws after the result's own
/// content (providers never see `details`).
pub fn with_display_details(mut details: Value, shown: &[DisplayImage]) -> Value {
    let blocks: Vec<Value> = shown
        .iter()
        .filter(|image| image.png.len() <= MAX_DISPLAY_IMAGE_BYTES)
        .map(|image| {
            serde_json::json!({
                "type": "image",
                "data": jsrt::base64::encode(&image.png),
                "mimeType": "image/png",
            })
        })
        .collect();
    if blocks.is_empty() {
        return details;
    }
    if !details.is_object() {
        details = serde_json::json!({});
    }
    details["displayImages"] = Value::Array(blocks);
    details
}

/// The tool result content of a cell (`CellImages.content`): the text (with the non-vision note when images are
/// attached), then one image block per image.
pub fn cell_content(images: Option<&CellImages>, text: String) -> Vec<ai::UserContent> {
    let Some(images) = images.filter(|i| !i.images.is_empty()) else {
        return vec![crate::lane::text_block(text)];
    };
    let mut content = vec![crate::lane::text_block(images.text(&text))];
    content.extend(images.images.iter().map(|image| {
        ai::UserContent::Image(ai::ImageContent {
            data: jsrt::base64::encode(&image.data),
            mime_type: image.mime_type.to_string(),
        })
    }));
    content
}

/// `rlm.view_image` answered for the lane's running cell; everything else goes on.
pub struct ViewImageLayer {
    pub inner: Arc<dyn HostHandler>,
    pub images: Arc<ViewImages>,
    pub lane: String,
}

impl HostHandler for ViewImageLayer {
    fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
        if kind == VIEW_IMAGE_REQUEST {
            return self.images.attach(&self.lane, &payload);
        }
        self.inner.handle(kind, payload, signal)
    }
}

pub const DEFAULT_CONTEXT_IMAGES: usize = 8;
pub const OFFLOADED_IMAGE_TEXT: &str =
    "[image omitted from context to stay within the image budget; call view_image again to see it]";

/// `contextImageBudget`: `ULTRON_CONTEXT_IMAGES`, default 8; 0 disables the budget.
pub fn context_image_budget(value: Option<&str>) -> usize {
    let raw = value.map(jsrt::string::trim).unwrap_or_default();
    if raw.is_empty() {
        return DEFAULT_CONTEXT_IMAGES;
    }
    // `Number(raw)`: a whole number, possibly written with a fraction of zero, an exponent, or as a hex, octal or
    // binary literal.
    match jsrt::number::parse(raw) {
        n if n.is_finite() && n >= 0.0 && n.fract() == 0.0 => n as usize,
        _ => DEFAULT_CONTEXT_IMAGES,
    }
}

/// `imagesToDrop`: how many of `total` images (oldest first) to drop under budget `max`.
pub fn images_to_drop(total: usize, max: usize) -> usize {
    if max == 0 || total <= max {
        return 0;
    }
    let step = (max / 2).max(1);
    total.min((total - step) / step * step)
}

/// `withImageBudget`: the messages with the oldest tool-result images replaced by a note; `None` when nothing
/// changes.
pub fn with_image_budget(messages: &[ai::Message], max: usize) -> Option<Vec<ai::Message>> {
    let is_image = |c: &ai::UserContent| matches!(c, ai::UserContent::Image(_));
    let total: usize = messages
        .iter()
        .map(|m| match m {
            ai::Message::ToolResult(result) => result.content.iter().filter(|c| is_image(c)).count(),
            _ => 0,
        })
        .sum();
    let mut drop = images_to_drop(total, max);
    if drop == 0 {
        return None;
    }
    Some(
        messages
            .iter()
            .map(|message| match message {
                ai::Message::ToolResult(result) if drop > 0 && result.content.iter().any(is_image) => {
                    let mut result = result.clone();
                    for part in &mut result.content {
                        if drop > 0 && is_image(part) {
                            drop -= 1;
                            *part = crate::lane::text_block(OFFLOADED_IMAGE_TEXT.to_string());
                        }
                    }
                    ai::Message::ToolResult(result)
                }
                other => other.clone(),
            })
            .collect(),
    )
}

/// The budget applied to a request's messages (`installImageBudget`: off at 0).
pub fn apply_image_budget(messages: &mut Vec<ai::Message>, max: usize) {
    if max == 0 {
        return;
    }
    if let Some(kept) = with_image_budget(messages, max) {
        *messages = kept;
    }
}