Skip to main content

kernel_apis/
images.rs

1//! `images.generate` / `images.models` (`rlm.images` of images_api.py): image generation through the ported OpenRouter
2//! images API ([`ai::images`], TS `api/openrouter-images.ts`). The host makes the requests, saves every image to a
3//! file and answers with the paths, so no image bytes reach the cell or the model's context.
4//!
5//! The model is the request's `model`, else the `images.model` setting, else [`DEFAULT_IMAGE_MODEL`]; a model is
6//! `provider/id` or a bare catalog id. `size` is an aspect ratio (`16:9`) or a resolution (`1K`, `2K`, `4K`), sent as
7//! OpenRouter's `image_config`. `images.baseUrl` replaces the provider's base URL (a proxy, or a test's fake).
8
9use crate::{ApiModule, Request};
10use ai::http::Transport;
11use ai::images::{
12    AssistantImages, ImagesContent, ImagesContext, ImagesModel, ImagesOptions, ImagesPayloadHook, ImagesStopReason,
13    image_model, image_models, image_providers,
14};
15use ai::types::TextContent;
16use base64::Engine;
17use serde_json::{Value, json};
18use std::path::{Path, PathBuf};
19use std::sync::Arc;
20use std::time::Duration;
21
22/// The image model when neither the request nor the `images.model` setting names one.
23pub const DEFAULT_IMAGE_MODEL: &str = "openrouter/google/gemini-2.5-flash-image";
24/// At most this many images per call.
25pub const MAX_IMAGES: u64 = 4;
26const RESOLUTIONS: [&str; 3] = ["1K", "2K", "4K"];
27
28/// The API key of an image provider: `Ok(None)` when none is configured.
29pub type KeyLookup = Arc<dyn Fn(&str) -> Result<Option<String>, String> + Send + Sync>;
30
31/// What the module needs from the session.
32#[derive(Clone)]
33pub struct ImagesConfig {
34    /// The `images.model` setting.
35    pub default_model: Option<String>,
36    /// The `images.baseUrl` setting.
37    pub base_url: Option<String>,
38    /// Where images go without `out` (the session's output directory).
39    pub out_dir: PathBuf,
40    /// What a relative `out` is relative to (the session's cwd).
41    pub cwd: PathBuf,
42    pub api_key: KeyLookup,
43    pub transport: Arc<dyn Transport>,
44}
45
46pub struct ImagesModule {
47    config: ImagesConfig,
48}
49
50impl ImagesModule {
51    pub fn new(config: ImagesConfig) -> Self {
52        Self { config }
53    }
54}
55
56/// `provider/id` of a catalog model.
57fn model_name(model: &ImagesModel) -> String {
58    format!("{}/{}", model.provider, model.id)
59}
60
61/// A catalog model by `provider/id` or by its bare id.
62fn resolve_model(name: &str) -> Result<&'static ImagesModel, String> {
63    let qualified = image_providers().find_map(|provider| {
64        name.strip_prefix(provider)
65            .and_then(|rest| rest.strip_prefix('/'))
66            .and_then(|id| image_model(provider, id))
67    });
68    qualified
69        .or_else(|| image_providers().find_map(|provider| image_model(provider, name)))
70        .ok_or_else(|| format!("Unknown image model: {name} (rlm.images.models() lists them)"))
71}
72
73/// OpenRouter's `image_config` for a `size`.
74fn image_config(size: &str) -> Result<Value, String> {
75    if RESOLUTIONS.contains(&size) {
76        return Ok(json!({ "image_size": size }));
77    }
78    let ratio = size.split_once(':').filter(|(w, h)| {
79        [w, h]
80            .iter()
81            .all(|part| !part.is_empty() && part.bytes().all(|b| b.is_ascii_digit()) && !part.starts_with('0'))
82    });
83    match ratio {
84        Some(_) => Ok(json!({ "aspect_ratio": size })),
85        None => Err(format!(
86            "Unsupported image size: {size}; use an aspect ratio such as \"16:9\" or one of 1K, 2K, 4K"
87        )),
88    }
89}
90
91/// The file extension of an image MIME type.
92fn extension(mime: &str) -> &'static str {
93    match mime {
94        "image/png" => "png",
95        "image/jpeg" | "image/jpg" => "jpg",
96        "image/webp" => "webp",
97        "image/gif" => "gif",
98        "image/svg+xml" => "svg",
99        _ => "img",
100    }
101}
102
103/// Where the images go: `count` paths for images of `mimes`. Without `out`, or with `out` an existing directory,
104/// `image-<k>.<ext>` with the first unused numbers; otherwise `out` itself (one image; its extension added when it has
105/// none) or `<stem>-<i>.<ext>` (several).
106fn output_paths(out: Option<&Path>, default_dir: &Path, mimes: &[&str]) -> Vec<PathBuf> {
107    let directory = match out {
108        None => Some(default_dir.to_path_buf()),
109        Some(path) if path.is_dir() => Some(path.to_path_buf()),
110        Some(_) => None,
111    };
112    if let Some(directory) = directory {
113        let mut next = 1;
114        return mimes
115            .iter()
116            .map(|mime| {
117                loop {
118                    let path = directory.join(format!("image-{next}.{}", extension(mime)));
119                    next += 1;
120                    if !path.exists() {
121                        break path;
122                    }
123                }
124            })
125            .collect();
126    }
127    let out = out.unwrap_or(default_dir);
128    let given = out.extension().map(|e| e.to_string_lossy().into_owned());
129    if mimes.len() == 1 {
130        return vec![match given {
131            Some(_) => out.to_path_buf(),
132            None => out.with_extension(extension(mimes[0])),
133        }];
134    }
135    let stem = out.with_extension("");
136    mimes
137        .iter()
138        .enumerate()
139        .map(|(index, mime)| {
140            let ext = given.clone().unwrap_or_else(|| extension(mime).to_string());
141            PathBuf::from(format!("{}-{}.{ext}", stem.display(), index + 1))
142        })
143        .collect()
144}
145
146fn text_field<'a>(request: &'a Request<'_>, key: &str) -> Result<Option<&'a str>, String> {
147    match request.payload.get(key) {
148        None | Some(Value::Null) => Ok(None),
149        Some(Value::String(text)) if !text.trim().is_empty() => Ok(Some(text)),
150        Some(_) => Err(format!("images.generate {key} must be a non-empty string")),
151    }
152}
153
154impl ImagesModule {
155    fn models(&self) -> Value {
156        Value::Array(
157            image_providers()
158                .flat_map(image_models)
159                .map(|model| json!({ "model": model_name(model), "name": model.name }))
160                .collect(),
161        )
162    }
163
164    fn generate(&self, request: &Request<'_>) -> Result<Value, String> {
165        kernel::ops::fields(request.kind, request.payload)?;
166        let prompt = text_field(request, "prompt")?.ok_or("images.generate prompt must be a non-empty string")?;
167        let count = match request.payload.get("n") {
168            None | Some(Value::Null) => 1,
169            Some(value) => value
170                .as_u64()
171                .filter(|n| (1..=MAX_IMAGES).contains(n))
172                .ok_or(format!("images.generate n must be an integer from 1 to {MAX_IMAGES}"))?,
173        } as usize;
174        let name = text_field(request, "model")?
175            .or(self.config.default_model.as_deref())
176            .unwrap_or(DEFAULT_IMAGE_MODEL);
177        let catalog = resolve_model(name)?;
178        let hook: Option<ImagesPayloadHook> = match text_field(request, "size")? {
179            Some(size) => {
180                let config = image_config(size)?;
181                Some(Arc::new(move |params: &Value, _: &ImagesModel| {
182                    let mut params = params.clone();
183                    params["image_config"] = config.clone();
184                    Some(params)
185                }))
186            }
187            None => None,
188        };
189        let out = text_field(request, "out")?.map(|out| self.config.cwd.join(out));
190        let api_key = (self.config.api_key)(&catalog.provider)?.ok_or_else(|| {
191            format!(
192                "No API key for {} image models: set OPENROUTER_API_KEY or log in with /login",
193                catalog.provider
194            )
195        })?;
196        let mut model = catalog.clone();
197        if let Some(base_url) = &self.config.base_url {
198            model.base_url = base_url.clone();
199        }
200        let images = self.request_images(&model, prompt, count, api_key, hook, request.signal)?;
201        let mimes: Vec<&str> = images.iter().map(|(mime, _)| mime.as_str()).collect();
202        let paths = output_paths(out.as_deref(), &self.config.out_dir, &mimes);
203        for (path, (_, bytes)) in paths.iter().zip(&images) {
204            if let Some(parent) = path.parent() {
205                std::fs::create_dir_all(parent).map_err(|e| format!("images.generate: {}: {e}", parent.display()))?;
206            }
207            std::fs::write(path, bytes).map_err(|e| format!("images.generate: {}: {e}", path.display()))?;
208        }
209        let paths: Vec<String> = paths.iter().map(|p| p.display().to_string()).collect();
210        Ok(json!({ "paths": paths, "model": model_name(&model) }))
211    }
212
213    /// One request after another until `count` images came back; the first failure fails the call.
214    fn request_images(
215        &self,
216        model: &ImagesModel,
217        prompt: &str,
218        count: usize,
219        api_key: String,
220        hook: Option<ImagesPayloadHook>,
221        signal: &kernel::Signal,
222    ) -> Result<Vec<(String, Vec<u8>)>, String> {
223        let abort = ai::AbortSignal::new();
224        let options = ImagesOptions {
225            api_key: Some(api_key),
226            signal: Some(abort.clone()),
227            on_payload: hook,
228            ..Default::default()
229        };
230        let context = ImagesContext {
231            input: vec![ImagesContent::Text(TextContent {
232                text: prompt.to_string(),
233                text_signature: None,
234            })],
235        };
236        // The cell's signal aborts the request (a cancelled or timed-out cell).
237        let (cell, done) = (signal.clone(), kernel::Signal::new());
238        let finished = done.clone();
239        let watcher = std::thread::spawn(move || {
240            while !finished.is_aborted() {
241                if cell.wait(Duration::from_millis(200)) {
242                    abort.abort();
243                    return;
244                }
245            }
246        });
247        let runtime = tokio::runtime::Builder::new_current_thread()
248            .enable_all()
249            .build()
250            .map_err(|e| e.to_string());
251        let result = runtime.and_then(|runtime| {
252            let mut images = Vec::new();
253            while images.len() < count {
254                let answer = runtime.block_on(ai::images::generate_images(
255                    model,
256                    &context,
257                    &options,
258                    self.config.transport.as_ref(),
259                ))?;
260                let decoded = decode(answer)?;
261                images.extend(decoded);
262            }
263            images.truncate(count);
264            Ok(images)
265        });
266        done.abort("done");
267        let _ = watcher.join();
268        result
269    }
270}
271
272/// The images of one answer, decoded; an error answer, or one without images, is the call's error.
273fn decode(answer: AssistantImages) -> Result<Vec<(String, Vec<u8>)>, String> {
274    match answer.stop_reason {
275        ImagesStopReason::Aborted => return Err("images.generate was cancelled".into()),
276        ImagesStopReason::Error => {
277            return Err(format!(
278                "Image generation failed: {}",
279                answer.error_message.unwrap_or_else(|| "unknown error".into())
280            ));
281        }
282        ImagesStopReason::Stop => {}
283    }
284    let mut images = Vec::new();
285    let mut text = Vec::new();
286    for item in answer.output {
287        match item {
288            ImagesContent::Image(image) => {
289                let bytes = base64::engine::general_purpose::STANDARD
290                    .decode(image.data.as_bytes())
291                    .map_err(|e| format!("Image generation failed: the image is not valid base64: {e}"))?;
292                images.push((image.mime_type, bytes));
293            }
294            ImagesContent::Text(part) => text.push(part.text),
295        }
296    }
297    if images.is_empty() {
298        let said = text.join(" ");
299        return Err(if said.trim().is_empty() {
300            format!("Image generation failed: {} returned no image", answer.model)
301        } else {
302            format!("Image generation failed: {} returned no image: {said}", answer.model)
303        });
304    }
305    Ok(images)
306}
307
308impl ApiModule for ImagesModule {
309    fn prefixes(&self) -> &[&str] {
310        &["images."]
311    }
312
313    fn handle(&self, request: &Request<'_>) -> Result<Value, String> {
314        match request.kind {
315            "images.generate" => self.generate(request),
316            "images.models" => Ok(self.models()),
317            other => Err(format!("Unknown images request: {other}")),
318        }
319    }
320}