systemprompt_api/services/gateway/image_fetch/
mod.rs

1//! Resolving caller-supplied image URLs to inline base64 for wires that
2//! cannot carry a URL.
3//!
4//! Gemini's `generateContent` has no URL image part at all — `inlineData` or a
5//! Files API handle are the only shapes it accepts — so the wire codec, which
6//! is synchronous and has no HTTP client, can only downgrade a URL image to
7//! text. This module does the fetch one layer up, in the dispatch pipeline,
8//! before the body is built, and rewrites the canonical request in place so
9//! the codec sees an image it can render.
10//!
11//! It is deliberately not a general-purpose fetcher. The URL comes from
12//! whoever sent the inference request, so every fetch is guarded by `guard`,
13//! bounded by a timeout, capped while the body streams, and accepted only if
14//! the server declares a MIME type Gemini takes.
15//!
16//! Copyright (c) systemprompt.io — Business Source License 1.1.
17//! See <https://systemprompt.io> for licensing details.
18
19mod guard;
20
21use base64::Engine as _;
22use base64::engine::general_purpose::STANDARD as BASE64;
23use systemprompt_client::GuardedConnectError;
24use systemprompt_models::net::trusted_http_hosts_from_env;
25
26use super::protocol::canonical::{CanonicalContent, CanonicalRequest, ImageSource};
27
28// Why: Gemini limits inline generateContent requests to 20 MB; base64 expands
29// bytes by roughly 4/3.
30pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024;
31
32pub const ACCEPTED_MIME: [&str; 5] = [
33    "image/png",
34    "image/jpeg",
35    "image/webp",
36    "image/heic",
37    "image/heif",
38];
39
40const MAX_REDIRECTS: u8 = 3;
41const FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
42
43/// A caller-supplied image URL that could not be turned into inline data.
44///
45/// `caller_fault` separates "this URL was never going to work" — blocked host,
46/// wrong content type, too large — from a transport failure reaching an
47/// otherwise legitimate host, so the route layer can answer 400 or 502.
48#[derive(Debug, thiserror::Error)]
49#[error("image url {url} could not be inlined: {message}")]
50pub struct ImageFetchFailed {
51    pub url: String,
52    pub message: String,
53    pub caller_fault: bool,
54}
55
56/// Per-request bounds, so a test can point the fetcher at a loopback mock
57/// without the process-wide trust list that production reads from the
58/// environment.
59#[derive(Debug, Clone)]
60pub struct ImageFetchPolicy {
61    pub timeout: std::time::Duration,
62    pub max_bytes: usize,
63    pub max_redirects: u8,
64    pub trusted_hosts: Vec<String>,
65}
66
67impl Default for ImageFetchPolicy {
68    fn default() -> Self {
69        Self {
70            timeout: FETCH_TIMEOUT,
71            max_bytes: MAX_IMAGE_BYTES,
72            max_redirects: MAX_REDIRECTS,
73            trusted_hosts: trusted_http_hosts_from_env(),
74        }
75    }
76}
77
78/// Fetched bytes plus the MIME type the server declared for them.
79#[derive(Debug, Clone)]
80pub struct InlineImage {
81    pub media_type: String,
82    pub base64: String,
83}
84
85pub async fn inline_url_images(
86    request: &mut CanonicalRequest,
87    policy: &ImageFetchPolicy,
88) -> Result<usize, ImageFetchFailed> {
89    let mut count = 0usize;
90    for message in &mut request.messages {
91        for content in &mut message.content {
92            let CanonicalContent::Image(ImageSource::Url { url, detail }) = content else {
93                continue;
94            };
95            let fetched = fetch(url, policy).await?;
96            *content = CanonicalContent::Image(ImageSource::Base64 {
97                media_type: fetched.media_type,
98                data: fetched.base64,
99                detail: *detail,
100            });
101            count += 1;
102        }
103    }
104    Ok(count)
105}
106
107pub async fn fetch(url: &str, policy: &ImageFetchPolicy) -> Result<InlineImage, ImageFetchFailed> {
108    let fail = |message: String, caller_fault: bool| ImageFetchFailed {
109        url: url.to_owned(),
110        message,
111        caller_fault,
112    };
113    tokio::time::timeout(policy.timeout, fetch_inner(url, policy))
114        .await
115        .map_or_else(
116            |_| Err(fail(format!("fetch exceeded {:?}", policy.timeout), false)),
117            |result| result.map_err(|(message, caller_fault)| fail(message, caller_fault)),
118        )
119}
120
121type FetchError = (String, bool);
122
123async fn fetch_inner(url: &str, policy: &ImageFetchPolicy) -> Result<InlineImage, FetchError> {
124    let checked = guard::checked_url(url, &policy.trusted_hosts).map_err(|e| (e, true))?;
125    let client = guard::client(policy).map_err(|e| (e, false))?;
126    let response = client
127        .get(checked)
128        .send()
129        .await
130        .map_err(|e| describe_send_error(&e, policy))?;
131    read_image(response, policy).await
132}
133
134// Why: reqwest reports a refused redirect and a timeout as generic send
135// failures; the guard's verdict and the deadline sit in the source chain.
136fn describe_send_error(error: &reqwest::Error, policy: &ImageFetchPolicy) -> FetchError {
137    if error.is_timeout() {
138        return (format!("fetch exceeded {:?}", policy.timeout), false);
139    }
140    let mut source = std::error::Error::source(error);
141    while let Some(inner) = source {
142        if let Some(guarded) = inner.downcast_ref::<GuardedConnectError>() {
143            return match guarded {
144                GuardedConnectError::RedirectRefused { .. }
145                | GuardedConnectError::TooManyRedirects(_) => {
146                    (format!("redirect rejected: {guarded}"), true)
147                },
148                GuardedConnectError::Unresolvable(_)
149                | GuardedConnectError::BlockedAddress { .. } => {
150                    (format!("host rejected: {guarded}"), true)
151                },
152            };
153        }
154        source = inner.source();
155    }
156    (format!("request failed: {error}"), error.is_redirect())
157}
158
159async fn read_image(
160    mut response: reqwest::Response,
161    policy: &ImageFetchPolicy,
162) -> Result<InlineImage, FetchError> {
163    let status = response.status();
164    if !status.is_success() {
165        return Err((format!("host returned {status}"), true));
166    }
167    let media_type = declared_mime(&response)?;
168    let mut body: Vec<u8> = Vec::new();
169    while let Some(chunk) = response
170        .chunk()
171        .await
172        .map_err(|e| (format!("read failed: {e}"), false))?
173    {
174        if body.len() + chunk.len() > policy.max_bytes {
175            return Err((format!("larger than {} bytes", policy.max_bytes), true));
176        }
177        body.extend_from_slice(&chunk);
178    }
179    if body.is_empty() {
180        return Err(("empty response body".to_owned(), true));
181    }
182    Ok(InlineImage {
183        media_type,
184        base64: BASE64.encode(&body),
185    })
186}
187
188fn declared_mime(response: &reqwest::Response) -> Result<String, FetchError> {
189    let raw = response
190        .headers()
191        .get(reqwest::header::CONTENT_TYPE)
192        .and_then(|v| v.to_str().ok())
193        .ok_or_else(|| ("no content-type".to_owned(), true))?;
194    let mime = raw
195        .split(';')
196        .next()
197        .unwrap_or_default()
198        .trim()
199        .to_ascii_lowercase();
200    if ACCEPTED_MIME.contains(&mime.as_str()) {
201        return Ok(mime);
202    }
203    Err((
204        format!("content-type {mime} is not an inlineable image"),
205        true,
206    ))
207}
systemprompt_api/services/gateway/image_fetch/mod.rs

systemprompt_api/services/gateway/image_fetch/
mod.rs
